dash/src/evo/deterministicmns.h

712 lines
23 KiB
C
Raw Normal View History

// Copyright (c) 2018-2021 The Dash Core developers
// Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
#ifndef BITCOIN_EVO_DETERMINISTICMNS_H
#define BITCOIN_EVO_DETERMINISTICMNS_H
Backport 11651 (#3358) * scripted-diff: Replace #include "" with #include <> (ryanofsky) -BEGIN VERIFY SCRIPT- for f in \ src/*.cpp \ src/*.h \ src/bench/*.cpp \ src/bench/*.h \ src/compat/*.cpp \ src/compat/*.h \ src/consensus/*.cpp \ src/consensus/*.h \ src/crypto/*.cpp \ src/crypto/*.h \ src/crypto/ctaes/*.h \ src/policy/*.cpp \ src/policy/*.h \ src/primitives/*.cpp \ src/primitives/*.h \ src/qt/*.cpp \ src/qt/*.h \ src/qt/test/*.cpp \ src/qt/test/*.h \ src/rpc/*.cpp \ src/rpc/*.h \ src/script/*.cpp \ src/script/*.h \ src/support/*.cpp \ src/support/*.h \ src/support/allocators/*.h \ src/test/*.cpp \ src/test/*.h \ src/wallet/*.cpp \ src/wallet/*.h \ src/wallet/test/*.cpp \ src/wallet/test/*.h \ src/zmq/*.cpp \ src/zmq/*.h do base=${f%/*}/ relbase=${base#src/} sed -i "s:#include \"\(.*\)\"\(.*\):if test -e \$base'\\1'; then echo \"#include <\"\$relbase\"\\1>\\2\"; else echo \"#include <\\1>\\2\"; fi:e" $f done -END VERIFY SCRIPT- Signed-off-by: Pasta <pasta@dashboost.org> * scripted-diff: Replace #include "" with #include <> (Dash Specific) -BEGIN VERIFY SCRIPT- for f in \ src/bls/*.cpp \ src/bls/*.h \ src/evo/*.cpp \ src/evo/*.h \ src/governance/*.cpp \ src/governance/*.h \ src/llmq/*.cpp \ src/llmq/*.h \ src/masternode/*.cpp \ src/masternode/*.h \ src/privatesend/*.cpp \ src/privatesend/*.h do base=${f%/*}/ relbase=${base#src/} sed -i "s:#include \"\(.*\)\"\(.*\):if test -e \$base'\\1'; then echo \"#include <\"\$relbase\"\\1>\\2\"; else echo \"#include <\\1>\\2\"; fi:e" $f done -END VERIFY SCRIPT- Signed-off-by: Pasta <pasta@dashboost.org> * build: Remove -I for everything but project root Remove -I from build system for everything but the project root, and built-in dependencies. Signed-off-by: Pasta <pasta@dashboost.org> # Conflicts: # src/Makefile.test.include * qt: refactor: Use absolute include paths in .ui files * qt: refactor: Changes to make include paths absolute This makes all include paths in the GUI absolute. Many changes are involved as every single source file in src/qt/ assumes to be able to use relative includes. Signed-off-by: Pasta <pasta@dashboost.org> # Conflicts: # src/qt/dash.cpp # src/qt/optionsmodel.cpp # src/qt/test/rpcnestedtests.cpp * test: refactor: Use absolute include paths for test data files * Recommend #include<> syntax in developer notes * refactor: Include obj/build.h instead of build.h * END BACKPORT #11651 Remove trailing whitespace causing travis failure * fix backport 11651 Signed-off-by: Pasta <pasta@dashboost.org> * More of 11651 * fix blockchain.cpp Signed-off-by: pasta <pasta@dashboost.org> * Add missing "qt/" in includes * Add missing "test/" in includes * Fix trailing whitespaces Co-authored-by: Wladimir J. van der Laan <laanwj@gmail.com> Co-authored-by: Russell Yanofsky <russ@yanofsky.org> Co-authored-by: MeshCollider <dobsonsa68@gmail.com> Co-authored-by: UdjinM6 <UdjinM6@users.noreply.github.com>
2020-03-19 23:46:56 +01:00
#include <arith_uint256.h>
#include <evo/evodb.h>
#include <evo/providertx.h>
2020-06-09 06:43:34 +02:00
#include <saltedhasher.h>
Backport 11651 (#3358) * scripted-diff: Replace #include "" with #include <> (ryanofsky) -BEGIN VERIFY SCRIPT- for f in \ src/*.cpp \ src/*.h \ src/bench/*.cpp \ src/bench/*.h \ src/compat/*.cpp \ src/compat/*.h \ src/consensus/*.cpp \ src/consensus/*.h \ src/crypto/*.cpp \ src/crypto/*.h \ src/crypto/ctaes/*.h \ src/policy/*.cpp \ src/policy/*.h \ src/primitives/*.cpp \ src/primitives/*.h \ src/qt/*.cpp \ src/qt/*.h \ src/qt/test/*.cpp \ src/qt/test/*.h \ src/rpc/*.cpp \ src/rpc/*.h \ src/script/*.cpp \ src/script/*.h \ src/support/*.cpp \ src/support/*.h \ src/support/allocators/*.h \ src/test/*.cpp \ src/test/*.h \ src/wallet/*.cpp \ src/wallet/*.h \ src/wallet/test/*.cpp \ src/wallet/test/*.h \ src/zmq/*.cpp \ src/zmq/*.h do base=${f%/*}/ relbase=${base#src/} sed -i "s:#include \"\(.*\)\"\(.*\):if test -e \$base'\\1'; then echo \"#include <\"\$relbase\"\\1>\\2\"; else echo \"#include <\\1>\\2\"; fi:e" $f done -END VERIFY SCRIPT- Signed-off-by: Pasta <pasta@dashboost.org> * scripted-diff: Replace #include "" with #include <> (Dash Specific) -BEGIN VERIFY SCRIPT- for f in \ src/bls/*.cpp \ src/bls/*.h \ src/evo/*.cpp \ src/evo/*.h \ src/governance/*.cpp \ src/governance/*.h \ src/llmq/*.cpp \ src/llmq/*.h \ src/masternode/*.cpp \ src/masternode/*.h \ src/privatesend/*.cpp \ src/privatesend/*.h do base=${f%/*}/ relbase=${base#src/} sed -i "s:#include \"\(.*\)\"\(.*\):if test -e \$base'\\1'; then echo \"#include <\"\$relbase\"\\1>\\2\"; else echo \"#include <\\1>\\2\"; fi:e" $f done -END VERIFY SCRIPT- Signed-off-by: Pasta <pasta@dashboost.org> * build: Remove -I for everything but project root Remove -I from build system for everything but the project root, and built-in dependencies. Signed-off-by: Pasta <pasta@dashboost.org> # Conflicts: # src/Makefile.test.include * qt: refactor: Use absolute include paths in .ui files * qt: refactor: Changes to make include paths absolute This makes all include paths in the GUI absolute. Many changes are involved as every single source file in src/qt/ assumes to be able to use relative includes. Signed-off-by: Pasta <pasta@dashboost.org> # Conflicts: # src/qt/dash.cpp # src/qt/optionsmodel.cpp # src/qt/test/rpcnestedtests.cpp * test: refactor: Use absolute include paths for test data files * Recommend #include<> syntax in developer notes * refactor: Include obj/build.h instead of build.h * END BACKPORT #11651 Remove trailing whitespace causing travis failure * fix backport 11651 Signed-off-by: Pasta <pasta@dashboost.org> * More of 11651 * fix blockchain.cpp Signed-off-by: pasta <pasta@dashboost.org> * Add missing "qt/" in includes * Add missing "test/" in includes * Fix trailing whitespaces Co-authored-by: Wladimir J. van der Laan <laanwj@gmail.com> Co-authored-by: Russell Yanofsky <russ@yanofsky.org> Co-authored-by: MeshCollider <dobsonsa68@gmail.com> Co-authored-by: UdjinM6 <UdjinM6@users.noreply.github.com>
2020-03-19 23:46:56 +01:00
#include <sync.h>
Backport 11651 (#3358) * scripted-diff: Replace #include "" with #include <> (ryanofsky) -BEGIN VERIFY SCRIPT- for f in \ src/*.cpp \ src/*.h \ src/bench/*.cpp \ src/bench/*.h \ src/compat/*.cpp \ src/compat/*.h \ src/consensus/*.cpp \ src/consensus/*.h \ src/crypto/*.cpp \ src/crypto/*.h \ src/crypto/ctaes/*.h \ src/policy/*.cpp \ src/policy/*.h \ src/primitives/*.cpp \ src/primitives/*.h \ src/qt/*.cpp \ src/qt/*.h \ src/qt/test/*.cpp \ src/qt/test/*.h \ src/rpc/*.cpp \ src/rpc/*.h \ src/script/*.cpp \ src/script/*.h \ src/support/*.cpp \ src/support/*.h \ src/support/allocators/*.h \ src/test/*.cpp \ src/test/*.h \ src/wallet/*.cpp \ src/wallet/*.h \ src/wallet/test/*.cpp \ src/wallet/test/*.h \ src/zmq/*.cpp \ src/zmq/*.h do base=${f%/*}/ relbase=${base#src/} sed -i "s:#include \"\(.*\)\"\(.*\):if test -e \$base'\\1'; then echo \"#include <\"\$relbase\"\\1>\\2\"; else echo \"#include <\\1>\\2\"; fi:e" $f done -END VERIFY SCRIPT- Signed-off-by: Pasta <pasta@dashboost.org> * scripted-diff: Replace #include "" with #include <> (Dash Specific) -BEGIN VERIFY SCRIPT- for f in \ src/bls/*.cpp \ src/bls/*.h \ src/evo/*.cpp \ src/evo/*.h \ src/governance/*.cpp \ src/governance/*.h \ src/llmq/*.cpp \ src/llmq/*.h \ src/masternode/*.cpp \ src/masternode/*.h \ src/privatesend/*.cpp \ src/privatesend/*.h do base=${f%/*}/ relbase=${base#src/} sed -i "s:#include \"\(.*\)\"\(.*\):if test -e \$base'\\1'; then echo \"#include <\"\$relbase\"\\1>\\2\"; else echo \"#include <\\1>\\2\"; fi:e" $f done -END VERIFY SCRIPT- Signed-off-by: Pasta <pasta@dashboost.org> * build: Remove -I for everything but project root Remove -I from build system for everything but the project root, and built-in dependencies. Signed-off-by: Pasta <pasta@dashboost.org> # Conflicts: # src/Makefile.test.include * qt: refactor: Use absolute include paths in .ui files * qt: refactor: Changes to make include paths absolute This makes all include paths in the GUI absolute. Many changes are involved as every single source file in src/qt/ assumes to be able to use relative includes. Signed-off-by: Pasta <pasta@dashboost.org> # Conflicts: # src/qt/dash.cpp # src/qt/optionsmodel.cpp # src/qt/test/rpcnestedtests.cpp * test: refactor: Use absolute include paths for test data files * Recommend #include<> syntax in developer notes * refactor: Include obj/build.h instead of build.h * END BACKPORT #11651 Remove trailing whitespace causing travis failure * fix backport 11651 Signed-off-by: Pasta <pasta@dashboost.org> * More of 11651 * fix blockchain.cpp Signed-off-by: pasta <pasta@dashboost.org> * Add missing "qt/" in includes * Add missing "test/" in includes * Fix trailing whitespaces Co-authored-by: Wladimir J. van der Laan <laanwj@gmail.com> Co-authored-by: Russell Yanofsky <russ@yanofsky.org> Co-authored-by: MeshCollider <dobsonsa68@gmail.com> Co-authored-by: UdjinM6 <UdjinM6@users.noreply.github.com>
2020-03-19 23:46:56 +01:00
#include <immer/map.hpp>
2020-06-09 06:43:34 +02:00
#include <unordered_map>
class CBlock;
class CBlockIndex;
class CValidationState;
class CSimplifiedMNListDiff;
namespace llmq
{
class CFinalCommitment;
} // namespace llmq
class CDeterministicMNState
{
private:
int nPoSeBanHeight{-1};
friend class CDeterministicMNStateDiff;
public:
int nRegisteredHeight{-1};
int nLastPaidHeight{0};
2018-08-31 11:33:37 +02:00
int nPoSePenalty{0};
int nPoSeRevivedHeight{-1};
uint16_t nRevocationReason{CProUpRevTx::REASON_NOT_SPECIFIED};
// the block hash X blocks after registration, used in quorum calculations
uint256 confirmedHash;
// sha256(proTxHash, confirmedHash) to speed up quorum calculations
// please note that this is NOT a double-sha256 hash
uint256 confirmedHashWithProRegTxHash;
CKeyID keyIDOwner;
CBLSLazyPublicKey pubKeyOperator;
CKeyID keyIDVoting;
CService addr;
CScript scriptPayout;
CScript scriptOperatorPayout;
public:
CDeterministicMNState() = default;
2020-01-05 01:55:41 +01:00
explicit CDeterministicMNState(const CProRegTx& proTx)
{
keyIDOwner = proTx.keyIDOwner;
pubKeyOperator.Set(proTx.pubKeyOperator);
keyIDVoting = proTx.keyIDVoting;
addr = proTx.addr;
scriptPayout = proTx.scriptPayout;
}
template <typename Stream>
CDeterministicMNState(deserialize_type, Stream& s)
{
s >> *this;
}
ADD_SERIALIZE_METHODS;
template <typename Stream, typename Operation>
inline void SerializationOp(Stream& s, Operation ser_action)
{
READWRITE(nRegisteredHeight);
READWRITE(nLastPaidHeight);
2018-08-31 11:33:37 +02:00
READWRITE(nPoSePenalty);
READWRITE(nPoSeRevivedHeight);
READWRITE(nPoSeBanHeight);
READWRITE(nRevocationReason);
READWRITE(confirmedHash);
READWRITE(confirmedHashWithProRegTxHash);
READWRITE(keyIDOwner);
READWRITE(pubKeyOperator);
READWRITE(keyIDVoting);
READWRITE(addr);
Optimize on-disk deterministic masternode storage to reduce size of evodb (#3017) * Implement CompactFull() in CDBWrapper This allows to compact the whole DB in one go. * Implement more compact version of CDeterministicMNListDiff This introduces CDeterministicMNStateDiff which requires to only store fields on-disk which actually changed. * Avoid writing mnUniquePropertyMap to disk when storing snapshots This map can be rebuilt by simply using AddMN for each deserialized MN. * Implement Serialize/Unserialize in CScript This allows us to directly use READWRITE() on scripts and removes the need for the ugly cast to CScriptBase. This commit also changes all Dash specific uses of CScript to not use the cast. * Keep track of registeration counts and introduce internalID for masternodes The "internalId" is simply the number of MNs registered so far when the new MN is added. It is deterministic and stays the same forever. * Use internalId as keys in MN list diffs This reduces the used size on-disk. * Two simple speedups in MN list diff handling 1. Avoid full compare if dmn or state pointers match in BuildDiff 2. Use std::move when adding diff to listDiff in GetListForBlock * Implement upgrade code for old CDeterministicMNListDiff format to new format * Track tipIndex instead of tipHeight/tipBlockHash * Store and pass around CBlockIndex* instead of block hash and height This allows us to switch CDeterministicMNManager::GetListForBlock to work with CBlockIndex. * Refactor CDeterministicMNManager::GetListForBlock to require CBlockIndex* Instead of requiring a block hash. This allows us to remove blockHash and prevBlockHash from CDeterministicMNListDiff without the use of cs_main locks in GetListForBlock. * Remove prevBlockHash, blockHash and nHeight from CDeterministicMNListDiff * Remove access to determinisitcMNManager in CMasternodeMetaMan::ToString() The deterministic MN manager is not fully initialized yet at the time this is called, which results in an empty list being returned everytime. * Better logic to determine if an upgrade is needed Reuse the "best block" logic to figure out if an upgrade is needed. Also use it to ensure that older nodes are unable to start after the upgrade was performed. * Return null block hash if it was requested with getmnlistdiff * bump CGovernanceManager::SERIALIZATION_VERSION_STRING * Check SERIALIZATION_VERSION_STRING before deserializing anything else * Invoke Clear() before deserializing just to be sure
2019-07-09 07:59:57 +02:00
READWRITE(scriptPayout);
READWRITE(scriptOperatorPayout);
}
void ResetOperatorFields()
{
pubKeyOperator.Set(CBLSPublicKey());
addr = CService();
scriptOperatorPayout = CScript();
nRevocationReason = CProUpRevTx::REASON_NOT_SPECIFIED;
}
void BanIfNotBanned(int height)
{
if (!IsBanned()) {
nPoSeBanHeight = height;
}
}
int GetBannedHeight() const
{
return nPoSeBanHeight;
}
bool IsBanned() const
{
return nPoSeBanHeight != -1;
}
void Revive(int nRevivedHeight)
{
nPoSePenalty = 0;
nPoSeBanHeight = -1;
nPoSeRevivedHeight = nRevivedHeight;
}
void UpdateConfirmedHash(const uint256& _proTxHash, const uint256& _confirmedHash)
{
confirmedHash = _confirmedHash;
CSHA256 h;
h.Write(_proTxHash.begin(), _proTxHash.size());
h.Write(_confirmedHash.begin(), _confirmedHash.size());
h.Finalize(confirmedHashWithProRegTxHash.begin());
}
Optimize on-disk deterministic masternode storage to reduce size of evodb (#3017) * Implement CompactFull() in CDBWrapper This allows to compact the whole DB in one go. * Implement more compact version of CDeterministicMNListDiff This introduces CDeterministicMNStateDiff which requires to only store fields on-disk which actually changed. * Avoid writing mnUniquePropertyMap to disk when storing snapshots This map can be rebuilt by simply using AddMN for each deserialized MN. * Implement Serialize/Unserialize in CScript This allows us to directly use READWRITE() on scripts and removes the need for the ugly cast to CScriptBase. This commit also changes all Dash specific uses of CScript to not use the cast. * Keep track of registeration counts and introduce internalID for masternodes The "internalId" is simply the number of MNs registered so far when the new MN is added. It is deterministic and stays the same forever. * Use internalId as keys in MN list diffs This reduces the used size on-disk. * Two simple speedups in MN list diff handling 1. Avoid full compare if dmn or state pointers match in BuildDiff 2. Use std::move when adding diff to listDiff in GetListForBlock * Implement upgrade code for old CDeterministicMNListDiff format to new format * Track tipIndex instead of tipHeight/tipBlockHash * Store and pass around CBlockIndex* instead of block hash and height This allows us to switch CDeterministicMNManager::GetListForBlock to work with CBlockIndex. * Refactor CDeterministicMNManager::GetListForBlock to require CBlockIndex* Instead of requiring a block hash. This allows us to remove blockHash and prevBlockHash from CDeterministicMNListDiff without the use of cs_main locks in GetListForBlock. * Remove prevBlockHash, blockHash and nHeight from CDeterministicMNListDiff * Remove access to determinisitcMNManager in CMasternodeMetaMan::ToString() The deterministic MN manager is not fully initialized yet at the time this is called, which results in an empty list being returned everytime. * Better logic to determine if an upgrade is needed Reuse the "best block" logic to figure out if an upgrade is needed. Also use it to ensure that older nodes are unable to start after the upgrade was performed. * Return null block hash if it was requested with getmnlistdiff * bump CGovernanceManager::SERIALIZATION_VERSION_STRING * Check SERIALIZATION_VERSION_STRING before deserializing anything else * Invoke Clear() before deserializing just to be sure
2019-07-09 07:59:57 +02:00
public:
std::string ToString() const;
void ToJson(UniValue& obj) const;
};
typedef std::shared_ptr<CDeterministicMNState> CDeterministicMNStatePtr;
typedef std::shared_ptr<const CDeterministicMNState> CDeterministicMNStateCPtr;
class CDeterministicMNStateDiff
{
public:
enum Field : uint32_t {
Field_nRegisteredHeight = 0x0001,
Field_nLastPaidHeight = 0x0002,
Field_nPoSePenalty = 0x0004,
Field_nPoSeRevivedHeight = 0x0008,
Field_nPoSeBanHeight = 0x0010,
Field_nRevocationReason = 0x0020,
Field_confirmedHash = 0x0040,
Field_confirmedHashWithProRegTxHash = 0x0080,
Field_keyIDOwner = 0x0100,
Field_pubKeyOperator = 0x0200,
Field_keyIDVoting = 0x0400,
Field_addr = 0x0800,
Field_scriptPayout = 0x1000,
Field_scriptOperatorPayout = 0x2000,
};
#define DMN_STATE_DIFF_ALL_FIELDS \
DMN_STATE_DIFF_LINE(nRegisteredHeight) \
DMN_STATE_DIFF_LINE(nLastPaidHeight) \
DMN_STATE_DIFF_LINE(nPoSePenalty) \
DMN_STATE_DIFF_LINE(nPoSeRevivedHeight) \
DMN_STATE_DIFF_LINE(nPoSeBanHeight) \
DMN_STATE_DIFF_LINE(nRevocationReason) \
DMN_STATE_DIFF_LINE(confirmedHash) \
DMN_STATE_DIFF_LINE(confirmedHashWithProRegTxHash) \
DMN_STATE_DIFF_LINE(keyIDOwner) \
DMN_STATE_DIFF_LINE(pubKeyOperator) \
DMN_STATE_DIFF_LINE(keyIDVoting) \
DMN_STATE_DIFF_LINE(addr) \
DMN_STATE_DIFF_LINE(scriptPayout) \
DMN_STATE_DIFF_LINE(scriptOperatorPayout)
public:
uint32_t fields{0};
// we reuse the state class, but only the members as noted by fields are valid
CDeterministicMNState state;
public:
CDeterministicMNStateDiff() = default;
Optimize on-disk deterministic masternode storage to reduce size of evodb (#3017) * Implement CompactFull() in CDBWrapper This allows to compact the whole DB in one go. * Implement more compact version of CDeterministicMNListDiff This introduces CDeterministicMNStateDiff which requires to only store fields on-disk which actually changed. * Avoid writing mnUniquePropertyMap to disk when storing snapshots This map can be rebuilt by simply using AddMN for each deserialized MN. * Implement Serialize/Unserialize in CScript This allows us to directly use READWRITE() on scripts and removes the need for the ugly cast to CScriptBase. This commit also changes all Dash specific uses of CScript to not use the cast. * Keep track of registeration counts and introduce internalID for masternodes The "internalId" is simply the number of MNs registered so far when the new MN is added. It is deterministic and stays the same forever. * Use internalId as keys in MN list diffs This reduces the used size on-disk. * Two simple speedups in MN list diff handling 1. Avoid full compare if dmn or state pointers match in BuildDiff 2. Use std::move when adding diff to listDiff in GetListForBlock * Implement upgrade code for old CDeterministicMNListDiff format to new format * Track tipIndex instead of tipHeight/tipBlockHash * Store and pass around CBlockIndex* instead of block hash and height This allows us to switch CDeterministicMNManager::GetListForBlock to work with CBlockIndex. * Refactor CDeterministicMNManager::GetListForBlock to require CBlockIndex* Instead of requiring a block hash. This allows us to remove blockHash and prevBlockHash from CDeterministicMNListDiff without the use of cs_main locks in GetListForBlock. * Remove prevBlockHash, blockHash and nHeight from CDeterministicMNListDiff * Remove access to determinisitcMNManager in CMasternodeMetaMan::ToString() The deterministic MN manager is not fully initialized yet at the time this is called, which results in an empty list being returned everytime. * Better logic to determine if an upgrade is needed Reuse the "best block" logic to figure out if an upgrade is needed. Also use it to ensure that older nodes are unable to start after the upgrade was performed. * Return null block hash if it was requested with getmnlistdiff * bump CGovernanceManager::SERIALIZATION_VERSION_STRING * Check SERIALIZATION_VERSION_STRING before deserializing anything else * Invoke Clear() before deserializing just to be sure
2019-07-09 07:59:57 +02:00
CDeterministicMNStateDiff(const CDeterministicMNState& a, const CDeterministicMNState& b)
{
Optimize on-disk deterministic masternode storage to reduce size of evodb (#3017) * Implement CompactFull() in CDBWrapper This allows to compact the whole DB in one go. * Implement more compact version of CDeterministicMNListDiff This introduces CDeterministicMNStateDiff which requires to only store fields on-disk which actually changed. * Avoid writing mnUniquePropertyMap to disk when storing snapshots This map can be rebuilt by simply using AddMN for each deserialized MN. * Implement Serialize/Unserialize in CScript This allows us to directly use READWRITE() on scripts and removes the need for the ugly cast to CScriptBase. This commit also changes all Dash specific uses of CScript to not use the cast. * Keep track of registeration counts and introduce internalID for masternodes The "internalId" is simply the number of MNs registered so far when the new MN is added. It is deterministic and stays the same forever. * Use internalId as keys in MN list diffs This reduces the used size on-disk. * Two simple speedups in MN list diff handling 1. Avoid full compare if dmn or state pointers match in BuildDiff 2. Use std::move when adding diff to listDiff in GetListForBlock * Implement upgrade code for old CDeterministicMNListDiff format to new format * Track tipIndex instead of tipHeight/tipBlockHash * Store and pass around CBlockIndex* instead of block hash and height This allows us to switch CDeterministicMNManager::GetListForBlock to work with CBlockIndex. * Refactor CDeterministicMNManager::GetListForBlock to require CBlockIndex* Instead of requiring a block hash. This allows us to remove blockHash and prevBlockHash from CDeterministicMNListDiff without the use of cs_main locks in GetListForBlock. * Remove prevBlockHash, blockHash and nHeight from CDeterministicMNListDiff * Remove access to determinisitcMNManager in CMasternodeMetaMan::ToString() The deterministic MN manager is not fully initialized yet at the time this is called, which results in an empty list being returned everytime. * Better logic to determine if an upgrade is needed Reuse the "best block" logic to figure out if an upgrade is needed. Also use it to ensure that older nodes are unable to start after the upgrade was performed. * Return null block hash if it was requested with getmnlistdiff * bump CGovernanceManager::SERIALIZATION_VERSION_STRING * Check SERIALIZATION_VERSION_STRING before deserializing anything else * Invoke Clear() before deserializing just to be sure
2019-07-09 07:59:57 +02:00
#define DMN_STATE_DIFF_LINE(f) if (a.f != b.f) { state.f = b.f; fields |= Field_##f; }
DMN_STATE_DIFF_ALL_FIELDS
#undef DMN_STATE_DIFF_LINE
}
Optimize on-disk deterministic masternode storage to reduce size of evodb (#3017) * Implement CompactFull() in CDBWrapper This allows to compact the whole DB in one go. * Implement more compact version of CDeterministicMNListDiff This introduces CDeterministicMNStateDiff which requires to only store fields on-disk which actually changed. * Avoid writing mnUniquePropertyMap to disk when storing snapshots This map can be rebuilt by simply using AddMN for each deserialized MN. * Implement Serialize/Unserialize in CScript This allows us to directly use READWRITE() on scripts and removes the need for the ugly cast to CScriptBase. This commit also changes all Dash specific uses of CScript to not use the cast. * Keep track of registeration counts and introduce internalID for masternodes The "internalId" is simply the number of MNs registered so far when the new MN is added. It is deterministic and stays the same forever. * Use internalId as keys in MN list diffs This reduces the used size on-disk. * Two simple speedups in MN list diff handling 1. Avoid full compare if dmn or state pointers match in BuildDiff 2. Use std::move when adding diff to listDiff in GetListForBlock * Implement upgrade code for old CDeterministicMNListDiff format to new format * Track tipIndex instead of tipHeight/tipBlockHash * Store and pass around CBlockIndex* instead of block hash and height This allows us to switch CDeterministicMNManager::GetListForBlock to work with CBlockIndex. * Refactor CDeterministicMNManager::GetListForBlock to require CBlockIndex* Instead of requiring a block hash. This allows us to remove blockHash and prevBlockHash from CDeterministicMNListDiff without the use of cs_main locks in GetListForBlock. * Remove prevBlockHash, blockHash and nHeight from CDeterministicMNListDiff * Remove access to determinisitcMNManager in CMasternodeMetaMan::ToString() The deterministic MN manager is not fully initialized yet at the time this is called, which results in an empty list being returned everytime. * Better logic to determine if an upgrade is needed Reuse the "best block" logic to figure out if an upgrade is needed. Also use it to ensure that older nodes are unable to start after the upgrade was performed. * Return null block hash if it was requested with getmnlistdiff * bump CGovernanceManager::SERIALIZATION_VERSION_STRING * Check SERIALIZATION_VERSION_STRING before deserializing anything else * Invoke Clear() before deserializing just to be sure
2019-07-09 07:59:57 +02:00
ADD_SERIALIZE_METHODS;
template <typename Stream, typename Operation>
inline void SerializationOp(Stream& s, Operation ser_action)
{
Optimize on-disk deterministic masternode storage to reduce size of evodb (#3017) * Implement CompactFull() in CDBWrapper This allows to compact the whole DB in one go. * Implement more compact version of CDeterministicMNListDiff This introduces CDeterministicMNStateDiff which requires to only store fields on-disk which actually changed. * Avoid writing mnUniquePropertyMap to disk when storing snapshots This map can be rebuilt by simply using AddMN for each deserialized MN. * Implement Serialize/Unserialize in CScript This allows us to directly use READWRITE() on scripts and removes the need for the ugly cast to CScriptBase. This commit also changes all Dash specific uses of CScript to not use the cast. * Keep track of registeration counts and introduce internalID for masternodes The "internalId" is simply the number of MNs registered so far when the new MN is added. It is deterministic and stays the same forever. * Use internalId as keys in MN list diffs This reduces the used size on-disk. * Two simple speedups in MN list diff handling 1. Avoid full compare if dmn or state pointers match in BuildDiff 2. Use std::move when adding diff to listDiff in GetListForBlock * Implement upgrade code for old CDeterministicMNListDiff format to new format * Track tipIndex instead of tipHeight/tipBlockHash * Store and pass around CBlockIndex* instead of block hash and height This allows us to switch CDeterministicMNManager::GetListForBlock to work with CBlockIndex. * Refactor CDeterministicMNManager::GetListForBlock to require CBlockIndex* Instead of requiring a block hash. This allows us to remove blockHash and prevBlockHash from CDeterministicMNListDiff without the use of cs_main locks in GetListForBlock. * Remove prevBlockHash, blockHash and nHeight from CDeterministicMNListDiff * Remove access to determinisitcMNManager in CMasternodeMetaMan::ToString() The deterministic MN manager is not fully initialized yet at the time this is called, which results in an empty list being returned everytime. * Better logic to determine if an upgrade is needed Reuse the "best block" logic to figure out if an upgrade is needed. Also use it to ensure that older nodes are unable to start after the upgrade was performed. * Return null block hash if it was requested with getmnlistdiff * bump CGovernanceManager::SERIALIZATION_VERSION_STRING * Check SERIALIZATION_VERSION_STRING before deserializing anything else * Invoke Clear() before deserializing just to be sure
2019-07-09 07:59:57 +02:00
READWRITE(VARINT(fields));
#define DMN_STATE_DIFF_LINE(f) if (fields & Field_##f) READWRITE(state.f);
DMN_STATE_DIFF_ALL_FIELDS
#undef DMN_STATE_DIFF_LINE
}
Optimize on-disk deterministic masternode storage to reduce size of evodb (#3017) * Implement CompactFull() in CDBWrapper This allows to compact the whole DB in one go. * Implement more compact version of CDeterministicMNListDiff This introduces CDeterministicMNStateDiff which requires to only store fields on-disk which actually changed. * Avoid writing mnUniquePropertyMap to disk when storing snapshots This map can be rebuilt by simply using AddMN for each deserialized MN. * Implement Serialize/Unserialize in CScript This allows us to directly use READWRITE() on scripts and removes the need for the ugly cast to CScriptBase. This commit also changes all Dash specific uses of CScript to not use the cast. * Keep track of registeration counts and introduce internalID for masternodes The "internalId" is simply the number of MNs registered so far when the new MN is added. It is deterministic and stays the same forever. * Use internalId as keys in MN list diffs This reduces the used size on-disk. * Two simple speedups in MN list diff handling 1. Avoid full compare if dmn or state pointers match in BuildDiff 2. Use std::move when adding diff to listDiff in GetListForBlock * Implement upgrade code for old CDeterministicMNListDiff format to new format * Track tipIndex instead of tipHeight/tipBlockHash * Store and pass around CBlockIndex* instead of block hash and height This allows us to switch CDeterministicMNManager::GetListForBlock to work with CBlockIndex. * Refactor CDeterministicMNManager::GetListForBlock to require CBlockIndex* Instead of requiring a block hash. This allows us to remove blockHash and prevBlockHash from CDeterministicMNListDiff without the use of cs_main locks in GetListForBlock. * Remove prevBlockHash, blockHash and nHeight from CDeterministicMNListDiff * Remove access to determinisitcMNManager in CMasternodeMetaMan::ToString() The deterministic MN manager is not fully initialized yet at the time this is called, which results in an empty list being returned everytime. * Better logic to determine if an upgrade is needed Reuse the "best block" logic to figure out if an upgrade is needed. Also use it to ensure that older nodes are unable to start after the upgrade was performed. * Return null block hash if it was requested with getmnlistdiff * bump CGovernanceManager::SERIALIZATION_VERSION_STRING * Check SERIALIZATION_VERSION_STRING before deserializing anything else * Invoke Clear() before deserializing just to be sure
2019-07-09 07:59:57 +02:00
void ApplyToState(CDeterministicMNState& target) const
{
#define DMN_STATE_DIFF_LINE(f) if (fields & Field_##f) target.f = state.f;
DMN_STATE_DIFF_ALL_FIELDS
#undef DMN_STATE_DIFF_LINE
}
};
class CDeterministicMN
{
private:
uint64_t internalId{std::numeric_limits<uint64_t>::max()};
public:
CDeterministicMN() = delete; // no default constructor, must specify internalId
explicit CDeterministicMN(uint64_t _internalId) : internalId(_internalId)
{
// only non-initial values
assert(_internalId != std::numeric_limits<uint64_t>::max());
}
// TODO: can be removed in a future version
CDeterministicMN(const CDeterministicMN& mn, uint64_t _internalId) : CDeterministicMN(mn) {
// only non-initial values
assert(_internalId != std::numeric_limits<uint64_t>::max());
internalId = _internalId;
}
template <typename Stream>
CDeterministicMN(deserialize_type, Stream& s)
{
s >> *this;
}
uint256 proTxHash;
2018-10-25 16:29:50 +02:00
COutPoint collateralOutpoint;
uint16_t nOperatorReward;
CDeterministicMNStateCPtr pdmnState;
public:
template <typename Stream, typename Operation>
Optimize on-disk deterministic masternode storage to reduce size of evodb (#3017) * Implement CompactFull() in CDBWrapper This allows to compact the whole DB in one go. * Implement more compact version of CDeterministicMNListDiff This introduces CDeterministicMNStateDiff which requires to only store fields on-disk which actually changed. * Avoid writing mnUniquePropertyMap to disk when storing snapshots This map can be rebuilt by simply using AddMN for each deserialized MN. * Implement Serialize/Unserialize in CScript This allows us to directly use READWRITE() on scripts and removes the need for the ugly cast to CScriptBase. This commit also changes all Dash specific uses of CScript to not use the cast. * Keep track of registeration counts and introduce internalID for masternodes The "internalId" is simply the number of MNs registered so far when the new MN is added. It is deterministic and stays the same forever. * Use internalId as keys in MN list diffs This reduces the used size on-disk. * Two simple speedups in MN list diff handling 1. Avoid full compare if dmn or state pointers match in BuildDiff 2. Use std::move when adding diff to listDiff in GetListForBlock * Implement upgrade code for old CDeterministicMNListDiff format to new format * Track tipIndex instead of tipHeight/tipBlockHash * Store and pass around CBlockIndex* instead of block hash and height This allows us to switch CDeterministicMNManager::GetListForBlock to work with CBlockIndex. * Refactor CDeterministicMNManager::GetListForBlock to require CBlockIndex* Instead of requiring a block hash. This allows us to remove blockHash and prevBlockHash from CDeterministicMNListDiff without the use of cs_main locks in GetListForBlock. * Remove prevBlockHash, blockHash and nHeight from CDeterministicMNListDiff * Remove access to determinisitcMNManager in CMasternodeMetaMan::ToString() The deterministic MN manager is not fully initialized yet at the time this is called, which results in an empty list being returned everytime. * Better logic to determine if an upgrade is needed Reuse the "best block" logic to figure out if an upgrade is needed. Also use it to ensure that older nodes are unable to start after the upgrade was performed. * Return null block hash if it was requested with getmnlistdiff * bump CGovernanceManager::SERIALIZATION_VERSION_STRING * Check SERIALIZATION_VERSION_STRING before deserializing anything else * Invoke Clear() before deserializing just to be sure
2019-07-09 07:59:57 +02:00
inline void SerializationOp(Stream& s, Operation ser_action, bool oldFormat)
{
READWRITE(proTxHash);
Optimize on-disk deterministic masternode storage to reduce size of evodb (#3017) * Implement CompactFull() in CDBWrapper This allows to compact the whole DB in one go. * Implement more compact version of CDeterministicMNListDiff This introduces CDeterministicMNStateDiff which requires to only store fields on-disk which actually changed. * Avoid writing mnUniquePropertyMap to disk when storing snapshots This map can be rebuilt by simply using AddMN for each deserialized MN. * Implement Serialize/Unserialize in CScript This allows us to directly use READWRITE() on scripts and removes the need for the ugly cast to CScriptBase. This commit also changes all Dash specific uses of CScript to not use the cast. * Keep track of registeration counts and introduce internalID for masternodes The "internalId" is simply the number of MNs registered so far when the new MN is added. It is deterministic and stays the same forever. * Use internalId as keys in MN list diffs This reduces the used size on-disk. * Two simple speedups in MN list diff handling 1. Avoid full compare if dmn or state pointers match in BuildDiff 2. Use std::move when adding diff to listDiff in GetListForBlock * Implement upgrade code for old CDeterministicMNListDiff format to new format * Track tipIndex instead of tipHeight/tipBlockHash * Store and pass around CBlockIndex* instead of block hash and height This allows us to switch CDeterministicMNManager::GetListForBlock to work with CBlockIndex. * Refactor CDeterministicMNManager::GetListForBlock to require CBlockIndex* Instead of requiring a block hash. This allows us to remove blockHash and prevBlockHash from CDeterministicMNListDiff without the use of cs_main locks in GetListForBlock. * Remove prevBlockHash, blockHash and nHeight from CDeterministicMNListDiff * Remove access to determinisitcMNManager in CMasternodeMetaMan::ToString() The deterministic MN manager is not fully initialized yet at the time this is called, which results in an empty list being returned everytime. * Better logic to determine if an upgrade is needed Reuse the "best block" logic to figure out if an upgrade is needed. Also use it to ensure that older nodes are unable to start after the upgrade was performed. * Return null block hash if it was requested with getmnlistdiff * bump CGovernanceManager::SERIALIZATION_VERSION_STRING * Check SERIALIZATION_VERSION_STRING before deserializing anything else * Invoke Clear() before deserializing just to be sure
2019-07-09 07:59:57 +02:00
if (!oldFormat) {
READWRITE(VARINT(internalId));
}
2018-10-25 16:29:50 +02:00
READWRITE(collateralOutpoint);
READWRITE(nOperatorReward);
READWRITE(pdmnState);
}
Optimize on-disk deterministic masternode storage to reduce size of evodb (#3017) * Implement CompactFull() in CDBWrapper This allows to compact the whole DB in one go. * Implement more compact version of CDeterministicMNListDiff This introduces CDeterministicMNStateDiff which requires to only store fields on-disk which actually changed. * Avoid writing mnUniquePropertyMap to disk when storing snapshots This map can be rebuilt by simply using AddMN for each deserialized MN. * Implement Serialize/Unserialize in CScript This allows us to directly use READWRITE() on scripts and removes the need for the ugly cast to CScriptBase. This commit also changes all Dash specific uses of CScript to not use the cast. * Keep track of registeration counts and introduce internalID for masternodes The "internalId" is simply the number of MNs registered so far when the new MN is added. It is deterministic and stays the same forever. * Use internalId as keys in MN list diffs This reduces the used size on-disk. * Two simple speedups in MN list diff handling 1. Avoid full compare if dmn or state pointers match in BuildDiff 2. Use std::move when adding diff to listDiff in GetListForBlock * Implement upgrade code for old CDeterministicMNListDiff format to new format * Track tipIndex instead of tipHeight/tipBlockHash * Store and pass around CBlockIndex* instead of block hash and height This allows us to switch CDeterministicMNManager::GetListForBlock to work with CBlockIndex. * Refactor CDeterministicMNManager::GetListForBlock to require CBlockIndex* Instead of requiring a block hash. This allows us to remove blockHash and prevBlockHash from CDeterministicMNListDiff without the use of cs_main locks in GetListForBlock. * Remove prevBlockHash, blockHash and nHeight from CDeterministicMNListDiff * Remove access to determinisitcMNManager in CMasternodeMetaMan::ToString() The deterministic MN manager is not fully initialized yet at the time this is called, which results in an empty list being returned everytime. * Better logic to determine if an upgrade is needed Reuse the "best block" logic to figure out if an upgrade is needed. Also use it to ensure that older nodes are unable to start after the upgrade was performed. * Return null block hash if it was requested with getmnlistdiff * bump CGovernanceManager::SERIALIZATION_VERSION_STRING * Check SERIALIZATION_VERSION_STRING before deserializing anything else * Invoke Clear() before deserializing just to be sure
2019-07-09 07:59:57 +02:00
template<typename Stream>
void Serialize(Stream& s) const
{
NCONST_PTR(this)->SerializationOp(s, CSerActionSerialize(), false);
}
template<typename Stream>
void Unserialize(Stream& s, bool oldFormat = false)
{
SerializationOp(s, CSerActionUnserialize(), oldFormat);
}
uint64_t GetInternalId() const;
std::string ToString() const;
void ToJson(UniValue& obj) const;
};
typedef std::shared_ptr<const CDeterministicMN> CDeterministicMNCPtr;
class CDeterministicMNListDiff;
template <typename Stream, typename K, typename T, typename Hash, typename Equal>
void SerializeImmerMap(Stream& os, const immer::map<K, T, Hash, Equal>& m)
{
WriteCompactSize(os, m.size());
for (typename immer::map<K, T, Hash, Equal>::const_iterator mi = m.begin(); mi != m.end(); ++mi)
Serialize(os, (*mi));
}
template <typename Stream, typename K, typename T, typename Hash, typename Equal>
void UnserializeImmerMap(Stream& is, immer::map<K, T, Hash, Equal>& m)
{
m = immer::map<K, T, Hash, Equal>();
unsigned int nSize = ReadCompactSize(is);
for (unsigned int i = 0; i < nSize; i++) {
std::pair<K, T> item;
Unserialize(is, item);
m = m.set(item.first, item.second);
}
}
// For some reason the compiler is not able to choose the correct Serialize/Deserialize methods without a specialized
// version of SerReadWrite. It otherwise always chooses the version that calls a.Serialize()
template<typename Stream, typename K, typename T, typename Hash, typename Equal>
inline void SerReadWrite(Stream& s, const immer::map<K, T, Hash, Equal>& m, CSerActionSerialize ser_action)
{
::SerializeImmerMap(s, m);
}
template<typename Stream, typename K, typename T, typename Hash, typename Equal>
inline void SerReadWrite(Stream& s, immer::map<K, T, Hash, Equal>& obj, CSerActionUnserialize ser_action)
{
::UnserializeImmerMap(s, obj);
}
class CDeterministicMNList
{
public:
typedef immer::map<uint256, CDeterministicMNCPtr> MnMap;
Optimize on-disk deterministic masternode storage to reduce size of evodb (#3017) * Implement CompactFull() in CDBWrapper This allows to compact the whole DB in one go. * Implement more compact version of CDeterministicMNListDiff This introduces CDeterministicMNStateDiff which requires to only store fields on-disk which actually changed. * Avoid writing mnUniquePropertyMap to disk when storing snapshots This map can be rebuilt by simply using AddMN for each deserialized MN. * Implement Serialize/Unserialize in CScript This allows us to directly use READWRITE() on scripts and removes the need for the ugly cast to CScriptBase. This commit also changes all Dash specific uses of CScript to not use the cast. * Keep track of registeration counts and introduce internalID for masternodes The "internalId" is simply the number of MNs registered so far when the new MN is added. It is deterministic and stays the same forever. * Use internalId as keys in MN list diffs This reduces the used size on-disk. * Two simple speedups in MN list diff handling 1. Avoid full compare if dmn or state pointers match in BuildDiff 2. Use std::move when adding diff to listDiff in GetListForBlock * Implement upgrade code for old CDeterministicMNListDiff format to new format * Track tipIndex instead of tipHeight/tipBlockHash * Store and pass around CBlockIndex* instead of block hash and height This allows us to switch CDeterministicMNManager::GetListForBlock to work with CBlockIndex. * Refactor CDeterministicMNManager::GetListForBlock to require CBlockIndex* Instead of requiring a block hash. This allows us to remove blockHash and prevBlockHash from CDeterministicMNListDiff without the use of cs_main locks in GetListForBlock. * Remove prevBlockHash, blockHash and nHeight from CDeterministicMNListDiff * Remove access to determinisitcMNManager in CMasternodeMetaMan::ToString() The deterministic MN manager is not fully initialized yet at the time this is called, which results in an empty list being returned everytime. * Better logic to determine if an upgrade is needed Reuse the "best block" logic to figure out if an upgrade is needed. Also use it to ensure that older nodes are unable to start after the upgrade was performed. * Return null block hash if it was requested with getmnlistdiff * bump CGovernanceManager::SERIALIZATION_VERSION_STRING * Check SERIALIZATION_VERSION_STRING before deserializing anything else * Invoke Clear() before deserializing just to be sure
2019-07-09 07:59:57 +02:00
typedef immer::map<uint64_t, uint256> MnInternalIdMap;
typedef immer::map<uint256, std::pair<uint256, uint32_t> > MnUniquePropertyMap;
private:
uint256 blockHash;
int nHeight{-1};
Optimize on-disk deterministic masternode storage to reduce size of evodb (#3017) * Implement CompactFull() in CDBWrapper This allows to compact the whole DB in one go. * Implement more compact version of CDeterministicMNListDiff This introduces CDeterministicMNStateDiff which requires to only store fields on-disk which actually changed. * Avoid writing mnUniquePropertyMap to disk when storing snapshots This map can be rebuilt by simply using AddMN for each deserialized MN. * Implement Serialize/Unserialize in CScript This allows us to directly use READWRITE() on scripts and removes the need for the ugly cast to CScriptBase. This commit also changes all Dash specific uses of CScript to not use the cast. * Keep track of registeration counts and introduce internalID for masternodes The "internalId" is simply the number of MNs registered so far when the new MN is added. It is deterministic and stays the same forever. * Use internalId as keys in MN list diffs This reduces the used size on-disk. * Two simple speedups in MN list diff handling 1. Avoid full compare if dmn or state pointers match in BuildDiff 2. Use std::move when adding diff to listDiff in GetListForBlock * Implement upgrade code for old CDeterministicMNListDiff format to new format * Track tipIndex instead of tipHeight/tipBlockHash * Store and pass around CBlockIndex* instead of block hash and height This allows us to switch CDeterministicMNManager::GetListForBlock to work with CBlockIndex. * Refactor CDeterministicMNManager::GetListForBlock to require CBlockIndex* Instead of requiring a block hash. This allows us to remove blockHash and prevBlockHash from CDeterministicMNListDiff without the use of cs_main locks in GetListForBlock. * Remove prevBlockHash, blockHash and nHeight from CDeterministicMNListDiff * Remove access to determinisitcMNManager in CMasternodeMetaMan::ToString() The deterministic MN manager is not fully initialized yet at the time this is called, which results in an empty list being returned everytime. * Better logic to determine if an upgrade is needed Reuse the "best block" logic to figure out if an upgrade is needed. Also use it to ensure that older nodes are unable to start after the upgrade was performed. * Return null block hash if it was requested with getmnlistdiff * bump CGovernanceManager::SERIALIZATION_VERSION_STRING * Check SERIALIZATION_VERSION_STRING before deserializing anything else * Invoke Clear() before deserializing just to be sure
2019-07-09 07:59:57 +02:00
uint32_t nTotalRegisteredCount{0};
MnMap mnMap;
Optimize on-disk deterministic masternode storage to reduce size of evodb (#3017) * Implement CompactFull() in CDBWrapper This allows to compact the whole DB in one go. * Implement more compact version of CDeterministicMNListDiff This introduces CDeterministicMNStateDiff which requires to only store fields on-disk which actually changed. * Avoid writing mnUniquePropertyMap to disk when storing snapshots This map can be rebuilt by simply using AddMN for each deserialized MN. * Implement Serialize/Unserialize in CScript This allows us to directly use READWRITE() on scripts and removes the need for the ugly cast to CScriptBase. This commit also changes all Dash specific uses of CScript to not use the cast. * Keep track of registeration counts and introduce internalID for masternodes The "internalId" is simply the number of MNs registered so far when the new MN is added. It is deterministic and stays the same forever. * Use internalId as keys in MN list diffs This reduces the used size on-disk. * Two simple speedups in MN list diff handling 1. Avoid full compare if dmn or state pointers match in BuildDiff 2. Use std::move when adding diff to listDiff in GetListForBlock * Implement upgrade code for old CDeterministicMNListDiff format to new format * Track tipIndex instead of tipHeight/tipBlockHash * Store and pass around CBlockIndex* instead of block hash and height This allows us to switch CDeterministicMNManager::GetListForBlock to work with CBlockIndex. * Refactor CDeterministicMNManager::GetListForBlock to require CBlockIndex* Instead of requiring a block hash. This allows us to remove blockHash and prevBlockHash from CDeterministicMNListDiff without the use of cs_main locks in GetListForBlock. * Remove prevBlockHash, blockHash and nHeight from CDeterministicMNListDiff * Remove access to determinisitcMNManager in CMasternodeMetaMan::ToString() The deterministic MN manager is not fully initialized yet at the time this is called, which results in an empty list being returned everytime. * Better logic to determine if an upgrade is needed Reuse the "best block" logic to figure out if an upgrade is needed. Also use it to ensure that older nodes are unable to start after the upgrade was performed. * Return null block hash if it was requested with getmnlistdiff * bump CGovernanceManager::SERIALIZATION_VERSION_STRING * Check SERIALIZATION_VERSION_STRING before deserializing anything else * Invoke Clear() before deserializing just to be sure
2019-07-09 07:59:57 +02:00
MnInternalIdMap mnInternalIdMap;
// map of unique properties like address and keys
// we keep track of this as checking for duplicates would otherwise be painfully slow
MnUniquePropertyMap mnUniquePropertyMap;
public:
CDeterministicMNList() = default;
Optimize on-disk deterministic masternode storage to reduce size of evodb (#3017) * Implement CompactFull() in CDBWrapper This allows to compact the whole DB in one go. * Implement more compact version of CDeterministicMNListDiff This introduces CDeterministicMNStateDiff which requires to only store fields on-disk which actually changed. * Avoid writing mnUniquePropertyMap to disk when storing snapshots This map can be rebuilt by simply using AddMN for each deserialized MN. * Implement Serialize/Unserialize in CScript This allows us to directly use READWRITE() on scripts and removes the need for the ugly cast to CScriptBase. This commit also changes all Dash specific uses of CScript to not use the cast. * Keep track of registeration counts and introduce internalID for masternodes The "internalId" is simply the number of MNs registered so far when the new MN is added. It is deterministic and stays the same forever. * Use internalId as keys in MN list diffs This reduces the used size on-disk. * Two simple speedups in MN list diff handling 1. Avoid full compare if dmn or state pointers match in BuildDiff 2. Use std::move when adding diff to listDiff in GetListForBlock * Implement upgrade code for old CDeterministicMNListDiff format to new format * Track tipIndex instead of tipHeight/tipBlockHash * Store and pass around CBlockIndex* instead of block hash and height This allows us to switch CDeterministicMNManager::GetListForBlock to work with CBlockIndex. * Refactor CDeterministicMNManager::GetListForBlock to require CBlockIndex* Instead of requiring a block hash. This allows us to remove blockHash and prevBlockHash from CDeterministicMNListDiff without the use of cs_main locks in GetListForBlock. * Remove prevBlockHash, blockHash and nHeight from CDeterministicMNListDiff * Remove access to determinisitcMNManager in CMasternodeMetaMan::ToString() The deterministic MN manager is not fully initialized yet at the time this is called, which results in an empty list being returned everytime. * Better logic to determine if an upgrade is needed Reuse the "best block" logic to figure out if an upgrade is needed. Also use it to ensure that older nodes are unable to start after the upgrade was performed. * Return null block hash if it was requested with getmnlistdiff * bump CGovernanceManager::SERIALIZATION_VERSION_STRING * Check SERIALIZATION_VERSION_STRING before deserializing anything else * Invoke Clear() before deserializing just to be sure
2019-07-09 07:59:57 +02:00
explicit CDeterministicMNList(const uint256& _blockHash, int _height, uint32_t _totalRegisteredCount) :
blockHash(_blockHash),
Optimize on-disk deterministic masternode storage to reduce size of evodb (#3017) * Implement CompactFull() in CDBWrapper This allows to compact the whole DB in one go. * Implement more compact version of CDeterministicMNListDiff This introduces CDeterministicMNStateDiff which requires to only store fields on-disk which actually changed. * Avoid writing mnUniquePropertyMap to disk when storing snapshots This map can be rebuilt by simply using AddMN for each deserialized MN. * Implement Serialize/Unserialize in CScript This allows us to directly use READWRITE() on scripts and removes the need for the ugly cast to CScriptBase. This commit also changes all Dash specific uses of CScript to not use the cast. * Keep track of registeration counts and introduce internalID for masternodes The "internalId" is simply the number of MNs registered so far when the new MN is added. It is deterministic and stays the same forever. * Use internalId as keys in MN list diffs This reduces the used size on-disk. * Two simple speedups in MN list diff handling 1. Avoid full compare if dmn or state pointers match in BuildDiff 2. Use std::move when adding diff to listDiff in GetListForBlock * Implement upgrade code for old CDeterministicMNListDiff format to new format * Track tipIndex instead of tipHeight/tipBlockHash * Store and pass around CBlockIndex* instead of block hash and height This allows us to switch CDeterministicMNManager::GetListForBlock to work with CBlockIndex. * Refactor CDeterministicMNManager::GetListForBlock to require CBlockIndex* Instead of requiring a block hash. This allows us to remove blockHash and prevBlockHash from CDeterministicMNListDiff without the use of cs_main locks in GetListForBlock. * Remove prevBlockHash, blockHash and nHeight from CDeterministicMNListDiff * Remove access to determinisitcMNManager in CMasternodeMetaMan::ToString() The deterministic MN manager is not fully initialized yet at the time this is called, which results in an empty list being returned everytime. * Better logic to determine if an upgrade is needed Reuse the "best block" logic to figure out if an upgrade is needed. Also use it to ensure that older nodes are unable to start after the upgrade was performed. * Return null block hash if it was requested with getmnlistdiff * bump CGovernanceManager::SERIALIZATION_VERSION_STRING * Check SERIALIZATION_VERSION_STRING before deserializing anything else * Invoke Clear() before deserializing just to be sure
2019-07-09 07:59:57 +02:00
nHeight(_height),
nTotalRegisteredCount(_totalRegisteredCount)
{
}
template <typename Stream, typename Operation>
Optimize on-disk deterministic masternode storage to reduce size of evodb (#3017) * Implement CompactFull() in CDBWrapper This allows to compact the whole DB in one go. * Implement more compact version of CDeterministicMNListDiff This introduces CDeterministicMNStateDiff which requires to only store fields on-disk which actually changed. * Avoid writing mnUniquePropertyMap to disk when storing snapshots This map can be rebuilt by simply using AddMN for each deserialized MN. * Implement Serialize/Unserialize in CScript This allows us to directly use READWRITE() on scripts and removes the need for the ugly cast to CScriptBase. This commit also changes all Dash specific uses of CScript to not use the cast. * Keep track of registeration counts and introduce internalID for masternodes The "internalId" is simply the number of MNs registered so far when the new MN is added. It is deterministic and stays the same forever. * Use internalId as keys in MN list diffs This reduces the used size on-disk. * Two simple speedups in MN list diff handling 1. Avoid full compare if dmn or state pointers match in BuildDiff 2. Use std::move when adding diff to listDiff in GetListForBlock * Implement upgrade code for old CDeterministicMNListDiff format to new format * Track tipIndex instead of tipHeight/tipBlockHash * Store and pass around CBlockIndex* instead of block hash and height This allows us to switch CDeterministicMNManager::GetListForBlock to work with CBlockIndex. * Refactor CDeterministicMNManager::GetListForBlock to require CBlockIndex* Instead of requiring a block hash. This allows us to remove blockHash and prevBlockHash from CDeterministicMNListDiff without the use of cs_main locks in GetListForBlock. * Remove prevBlockHash, blockHash and nHeight from CDeterministicMNListDiff * Remove access to determinisitcMNManager in CMasternodeMetaMan::ToString() The deterministic MN manager is not fully initialized yet at the time this is called, which results in an empty list being returned everytime. * Better logic to determine if an upgrade is needed Reuse the "best block" logic to figure out if an upgrade is needed. Also use it to ensure that older nodes are unable to start after the upgrade was performed. * Return null block hash if it was requested with getmnlistdiff * bump CGovernanceManager::SERIALIZATION_VERSION_STRING * Check SERIALIZATION_VERSION_STRING before deserializing anything else * Invoke Clear() before deserializing just to be sure
2019-07-09 07:59:57 +02:00
inline void SerializationOpBase(Stream& s, Operation ser_action)
{
READWRITE(blockHash);
READWRITE(nHeight);
Optimize on-disk deterministic masternode storage to reduce size of evodb (#3017) * Implement CompactFull() in CDBWrapper This allows to compact the whole DB in one go. * Implement more compact version of CDeterministicMNListDiff This introduces CDeterministicMNStateDiff which requires to only store fields on-disk which actually changed. * Avoid writing mnUniquePropertyMap to disk when storing snapshots This map can be rebuilt by simply using AddMN for each deserialized MN. * Implement Serialize/Unserialize in CScript This allows us to directly use READWRITE() on scripts and removes the need for the ugly cast to CScriptBase. This commit also changes all Dash specific uses of CScript to not use the cast. * Keep track of registeration counts and introduce internalID for masternodes The "internalId" is simply the number of MNs registered so far when the new MN is added. It is deterministic and stays the same forever. * Use internalId as keys in MN list diffs This reduces the used size on-disk. * Two simple speedups in MN list diff handling 1. Avoid full compare if dmn or state pointers match in BuildDiff 2. Use std::move when adding diff to listDiff in GetListForBlock * Implement upgrade code for old CDeterministicMNListDiff format to new format * Track tipIndex instead of tipHeight/tipBlockHash * Store and pass around CBlockIndex* instead of block hash and height This allows us to switch CDeterministicMNManager::GetListForBlock to work with CBlockIndex. * Refactor CDeterministicMNManager::GetListForBlock to require CBlockIndex* Instead of requiring a block hash. This allows us to remove blockHash and prevBlockHash from CDeterministicMNListDiff without the use of cs_main locks in GetListForBlock. * Remove prevBlockHash, blockHash and nHeight from CDeterministicMNListDiff * Remove access to determinisitcMNManager in CMasternodeMetaMan::ToString() The deterministic MN manager is not fully initialized yet at the time this is called, which results in an empty list being returned everytime. * Better logic to determine if an upgrade is needed Reuse the "best block" logic to figure out if an upgrade is needed. Also use it to ensure that older nodes are unable to start after the upgrade was performed. * Return null block hash if it was requested with getmnlistdiff * bump CGovernanceManager::SERIALIZATION_VERSION_STRING * Check SERIALIZATION_VERSION_STRING before deserializing anything else * Invoke Clear() before deserializing just to be sure
2019-07-09 07:59:57 +02:00
READWRITE(nTotalRegisteredCount);
}
template<typename Stream>
void Serialize(Stream& s) const
{
NCONST_PTR(this)->SerializationOpBase(s, CSerActionSerialize());
// Serialize the map as a vector
WriteCompactSize(s, mnMap.size());
for (const auto& p : mnMap) {
s << *p.second;
}
}
template<typename Stream>
void Unserialize(Stream& s) {
mnMap = MnMap();
mnUniquePropertyMap = MnUniquePropertyMap();
mnInternalIdMap = MnInternalIdMap();
SerializationOpBase(s, CSerActionUnserialize());
size_t cnt = ReadCompactSize(s);
for (size_t i = 0; i < cnt; i++) {
AddMN(std::make_shared<CDeterministicMN>(deserialize, s), false);
Optimize on-disk deterministic masternode storage to reduce size of evodb (#3017) * Implement CompactFull() in CDBWrapper This allows to compact the whole DB in one go. * Implement more compact version of CDeterministicMNListDiff This introduces CDeterministicMNStateDiff which requires to only store fields on-disk which actually changed. * Avoid writing mnUniquePropertyMap to disk when storing snapshots This map can be rebuilt by simply using AddMN for each deserialized MN. * Implement Serialize/Unserialize in CScript This allows us to directly use READWRITE() on scripts and removes the need for the ugly cast to CScriptBase. This commit also changes all Dash specific uses of CScript to not use the cast. * Keep track of registeration counts and introduce internalID for masternodes The "internalId" is simply the number of MNs registered so far when the new MN is added. It is deterministic and stays the same forever. * Use internalId as keys in MN list diffs This reduces the used size on-disk. * Two simple speedups in MN list diff handling 1. Avoid full compare if dmn or state pointers match in BuildDiff 2. Use std::move when adding diff to listDiff in GetListForBlock * Implement upgrade code for old CDeterministicMNListDiff format to new format * Track tipIndex instead of tipHeight/tipBlockHash * Store and pass around CBlockIndex* instead of block hash and height This allows us to switch CDeterministicMNManager::GetListForBlock to work with CBlockIndex. * Refactor CDeterministicMNManager::GetListForBlock to require CBlockIndex* Instead of requiring a block hash. This allows us to remove blockHash and prevBlockHash from CDeterministicMNListDiff without the use of cs_main locks in GetListForBlock. * Remove prevBlockHash, blockHash and nHeight from CDeterministicMNListDiff * Remove access to determinisitcMNManager in CMasternodeMetaMan::ToString() The deterministic MN manager is not fully initialized yet at the time this is called, which results in an empty list being returned everytime. * Better logic to determine if an upgrade is needed Reuse the "best block" logic to figure out if an upgrade is needed. Also use it to ensure that older nodes are unable to start after the upgrade was performed. * Return null block hash if it was requested with getmnlistdiff * bump CGovernanceManager::SERIALIZATION_VERSION_STRING * Check SERIALIZATION_VERSION_STRING before deserializing anything else * Invoke Clear() before deserializing just to be sure
2019-07-09 07:59:57 +02:00
}
}
public:
size_t GetAllMNsCount() const
{
return mnMap.size();
}
size_t GetValidMNsCount() const
{
size_t count = 0;
for (const auto& p : mnMap) {
if (IsMNValid(p.second)) {
count++;
}
}
return count;
}
template <typename Callback>
void ForEachMN(bool onlyValid, Callback&& cb) const
{
for (const auto& p : mnMap) {
if (!onlyValid || IsMNValid(p.second)) {
cb(p.second);
}
}
}
public:
const uint256& GetBlockHash() const
{
return blockHash;
}
void SetBlockHash(const uint256& _blockHash)
{
blockHash = _blockHash;
}
int GetHeight() const
{
return nHeight;
}
void SetHeight(int _height)
{
nHeight = _height;
}
Optimize on-disk deterministic masternode storage to reduce size of evodb (#3017) * Implement CompactFull() in CDBWrapper This allows to compact the whole DB in one go. * Implement more compact version of CDeterministicMNListDiff This introduces CDeterministicMNStateDiff which requires to only store fields on-disk which actually changed. * Avoid writing mnUniquePropertyMap to disk when storing snapshots This map can be rebuilt by simply using AddMN for each deserialized MN. * Implement Serialize/Unserialize in CScript This allows us to directly use READWRITE() on scripts and removes the need for the ugly cast to CScriptBase. This commit also changes all Dash specific uses of CScript to not use the cast. * Keep track of registeration counts and introduce internalID for masternodes The "internalId" is simply the number of MNs registered so far when the new MN is added. It is deterministic and stays the same forever. * Use internalId as keys in MN list diffs This reduces the used size on-disk. * Two simple speedups in MN list diff handling 1. Avoid full compare if dmn or state pointers match in BuildDiff 2. Use std::move when adding diff to listDiff in GetListForBlock * Implement upgrade code for old CDeterministicMNListDiff format to new format * Track tipIndex instead of tipHeight/tipBlockHash * Store and pass around CBlockIndex* instead of block hash and height This allows us to switch CDeterministicMNManager::GetListForBlock to work with CBlockIndex. * Refactor CDeterministicMNManager::GetListForBlock to require CBlockIndex* Instead of requiring a block hash. This allows us to remove blockHash and prevBlockHash from CDeterministicMNListDiff without the use of cs_main locks in GetListForBlock. * Remove prevBlockHash, blockHash and nHeight from CDeterministicMNListDiff * Remove access to determinisitcMNManager in CMasternodeMetaMan::ToString() The deterministic MN manager is not fully initialized yet at the time this is called, which results in an empty list being returned everytime. * Better logic to determine if an upgrade is needed Reuse the "best block" logic to figure out if an upgrade is needed. Also use it to ensure that older nodes are unable to start after the upgrade was performed. * Return null block hash if it was requested with getmnlistdiff * bump CGovernanceManager::SERIALIZATION_VERSION_STRING * Check SERIALIZATION_VERSION_STRING before deserializing anything else * Invoke Clear() before deserializing just to be sure
2019-07-09 07:59:57 +02:00
uint32_t GetTotalRegisteredCount() const
{
return nTotalRegisteredCount;
}
bool IsMNValid(const uint256& proTxHash) const;
bool IsMNPoSeBanned(const uint256& proTxHash) const;
static bool IsMNValid(const CDeterministicMNCPtr& dmn);
static bool IsMNPoSeBanned(const CDeterministicMNCPtr& dmn);
bool HasMN(const uint256& proTxHash) const
{
return GetMN(proTxHash) != nullptr;
}
bool HasMNByCollateral(const COutPoint& collateralOutpoint) const
{
return GetMNByCollateral(collateralOutpoint) != nullptr;
}
bool HasValidMNByCollateral(const COutPoint& collateralOutpoint) const
{
return GetValidMNByCollateral(collateralOutpoint) != nullptr;
}
CDeterministicMNCPtr GetMN(const uint256& proTxHash) const;
CDeterministicMNCPtr GetValidMN(const uint256& proTxHash) const;
CDeterministicMNCPtr GetMNByOperatorKey(const CBLSPublicKey& pubKey);
2018-10-25 16:29:50 +02:00
CDeterministicMNCPtr GetMNByCollateral(const COutPoint& collateralOutpoint) const;
CDeterministicMNCPtr GetValidMNByCollateral(const COutPoint& collateralOutpoint) const;
CDeterministicMNCPtr GetMNByService(const CService& service) const;
Optimize on-disk deterministic masternode storage to reduce size of evodb (#3017) * Implement CompactFull() in CDBWrapper This allows to compact the whole DB in one go. * Implement more compact version of CDeterministicMNListDiff This introduces CDeterministicMNStateDiff which requires to only store fields on-disk which actually changed. * Avoid writing mnUniquePropertyMap to disk when storing snapshots This map can be rebuilt by simply using AddMN for each deserialized MN. * Implement Serialize/Unserialize in CScript This allows us to directly use READWRITE() on scripts and removes the need for the ugly cast to CScriptBase. This commit also changes all Dash specific uses of CScript to not use the cast. * Keep track of registeration counts and introduce internalID for masternodes The "internalId" is simply the number of MNs registered so far when the new MN is added. It is deterministic and stays the same forever. * Use internalId as keys in MN list diffs This reduces the used size on-disk. * Two simple speedups in MN list diff handling 1. Avoid full compare if dmn or state pointers match in BuildDiff 2. Use std::move when adding diff to listDiff in GetListForBlock * Implement upgrade code for old CDeterministicMNListDiff format to new format * Track tipIndex instead of tipHeight/tipBlockHash * Store and pass around CBlockIndex* instead of block hash and height This allows us to switch CDeterministicMNManager::GetListForBlock to work with CBlockIndex. * Refactor CDeterministicMNManager::GetListForBlock to require CBlockIndex* Instead of requiring a block hash. This allows us to remove blockHash and prevBlockHash from CDeterministicMNListDiff without the use of cs_main locks in GetListForBlock. * Remove prevBlockHash, blockHash and nHeight from CDeterministicMNListDiff * Remove access to determinisitcMNManager in CMasternodeMetaMan::ToString() The deterministic MN manager is not fully initialized yet at the time this is called, which results in an empty list being returned everytime. * Better logic to determine if an upgrade is needed Reuse the "best block" logic to figure out if an upgrade is needed. Also use it to ensure that older nodes are unable to start after the upgrade was performed. * Return null block hash if it was requested with getmnlistdiff * bump CGovernanceManager::SERIALIZATION_VERSION_STRING * Check SERIALIZATION_VERSION_STRING before deserializing anything else * Invoke Clear() before deserializing just to be sure
2019-07-09 07:59:57 +02:00
CDeterministicMNCPtr GetMNByInternalId(uint64_t internalId) const;
CDeterministicMNCPtr GetMNPayee() const;
/**
* Calculates the projected MN payees for the next *count* blocks. The result is not guaranteed to be correct
* as PoSe banning might occur later
* @param count
* @return
*/
std::vector<CDeterministicMNCPtr> GetProjectedMNPayees(int nCount) const;
/**
* Calculate a quorum based on the modifier. The resulting list is deterministically sorted by score
* @param maxSize
* @param modifier
* @return
*/
std::vector<CDeterministicMNCPtr> CalculateQuorum(size_t maxSize, const uint256& modifier) const;
std::vector<std::pair<arith_uint256, CDeterministicMNCPtr>> CalculateScores(const uint256& modifier) const;
/**
* Calculates the maximum penalty which is allowed at the height of this MN list. It is dynamic and might change
* for every block.
* @return
*/
int CalcMaxPoSePenalty() const;
/**
* Returns a the given percentage from the max penalty for this MN list. Always use this method to calculate the
* value later passed to PoSePunish. The percentage should be high enough to take per-block penalty decreasing for MNs
* into account. This means, if you want to accept 2 failures per payment cycle, you should choose a percentage that
* is higher then 50%, e.g. 66%.
* @param percent
* @return
*/
int CalcPenalty(int percent) const;
/**
* Punishes a MN for misbehavior. If the resulting penalty score of the MN reaches the max penalty, it is banned.
* Penalty scores are only increased when the MN is not already banned, which means that after banning the penalty
* might appear lower then the current max penalty, while the MN is still banned.
* @param proTxHash
* @param penalty
*/
void PoSePunish(const uint256& proTxHash, int penalty, bool debugLogs);
/**
* Decrease penalty score of MN by 1.
* Only allowed on non-banned MNs.
* @param proTxHash
*/
void PoSeDecrease(const uint256& proTxHash);
CDeterministicMNListDiff BuildDiff(const CDeterministicMNList& to) const;
CSimplifiedMNListDiff BuildSimplifiedDiff(const CDeterministicMNList& to) const;
Optimize on-disk deterministic masternode storage to reduce size of evodb (#3017) * Implement CompactFull() in CDBWrapper This allows to compact the whole DB in one go. * Implement more compact version of CDeterministicMNListDiff This introduces CDeterministicMNStateDiff which requires to only store fields on-disk which actually changed. * Avoid writing mnUniquePropertyMap to disk when storing snapshots This map can be rebuilt by simply using AddMN for each deserialized MN. * Implement Serialize/Unserialize in CScript This allows us to directly use READWRITE() on scripts and removes the need for the ugly cast to CScriptBase. This commit also changes all Dash specific uses of CScript to not use the cast. * Keep track of registeration counts and introduce internalID for masternodes The "internalId" is simply the number of MNs registered so far when the new MN is added. It is deterministic and stays the same forever. * Use internalId as keys in MN list diffs This reduces the used size on-disk. * Two simple speedups in MN list diff handling 1. Avoid full compare if dmn or state pointers match in BuildDiff 2. Use std::move when adding diff to listDiff in GetListForBlock * Implement upgrade code for old CDeterministicMNListDiff format to new format * Track tipIndex instead of tipHeight/tipBlockHash * Store and pass around CBlockIndex* instead of block hash and height This allows us to switch CDeterministicMNManager::GetListForBlock to work with CBlockIndex. * Refactor CDeterministicMNManager::GetListForBlock to require CBlockIndex* Instead of requiring a block hash. This allows us to remove blockHash and prevBlockHash from CDeterministicMNListDiff without the use of cs_main locks in GetListForBlock. * Remove prevBlockHash, blockHash and nHeight from CDeterministicMNListDiff * Remove access to determinisitcMNManager in CMasternodeMetaMan::ToString() The deterministic MN manager is not fully initialized yet at the time this is called, which results in an empty list being returned everytime. * Better logic to determine if an upgrade is needed Reuse the "best block" logic to figure out if an upgrade is needed. Also use it to ensure that older nodes are unable to start after the upgrade was performed. * Return null block hash if it was requested with getmnlistdiff * bump CGovernanceManager::SERIALIZATION_VERSION_STRING * Check SERIALIZATION_VERSION_STRING before deserializing anything else * Invoke Clear() before deserializing just to be sure
2019-07-09 07:59:57 +02:00
CDeterministicMNList ApplyDiff(const CBlockIndex* pindex, const CDeterministicMNListDiff& diff) const;
void AddMN(const CDeterministicMNCPtr& dmn, bool fBumpTotalCount = true);
Optimize on-disk deterministic masternode storage to reduce size of evodb (#3017) * Implement CompactFull() in CDBWrapper This allows to compact the whole DB in one go. * Implement more compact version of CDeterministicMNListDiff This introduces CDeterministicMNStateDiff which requires to only store fields on-disk which actually changed. * Avoid writing mnUniquePropertyMap to disk when storing snapshots This map can be rebuilt by simply using AddMN for each deserialized MN. * Implement Serialize/Unserialize in CScript This allows us to directly use READWRITE() on scripts and removes the need for the ugly cast to CScriptBase. This commit also changes all Dash specific uses of CScript to not use the cast. * Keep track of registeration counts and introduce internalID for masternodes The "internalId" is simply the number of MNs registered so far when the new MN is added. It is deterministic and stays the same forever. * Use internalId as keys in MN list diffs This reduces the used size on-disk. * Two simple speedups in MN list diff handling 1. Avoid full compare if dmn or state pointers match in BuildDiff 2. Use std::move when adding diff to listDiff in GetListForBlock * Implement upgrade code for old CDeterministicMNListDiff format to new format * Track tipIndex instead of tipHeight/tipBlockHash * Store and pass around CBlockIndex* instead of block hash and height This allows us to switch CDeterministicMNManager::GetListForBlock to work with CBlockIndex. * Refactor CDeterministicMNManager::GetListForBlock to require CBlockIndex* Instead of requiring a block hash. This allows us to remove blockHash and prevBlockHash from CDeterministicMNListDiff without the use of cs_main locks in GetListForBlock. * Remove prevBlockHash, blockHash and nHeight from CDeterministicMNListDiff * Remove access to determinisitcMNManager in CMasternodeMetaMan::ToString() The deterministic MN manager is not fully initialized yet at the time this is called, which results in an empty list being returned everytime. * Better logic to determine if an upgrade is needed Reuse the "best block" logic to figure out if an upgrade is needed. Also use it to ensure that older nodes are unable to start after the upgrade was performed. * Return null block hash if it was requested with getmnlistdiff * bump CGovernanceManager::SERIALIZATION_VERSION_STRING * Check SERIALIZATION_VERSION_STRING before deserializing anything else * Invoke Clear() before deserializing just to be sure
2019-07-09 07:59:57 +02:00
void UpdateMN(const CDeterministicMNCPtr& oldDmn, const CDeterministicMNStateCPtr& pdmnState);
void UpdateMN(const uint256& proTxHash, const CDeterministicMNStateCPtr& pdmnState);
Optimize on-disk deterministic masternode storage to reduce size of evodb (#3017) * Implement CompactFull() in CDBWrapper This allows to compact the whole DB in one go. * Implement more compact version of CDeterministicMNListDiff This introduces CDeterministicMNStateDiff which requires to only store fields on-disk which actually changed. * Avoid writing mnUniquePropertyMap to disk when storing snapshots This map can be rebuilt by simply using AddMN for each deserialized MN. * Implement Serialize/Unserialize in CScript This allows us to directly use READWRITE() on scripts and removes the need for the ugly cast to CScriptBase. This commit also changes all Dash specific uses of CScript to not use the cast. * Keep track of registeration counts and introduce internalID for masternodes The "internalId" is simply the number of MNs registered so far when the new MN is added. It is deterministic and stays the same forever. * Use internalId as keys in MN list diffs This reduces the used size on-disk. * Two simple speedups in MN list diff handling 1. Avoid full compare if dmn or state pointers match in BuildDiff 2. Use std::move when adding diff to listDiff in GetListForBlock * Implement upgrade code for old CDeterministicMNListDiff format to new format * Track tipIndex instead of tipHeight/tipBlockHash * Store and pass around CBlockIndex* instead of block hash and height This allows us to switch CDeterministicMNManager::GetListForBlock to work with CBlockIndex. * Refactor CDeterministicMNManager::GetListForBlock to require CBlockIndex* Instead of requiring a block hash. This allows us to remove blockHash and prevBlockHash from CDeterministicMNListDiff without the use of cs_main locks in GetListForBlock. * Remove prevBlockHash, blockHash and nHeight from CDeterministicMNListDiff * Remove access to determinisitcMNManager in CMasternodeMetaMan::ToString() The deterministic MN manager is not fully initialized yet at the time this is called, which results in an empty list being returned everytime. * Better logic to determine if an upgrade is needed Reuse the "best block" logic to figure out if an upgrade is needed. Also use it to ensure that older nodes are unable to start after the upgrade was performed. * Return null block hash if it was requested with getmnlistdiff * bump CGovernanceManager::SERIALIZATION_VERSION_STRING * Check SERIALIZATION_VERSION_STRING before deserializing anything else * Invoke Clear() before deserializing just to be sure
2019-07-09 07:59:57 +02:00
void UpdateMN(const CDeterministicMNCPtr& oldDmn, const CDeterministicMNStateDiff& stateDiff);
void RemoveMN(const uint256& proTxHash);
template <typename T>
bool HasUniqueProperty(const T& v) const
{
return mnUniquePropertyMap.count(::SerializeHash(v)) != 0;
}
template <typename T>
CDeterministicMNCPtr GetUniquePropertyMN(const T& v) const
{
auto p = mnUniquePropertyMap.find(::SerializeHash(v));
if (!p) {
return nullptr;
}
return GetMN(p->first);
}
private:
template <typename T>
NODISCARD bool AddUniqueProperty(const CDeterministicMNCPtr& dmn, const T& v)
{
static const T nullValue;
if (v == nullValue) {
return false;
}
auto hash = ::SerializeHash(v);
auto oldEntry = mnUniquePropertyMap.find(hash);
if (oldEntry != nullptr && oldEntry->first != dmn->proTxHash) {
return false;
}
std::pair<uint256, uint32_t> newEntry(dmn->proTxHash, 1);
if (oldEntry != nullptr) {
newEntry.second = oldEntry->second + 1;
}
mnUniquePropertyMap = mnUniquePropertyMap.set(hash, newEntry);
return true;
}
template <typename T>
NODISCARD bool DeleteUniqueProperty(const CDeterministicMNCPtr& dmn, const T& oldValue)
{
static const T nullValue;
if (oldValue == nullValue) {
return false;
}
auto oldHash = ::SerializeHash(oldValue);
auto p = mnUniquePropertyMap.find(oldHash);
if (p == nullptr || p->first != dmn->proTxHash) {
return false;
}
if (p->second == 1) {
mnUniquePropertyMap = mnUniquePropertyMap.erase(oldHash);
} else {
mnUniquePropertyMap = mnUniquePropertyMap.set(oldHash, std::make_pair(dmn->proTxHash, p->second - 1));
}
return true;
}
template <typename T>
NODISCARD bool UpdateUniqueProperty(const CDeterministicMNCPtr& dmn, const T& oldValue, const T& newValue)
{
if (oldValue == newValue) {
return true;
}
static const T nullValue;
if (oldValue != nullValue && !DeleteUniqueProperty(dmn, oldValue)) {
return false;
}
if (newValue != nullValue && !AddUniqueProperty(dmn, newValue)) {
return false;
}
return true;
}
};
class CDeterministicMNListDiff
{
public:
2020-06-09 06:43:34 +02:00
int nHeight{-1}; //memory only
Optimize on-disk deterministic masternode storage to reduce size of evodb (#3017) * Implement CompactFull() in CDBWrapper This allows to compact the whole DB in one go. * Implement more compact version of CDeterministicMNListDiff This introduces CDeterministicMNStateDiff which requires to only store fields on-disk which actually changed. * Avoid writing mnUniquePropertyMap to disk when storing snapshots This map can be rebuilt by simply using AddMN for each deserialized MN. * Implement Serialize/Unserialize in CScript This allows us to directly use READWRITE() on scripts and removes the need for the ugly cast to CScriptBase. This commit also changes all Dash specific uses of CScript to not use the cast. * Keep track of registeration counts and introduce internalID for masternodes The "internalId" is simply the number of MNs registered so far when the new MN is added. It is deterministic and stays the same forever. * Use internalId as keys in MN list diffs This reduces the used size on-disk. * Two simple speedups in MN list diff handling 1. Avoid full compare if dmn or state pointers match in BuildDiff 2. Use std::move when adding diff to listDiff in GetListForBlock * Implement upgrade code for old CDeterministicMNListDiff format to new format * Track tipIndex instead of tipHeight/tipBlockHash * Store and pass around CBlockIndex* instead of block hash and height This allows us to switch CDeterministicMNManager::GetListForBlock to work with CBlockIndex. * Refactor CDeterministicMNManager::GetListForBlock to require CBlockIndex* Instead of requiring a block hash. This allows us to remove blockHash and prevBlockHash from CDeterministicMNListDiff without the use of cs_main locks in GetListForBlock. * Remove prevBlockHash, blockHash and nHeight from CDeterministicMNListDiff * Remove access to determinisitcMNManager in CMasternodeMetaMan::ToString() The deterministic MN manager is not fully initialized yet at the time this is called, which results in an empty list being returned everytime. * Better logic to determine if an upgrade is needed Reuse the "best block" logic to figure out if an upgrade is needed. Also use it to ensure that older nodes are unable to start after the upgrade was performed. * Return null block hash if it was requested with getmnlistdiff * bump CGovernanceManager::SERIALIZATION_VERSION_STRING * Check SERIALIZATION_VERSION_STRING before deserializing anything else * Invoke Clear() before deserializing just to be sure
2019-07-09 07:59:57 +02:00
std::vector<CDeterministicMNCPtr> addedMNs;
// keys are all relating to the internalId of MNs
std::map<uint64_t, CDeterministicMNStateDiff> updatedMNs;
std::set<uint64_t> removedMns;
public:
Optimize on-disk deterministic masternode storage to reduce size of evodb (#3017) * Implement CompactFull() in CDBWrapper This allows to compact the whole DB in one go. * Implement more compact version of CDeterministicMNListDiff This introduces CDeterministicMNStateDiff which requires to only store fields on-disk which actually changed. * Avoid writing mnUniquePropertyMap to disk when storing snapshots This map can be rebuilt by simply using AddMN for each deserialized MN. * Implement Serialize/Unserialize in CScript This allows us to directly use READWRITE() on scripts and removes the need for the ugly cast to CScriptBase. This commit also changes all Dash specific uses of CScript to not use the cast. * Keep track of registeration counts and introduce internalID for masternodes The "internalId" is simply the number of MNs registered so far when the new MN is added. It is deterministic and stays the same forever. * Use internalId as keys in MN list diffs This reduces the used size on-disk. * Two simple speedups in MN list diff handling 1. Avoid full compare if dmn or state pointers match in BuildDiff 2. Use std::move when adding diff to listDiff in GetListForBlock * Implement upgrade code for old CDeterministicMNListDiff format to new format * Track tipIndex instead of tipHeight/tipBlockHash * Store and pass around CBlockIndex* instead of block hash and height This allows us to switch CDeterministicMNManager::GetListForBlock to work with CBlockIndex. * Refactor CDeterministicMNManager::GetListForBlock to require CBlockIndex* Instead of requiring a block hash. This allows us to remove blockHash and prevBlockHash from CDeterministicMNListDiff without the use of cs_main locks in GetListForBlock. * Remove prevBlockHash, blockHash and nHeight from CDeterministicMNListDiff * Remove access to determinisitcMNManager in CMasternodeMetaMan::ToString() The deterministic MN manager is not fully initialized yet at the time this is called, which results in an empty list being returned everytime. * Better logic to determine if an upgrade is needed Reuse the "best block" logic to figure out if an upgrade is needed. Also use it to ensure that older nodes are unable to start after the upgrade was performed. * Return null block hash if it was requested with getmnlistdiff * bump CGovernanceManager::SERIALIZATION_VERSION_STRING * Check SERIALIZATION_VERSION_STRING before deserializing anything else * Invoke Clear() before deserializing just to be sure
2019-07-09 07:59:57 +02:00
template<typename Stream>
void Serialize(Stream& s) const
{
s << addedMNs;
WriteCompactSize(s, updatedMNs.size());
for (const auto& p : updatedMNs) {
WriteVarInt<Stream, VarIntMode::DEFAULT, uint64_t>(s, p.first);
Optimize on-disk deterministic masternode storage to reduce size of evodb (#3017) * Implement CompactFull() in CDBWrapper This allows to compact the whole DB in one go. * Implement more compact version of CDeterministicMNListDiff This introduces CDeterministicMNStateDiff which requires to only store fields on-disk which actually changed. * Avoid writing mnUniquePropertyMap to disk when storing snapshots This map can be rebuilt by simply using AddMN for each deserialized MN. * Implement Serialize/Unserialize in CScript This allows us to directly use READWRITE() on scripts and removes the need for the ugly cast to CScriptBase. This commit also changes all Dash specific uses of CScript to not use the cast. * Keep track of registeration counts and introduce internalID for masternodes The "internalId" is simply the number of MNs registered so far when the new MN is added. It is deterministic and stays the same forever. * Use internalId as keys in MN list diffs This reduces the used size on-disk. * Two simple speedups in MN list diff handling 1. Avoid full compare if dmn or state pointers match in BuildDiff 2. Use std::move when adding diff to listDiff in GetListForBlock * Implement upgrade code for old CDeterministicMNListDiff format to new format * Track tipIndex instead of tipHeight/tipBlockHash * Store and pass around CBlockIndex* instead of block hash and height This allows us to switch CDeterministicMNManager::GetListForBlock to work with CBlockIndex. * Refactor CDeterministicMNManager::GetListForBlock to require CBlockIndex* Instead of requiring a block hash. This allows us to remove blockHash and prevBlockHash from CDeterministicMNListDiff without the use of cs_main locks in GetListForBlock. * Remove prevBlockHash, blockHash and nHeight from CDeterministicMNListDiff * Remove access to determinisitcMNManager in CMasternodeMetaMan::ToString() The deterministic MN manager is not fully initialized yet at the time this is called, which results in an empty list being returned everytime. * Better logic to determine if an upgrade is needed Reuse the "best block" logic to figure out if an upgrade is needed. Also use it to ensure that older nodes are unable to start after the upgrade was performed. * Return null block hash if it was requested with getmnlistdiff * bump CGovernanceManager::SERIALIZATION_VERSION_STRING * Check SERIALIZATION_VERSION_STRING before deserializing anything else * Invoke Clear() before deserializing just to be sure
2019-07-09 07:59:57 +02:00
s << p.second;
}
WriteCompactSize(s, removedMns.size());
for (const auto& p : removedMns) {
WriteVarInt<Stream, VarIntMode::DEFAULT, uint64_t>(s, p);
Optimize on-disk deterministic masternode storage to reduce size of evodb (#3017) * Implement CompactFull() in CDBWrapper This allows to compact the whole DB in one go. * Implement more compact version of CDeterministicMNListDiff This introduces CDeterministicMNStateDiff which requires to only store fields on-disk which actually changed. * Avoid writing mnUniquePropertyMap to disk when storing snapshots This map can be rebuilt by simply using AddMN for each deserialized MN. * Implement Serialize/Unserialize in CScript This allows us to directly use READWRITE() on scripts and removes the need for the ugly cast to CScriptBase. This commit also changes all Dash specific uses of CScript to not use the cast. * Keep track of registeration counts and introduce internalID for masternodes The "internalId" is simply the number of MNs registered so far when the new MN is added. It is deterministic and stays the same forever. * Use internalId as keys in MN list diffs This reduces the used size on-disk. * Two simple speedups in MN list diff handling 1. Avoid full compare if dmn or state pointers match in BuildDiff 2. Use std::move when adding diff to listDiff in GetListForBlock * Implement upgrade code for old CDeterministicMNListDiff format to new format * Track tipIndex instead of tipHeight/tipBlockHash * Store and pass around CBlockIndex* instead of block hash and height This allows us to switch CDeterministicMNManager::GetListForBlock to work with CBlockIndex. * Refactor CDeterministicMNManager::GetListForBlock to require CBlockIndex* Instead of requiring a block hash. This allows us to remove blockHash and prevBlockHash from CDeterministicMNListDiff without the use of cs_main locks in GetListForBlock. * Remove prevBlockHash, blockHash and nHeight from CDeterministicMNListDiff * Remove access to determinisitcMNManager in CMasternodeMetaMan::ToString() The deterministic MN manager is not fully initialized yet at the time this is called, which results in an empty list being returned everytime. * Better logic to determine if an upgrade is needed Reuse the "best block" logic to figure out if an upgrade is needed. Also use it to ensure that older nodes are unable to start after the upgrade was performed. * Return null block hash if it was requested with getmnlistdiff * bump CGovernanceManager::SERIALIZATION_VERSION_STRING * Check SERIALIZATION_VERSION_STRING before deserializing anything else * Invoke Clear() before deserializing just to be sure
2019-07-09 07:59:57 +02:00
}
}
Optimize on-disk deterministic masternode storage to reduce size of evodb (#3017) * Implement CompactFull() in CDBWrapper This allows to compact the whole DB in one go. * Implement more compact version of CDeterministicMNListDiff This introduces CDeterministicMNStateDiff which requires to only store fields on-disk which actually changed. * Avoid writing mnUniquePropertyMap to disk when storing snapshots This map can be rebuilt by simply using AddMN for each deserialized MN. * Implement Serialize/Unserialize in CScript This allows us to directly use READWRITE() on scripts and removes the need for the ugly cast to CScriptBase. This commit also changes all Dash specific uses of CScript to not use the cast. * Keep track of registeration counts and introduce internalID for masternodes The "internalId" is simply the number of MNs registered so far when the new MN is added. It is deterministic and stays the same forever. * Use internalId as keys in MN list diffs This reduces the used size on-disk. * Two simple speedups in MN list diff handling 1. Avoid full compare if dmn or state pointers match in BuildDiff 2. Use std::move when adding diff to listDiff in GetListForBlock * Implement upgrade code for old CDeterministicMNListDiff format to new format * Track tipIndex instead of tipHeight/tipBlockHash * Store and pass around CBlockIndex* instead of block hash and height This allows us to switch CDeterministicMNManager::GetListForBlock to work with CBlockIndex. * Refactor CDeterministicMNManager::GetListForBlock to require CBlockIndex* Instead of requiring a block hash. This allows us to remove blockHash and prevBlockHash from CDeterministicMNListDiff without the use of cs_main locks in GetListForBlock. * Remove prevBlockHash, blockHash and nHeight from CDeterministicMNListDiff * Remove access to determinisitcMNManager in CMasternodeMetaMan::ToString() The deterministic MN manager is not fully initialized yet at the time this is called, which results in an empty list being returned everytime. * Better logic to determine if an upgrade is needed Reuse the "best block" logic to figure out if an upgrade is needed. Also use it to ensure that older nodes are unable to start after the upgrade was performed. * Return null block hash if it was requested with getmnlistdiff * bump CGovernanceManager::SERIALIZATION_VERSION_STRING * Check SERIALIZATION_VERSION_STRING before deserializing anything else * Invoke Clear() before deserializing just to be sure
2019-07-09 07:59:57 +02:00
template<typename Stream>
void Unserialize(Stream& s)
{
Optimize on-disk deterministic masternode storage to reduce size of evodb (#3017) * Implement CompactFull() in CDBWrapper This allows to compact the whole DB in one go. * Implement more compact version of CDeterministicMNListDiff This introduces CDeterministicMNStateDiff which requires to only store fields on-disk which actually changed. * Avoid writing mnUniquePropertyMap to disk when storing snapshots This map can be rebuilt by simply using AddMN for each deserialized MN. * Implement Serialize/Unserialize in CScript This allows us to directly use READWRITE() on scripts and removes the need for the ugly cast to CScriptBase. This commit also changes all Dash specific uses of CScript to not use the cast. * Keep track of registeration counts and introduce internalID for masternodes The "internalId" is simply the number of MNs registered so far when the new MN is added. It is deterministic and stays the same forever. * Use internalId as keys in MN list diffs This reduces the used size on-disk. * Two simple speedups in MN list diff handling 1. Avoid full compare if dmn or state pointers match in BuildDiff 2. Use std::move when adding diff to listDiff in GetListForBlock * Implement upgrade code for old CDeterministicMNListDiff format to new format * Track tipIndex instead of tipHeight/tipBlockHash * Store and pass around CBlockIndex* instead of block hash and height This allows us to switch CDeterministicMNManager::GetListForBlock to work with CBlockIndex. * Refactor CDeterministicMNManager::GetListForBlock to require CBlockIndex* Instead of requiring a block hash. This allows us to remove blockHash and prevBlockHash from CDeterministicMNListDiff without the use of cs_main locks in GetListForBlock. * Remove prevBlockHash, blockHash and nHeight from CDeterministicMNListDiff * Remove access to determinisitcMNManager in CMasternodeMetaMan::ToString() The deterministic MN manager is not fully initialized yet at the time this is called, which results in an empty list being returned everytime. * Better logic to determine if an upgrade is needed Reuse the "best block" logic to figure out if an upgrade is needed. Also use it to ensure that older nodes are unable to start after the upgrade was performed. * Return null block hash if it was requested with getmnlistdiff * bump CGovernanceManager::SERIALIZATION_VERSION_STRING * Check SERIALIZATION_VERSION_STRING before deserializing anything else * Invoke Clear() before deserializing just to be sure
2019-07-09 07:59:57 +02:00
updatedMNs.clear();
removedMns.clear();
size_t tmp;
uint64_t tmp2;
s >> addedMNs;
tmp = ReadCompactSize(s);
for (size_t i = 0; i < tmp; i++) {
CDeterministicMNStateDiff diff;
tmp2 = ReadVarInt<Stream, VarIntMode::DEFAULT, uint64_t>(s);
Optimize on-disk deterministic masternode storage to reduce size of evodb (#3017) * Implement CompactFull() in CDBWrapper This allows to compact the whole DB in one go. * Implement more compact version of CDeterministicMNListDiff This introduces CDeterministicMNStateDiff which requires to only store fields on-disk which actually changed. * Avoid writing mnUniquePropertyMap to disk when storing snapshots This map can be rebuilt by simply using AddMN for each deserialized MN. * Implement Serialize/Unserialize in CScript This allows us to directly use READWRITE() on scripts and removes the need for the ugly cast to CScriptBase. This commit also changes all Dash specific uses of CScript to not use the cast. * Keep track of registeration counts and introduce internalID for masternodes The "internalId" is simply the number of MNs registered so far when the new MN is added. It is deterministic and stays the same forever. * Use internalId as keys in MN list diffs This reduces the used size on-disk. * Two simple speedups in MN list diff handling 1. Avoid full compare if dmn or state pointers match in BuildDiff 2. Use std::move when adding diff to listDiff in GetListForBlock * Implement upgrade code for old CDeterministicMNListDiff format to new format * Track tipIndex instead of tipHeight/tipBlockHash * Store and pass around CBlockIndex* instead of block hash and height This allows us to switch CDeterministicMNManager::GetListForBlock to work with CBlockIndex. * Refactor CDeterministicMNManager::GetListForBlock to require CBlockIndex* Instead of requiring a block hash. This allows us to remove blockHash and prevBlockHash from CDeterministicMNListDiff without the use of cs_main locks in GetListForBlock. * Remove prevBlockHash, blockHash and nHeight from CDeterministicMNListDiff * Remove access to determinisitcMNManager in CMasternodeMetaMan::ToString() The deterministic MN manager is not fully initialized yet at the time this is called, which results in an empty list being returned everytime. * Better logic to determine if an upgrade is needed Reuse the "best block" logic to figure out if an upgrade is needed. Also use it to ensure that older nodes are unable to start after the upgrade was performed. * Return null block hash if it was requested with getmnlistdiff * bump CGovernanceManager::SERIALIZATION_VERSION_STRING * Check SERIALIZATION_VERSION_STRING before deserializing anything else * Invoke Clear() before deserializing just to be sure
2019-07-09 07:59:57 +02:00
s >> diff;
updatedMNs.emplace(tmp2, std::move(diff));
}
tmp = ReadCompactSize(s);
for (size_t i = 0; i < tmp; i++) {
tmp2 = ReadVarInt<Stream, VarIntMode::DEFAULT, uint64_t>(s);
Optimize on-disk deterministic masternode storage to reduce size of evodb (#3017) * Implement CompactFull() in CDBWrapper This allows to compact the whole DB in one go. * Implement more compact version of CDeterministicMNListDiff This introduces CDeterministicMNStateDiff which requires to only store fields on-disk which actually changed. * Avoid writing mnUniquePropertyMap to disk when storing snapshots This map can be rebuilt by simply using AddMN for each deserialized MN. * Implement Serialize/Unserialize in CScript This allows us to directly use READWRITE() on scripts and removes the need for the ugly cast to CScriptBase. This commit also changes all Dash specific uses of CScript to not use the cast. * Keep track of registeration counts and introduce internalID for masternodes The "internalId" is simply the number of MNs registered so far when the new MN is added. It is deterministic and stays the same forever. * Use internalId as keys in MN list diffs This reduces the used size on-disk. * Two simple speedups in MN list diff handling 1. Avoid full compare if dmn or state pointers match in BuildDiff 2. Use std::move when adding diff to listDiff in GetListForBlock * Implement upgrade code for old CDeterministicMNListDiff format to new format * Track tipIndex instead of tipHeight/tipBlockHash * Store and pass around CBlockIndex* instead of block hash and height This allows us to switch CDeterministicMNManager::GetListForBlock to work with CBlockIndex. * Refactor CDeterministicMNManager::GetListForBlock to require CBlockIndex* Instead of requiring a block hash. This allows us to remove blockHash and prevBlockHash from CDeterministicMNListDiff without the use of cs_main locks in GetListForBlock. * Remove prevBlockHash, blockHash and nHeight from CDeterministicMNListDiff * Remove access to determinisitcMNManager in CMasternodeMetaMan::ToString() The deterministic MN manager is not fully initialized yet at the time this is called, which results in an empty list being returned everytime. * Better logic to determine if an upgrade is needed Reuse the "best block" logic to figure out if an upgrade is needed. Also use it to ensure that older nodes are unable to start after the upgrade was performed. * Return null block hash if it was requested with getmnlistdiff * bump CGovernanceManager::SERIALIZATION_VERSION_STRING * Check SERIALIZATION_VERSION_STRING before deserializing anything else * Invoke Clear() before deserializing just to be sure
2019-07-09 07:59:57 +02:00
removedMns.emplace(tmp2);
}
}
public:
bool HasChanges() const
{
return !addedMNs.empty() || !updatedMNs.empty() || !removedMns.empty();
}
};
Optimize on-disk deterministic masternode storage to reduce size of evodb (#3017) * Implement CompactFull() in CDBWrapper This allows to compact the whole DB in one go. * Implement more compact version of CDeterministicMNListDiff This introduces CDeterministicMNStateDiff which requires to only store fields on-disk which actually changed. * Avoid writing mnUniquePropertyMap to disk when storing snapshots This map can be rebuilt by simply using AddMN for each deserialized MN. * Implement Serialize/Unserialize in CScript This allows us to directly use READWRITE() on scripts and removes the need for the ugly cast to CScriptBase. This commit also changes all Dash specific uses of CScript to not use the cast. * Keep track of registeration counts and introduce internalID for masternodes The "internalId" is simply the number of MNs registered so far when the new MN is added. It is deterministic and stays the same forever. * Use internalId as keys in MN list diffs This reduces the used size on-disk. * Two simple speedups in MN list diff handling 1. Avoid full compare if dmn or state pointers match in BuildDiff 2. Use std::move when adding diff to listDiff in GetListForBlock * Implement upgrade code for old CDeterministicMNListDiff format to new format * Track tipIndex instead of tipHeight/tipBlockHash * Store and pass around CBlockIndex* instead of block hash and height This allows us to switch CDeterministicMNManager::GetListForBlock to work with CBlockIndex. * Refactor CDeterministicMNManager::GetListForBlock to require CBlockIndex* Instead of requiring a block hash. This allows us to remove blockHash and prevBlockHash from CDeterministicMNListDiff without the use of cs_main locks in GetListForBlock. * Remove prevBlockHash, blockHash and nHeight from CDeterministicMNListDiff * Remove access to determinisitcMNManager in CMasternodeMetaMan::ToString() The deterministic MN manager is not fully initialized yet at the time this is called, which results in an empty list being returned everytime. * Better logic to determine if an upgrade is needed Reuse the "best block" logic to figure out if an upgrade is needed. Also use it to ensure that older nodes are unable to start after the upgrade was performed. * Return null block hash if it was requested with getmnlistdiff * bump CGovernanceManager::SERIALIZATION_VERSION_STRING * Check SERIALIZATION_VERSION_STRING before deserializing anything else * Invoke Clear() before deserializing just to be sure
2019-07-09 07:59:57 +02:00
// TODO can be removed in a future version
class CDeterministicMNListDiff_OldFormat
{
public:
uint256 prevBlockHash;
uint256 blockHash;
int nHeight{-1};
std::map<uint256, CDeterministicMNCPtr> addedMNs;
std::map<uint256, CDeterministicMNStateCPtr> updatedMNs;
std::set<uint256> removedMns;
public:
template<typename Stream>
void Unserialize(Stream& s) {
addedMNs.clear();
s >> prevBlockHash;
s >> blockHash;
s >> nHeight;
size_t cnt = ReadCompactSize(s);
for (size_t i = 0; i < cnt; i++) {
uint256 proTxHash;
// NOTE: This is a hack and "0" is just a dummy id. The actual internalId is assigned to a copy
// of this dmn via corresponding ctor when we convert the diff format to a new one in UpgradeDiff
// thus the logic that we must set internalId before dmn is used in any meaningful way is preserved.
auto dmn = std::make_shared<CDeterministicMN>(0);
Optimize on-disk deterministic masternode storage to reduce size of evodb (#3017) * Implement CompactFull() in CDBWrapper This allows to compact the whole DB in one go. * Implement more compact version of CDeterministicMNListDiff This introduces CDeterministicMNStateDiff which requires to only store fields on-disk which actually changed. * Avoid writing mnUniquePropertyMap to disk when storing snapshots This map can be rebuilt by simply using AddMN for each deserialized MN. * Implement Serialize/Unserialize in CScript This allows us to directly use READWRITE() on scripts and removes the need for the ugly cast to CScriptBase. This commit also changes all Dash specific uses of CScript to not use the cast. * Keep track of registeration counts and introduce internalID for masternodes The "internalId" is simply the number of MNs registered so far when the new MN is added. It is deterministic and stays the same forever. * Use internalId as keys in MN list diffs This reduces the used size on-disk. * Two simple speedups in MN list diff handling 1. Avoid full compare if dmn or state pointers match in BuildDiff 2. Use std::move when adding diff to listDiff in GetListForBlock * Implement upgrade code for old CDeterministicMNListDiff format to new format * Track tipIndex instead of tipHeight/tipBlockHash * Store and pass around CBlockIndex* instead of block hash and height This allows us to switch CDeterministicMNManager::GetListForBlock to work with CBlockIndex. * Refactor CDeterministicMNManager::GetListForBlock to require CBlockIndex* Instead of requiring a block hash. This allows us to remove blockHash and prevBlockHash from CDeterministicMNListDiff without the use of cs_main locks in GetListForBlock. * Remove prevBlockHash, blockHash and nHeight from CDeterministicMNListDiff * Remove access to determinisitcMNManager in CMasternodeMetaMan::ToString() The deterministic MN manager is not fully initialized yet at the time this is called, which results in an empty list being returned everytime. * Better logic to determine if an upgrade is needed Reuse the "best block" logic to figure out if an upgrade is needed. Also use it to ensure that older nodes are unable to start after the upgrade was performed. * Return null block hash if it was requested with getmnlistdiff * bump CGovernanceManager::SERIALIZATION_VERSION_STRING * Check SERIALIZATION_VERSION_STRING before deserializing anything else * Invoke Clear() before deserializing just to be sure
2019-07-09 07:59:57 +02:00
s >> proTxHash;
dmn->Unserialize(s, true);
addedMNs.emplace(proTxHash, dmn);
}
s >> updatedMNs;
s >> removedMns;
}
};
class CDeterministicMNManager
{
2020-06-09 06:43:34 +02:00
static const int DISK_SNAPSHOT_PERIOD = 576; // once per day
static const int DISK_SNAPSHOTS = 3; // keep cache for 3 disk snapshots to have 2 full days covered
static const int LIST_DIFFS_CACHE_SIZE = DISK_SNAPSHOT_PERIOD * DISK_SNAPSHOTS;
public:
CCriticalSection cs;
private:
CEvoDB& evoDb;
2020-06-09 06:43:34 +02:00
std::unordered_map<uint256, CDeterministicMNList, StaticSaltedHasher> mnListsCache;
std::unordered_map<uint256, CDeterministicMNListDiff, StaticSaltedHasher> mnListDiffsCache;
Optimize on-disk deterministic masternode storage to reduce size of evodb (#3017) * Implement CompactFull() in CDBWrapper This allows to compact the whole DB in one go. * Implement more compact version of CDeterministicMNListDiff This introduces CDeterministicMNStateDiff which requires to only store fields on-disk which actually changed. * Avoid writing mnUniquePropertyMap to disk when storing snapshots This map can be rebuilt by simply using AddMN for each deserialized MN. * Implement Serialize/Unserialize in CScript This allows us to directly use READWRITE() on scripts and removes the need for the ugly cast to CScriptBase. This commit also changes all Dash specific uses of CScript to not use the cast. * Keep track of registeration counts and introduce internalID for masternodes The "internalId" is simply the number of MNs registered so far when the new MN is added. It is deterministic and stays the same forever. * Use internalId as keys in MN list diffs This reduces the used size on-disk. * Two simple speedups in MN list diff handling 1. Avoid full compare if dmn or state pointers match in BuildDiff 2. Use std::move when adding diff to listDiff in GetListForBlock * Implement upgrade code for old CDeterministicMNListDiff format to new format * Track tipIndex instead of tipHeight/tipBlockHash * Store and pass around CBlockIndex* instead of block hash and height This allows us to switch CDeterministicMNManager::GetListForBlock to work with CBlockIndex. * Refactor CDeterministicMNManager::GetListForBlock to require CBlockIndex* Instead of requiring a block hash. This allows us to remove blockHash and prevBlockHash from CDeterministicMNListDiff without the use of cs_main locks in GetListForBlock. * Remove prevBlockHash, blockHash and nHeight from CDeterministicMNListDiff * Remove access to determinisitcMNManager in CMasternodeMetaMan::ToString() The deterministic MN manager is not fully initialized yet at the time this is called, which results in an empty list being returned everytime. * Better logic to determine if an upgrade is needed Reuse the "best block" logic to figure out if an upgrade is needed. Also use it to ensure that older nodes are unable to start after the upgrade was performed. * Return null block hash if it was requested with getmnlistdiff * bump CGovernanceManager::SERIALIZATION_VERSION_STRING * Check SERIALIZATION_VERSION_STRING before deserializing anything else * Invoke Clear() before deserializing just to be sure
2019-07-09 07:59:57 +02:00
const CBlockIndex* tipIndex{nullptr};
public:
2020-01-05 01:55:41 +01:00
explicit CDeterministicMNManager(CEvoDB& _evoDb);
bool ProcessBlock(const CBlock& block, const CBlockIndex* pindex, CValidationState& state, const CCoinsViewCache& view, bool fJustCheck);
bool UndoBlock(const CBlock& block, const CBlockIndex* pindex);
void UpdatedBlockTip(const CBlockIndex* pindex);
// the returned list will not contain the correct block hash (we can't know it yet as the coinbase TX is not updated yet)
bool BuildNewListFromBlock(const CBlock& block, const CBlockIndex* pindexPrev, CValidationState& state, const CCoinsViewCache& view, CDeterministicMNList& mnListRet, bool debugLogs);
static void HandleQuorumCommitment(llmq::CFinalCommitment& qc, const CBlockIndex* pindexQuorum, CDeterministicMNList& mnList, bool debugLogs);
static void DecreasePoSePenalties(CDeterministicMNList& mnList);
Optimize on-disk deterministic masternode storage to reduce size of evodb (#3017) * Implement CompactFull() in CDBWrapper This allows to compact the whole DB in one go. * Implement more compact version of CDeterministicMNListDiff This introduces CDeterministicMNStateDiff which requires to only store fields on-disk which actually changed. * Avoid writing mnUniquePropertyMap to disk when storing snapshots This map can be rebuilt by simply using AddMN for each deserialized MN. * Implement Serialize/Unserialize in CScript This allows us to directly use READWRITE() on scripts and removes the need for the ugly cast to CScriptBase. This commit also changes all Dash specific uses of CScript to not use the cast. * Keep track of registeration counts and introduce internalID for masternodes The "internalId" is simply the number of MNs registered so far when the new MN is added. It is deterministic and stays the same forever. * Use internalId as keys in MN list diffs This reduces the used size on-disk. * Two simple speedups in MN list diff handling 1. Avoid full compare if dmn or state pointers match in BuildDiff 2. Use std::move when adding diff to listDiff in GetListForBlock * Implement upgrade code for old CDeterministicMNListDiff format to new format * Track tipIndex instead of tipHeight/tipBlockHash * Store and pass around CBlockIndex* instead of block hash and height This allows us to switch CDeterministicMNManager::GetListForBlock to work with CBlockIndex. * Refactor CDeterministicMNManager::GetListForBlock to require CBlockIndex* Instead of requiring a block hash. This allows us to remove blockHash and prevBlockHash from CDeterministicMNListDiff without the use of cs_main locks in GetListForBlock. * Remove prevBlockHash, blockHash and nHeight from CDeterministicMNListDiff * Remove access to determinisitcMNManager in CMasternodeMetaMan::ToString() The deterministic MN manager is not fully initialized yet at the time this is called, which results in an empty list being returned everytime. * Better logic to determine if an upgrade is needed Reuse the "best block" logic to figure out if an upgrade is needed. Also use it to ensure that older nodes are unable to start after the upgrade was performed. * Return null block hash if it was requested with getmnlistdiff * bump CGovernanceManager::SERIALIZATION_VERSION_STRING * Check SERIALIZATION_VERSION_STRING before deserializing anything else * Invoke Clear() before deserializing just to be sure
2019-07-09 07:59:57 +02:00
CDeterministicMNList GetListForBlock(const CBlockIndex* pindex);
CDeterministicMNList GetListAtChainTip();
2018-10-25 16:29:50 +02:00
// Test if given TX is a ProRegTx which also contains the collateral at index n
static bool IsProTxWithCollateral(const CTransactionRef& tx, uint32_t n);
bool IsDIP3Enforced(int nHeight = -1);
Optimize on-disk deterministic masternode storage to reduce size of evodb (#3017) * Implement CompactFull() in CDBWrapper This allows to compact the whole DB in one go. * Implement more compact version of CDeterministicMNListDiff This introduces CDeterministicMNStateDiff which requires to only store fields on-disk which actually changed. * Avoid writing mnUniquePropertyMap to disk when storing snapshots This map can be rebuilt by simply using AddMN for each deserialized MN. * Implement Serialize/Unserialize in CScript This allows us to directly use READWRITE() on scripts and removes the need for the ugly cast to CScriptBase. This commit also changes all Dash specific uses of CScript to not use the cast. * Keep track of registeration counts and introduce internalID for masternodes The "internalId" is simply the number of MNs registered so far when the new MN is added. It is deterministic and stays the same forever. * Use internalId as keys in MN list diffs This reduces the used size on-disk. * Two simple speedups in MN list diff handling 1. Avoid full compare if dmn or state pointers match in BuildDiff 2. Use std::move when adding diff to listDiff in GetListForBlock * Implement upgrade code for old CDeterministicMNListDiff format to new format * Track tipIndex instead of tipHeight/tipBlockHash * Store and pass around CBlockIndex* instead of block hash and height This allows us to switch CDeterministicMNManager::GetListForBlock to work with CBlockIndex. * Refactor CDeterministicMNManager::GetListForBlock to require CBlockIndex* Instead of requiring a block hash. This allows us to remove blockHash and prevBlockHash from CDeterministicMNListDiff without the use of cs_main locks in GetListForBlock. * Remove prevBlockHash, blockHash and nHeight from CDeterministicMNListDiff * Remove access to determinisitcMNManager in CMasternodeMetaMan::ToString() The deterministic MN manager is not fully initialized yet at the time this is called, which results in an empty list being returned everytime. * Better logic to determine if an upgrade is needed Reuse the "best block" logic to figure out if an upgrade is needed. Also use it to ensure that older nodes are unable to start after the upgrade was performed. * Return null block hash if it was requested with getmnlistdiff * bump CGovernanceManager::SERIALIZATION_VERSION_STRING * Check SERIALIZATION_VERSION_STRING before deserializing anything else * Invoke Clear() before deserializing just to be sure
2019-07-09 07:59:57 +02:00
public:
// TODO these can all be removed in a future version
void UpgradeDiff(CDBBatch& batch, const CBlockIndex* pindexNext, const CDeterministicMNList& curMNList, CDeterministicMNList& newMNList);
bool UpgradeDBIfNeeded();
Optimize on-disk deterministic masternode storage to reduce size of evodb (#3017) * Implement CompactFull() in CDBWrapper This allows to compact the whole DB in one go. * Implement more compact version of CDeterministicMNListDiff This introduces CDeterministicMNStateDiff which requires to only store fields on-disk which actually changed. * Avoid writing mnUniquePropertyMap to disk when storing snapshots This map can be rebuilt by simply using AddMN for each deserialized MN. * Implement Serialize/Unserialize in CScript This allows us to directly use READWRITE() on scripts and removes the need for the ugly cast to CScriptBase. This commit also changes all Dash specific uses of CScript to not use the cast. * Keep track of registeration counts and introduce internalID for masternodes The "internalId" is simply the number of MNs registered so far when the new MN is added. It is deterministic and stays the same forever. * Use internalId as keys in MN list diffs This reduces the used size on-disk. * Two simple speedups in MN list diff handling 1. Avoid full compare if dmn or state pointers match in BuildDiff 2. Use std::move when adding diff to listDiff in GetListForBlock * Implement upgrade code for old CDeterministicMNListDiff format to new format * Track tipIndex instead of tipHeight/tipBlockHash * Store and pass around CBlockIndex* instead of block hash and height This allows us to switch CDeterministicMNManager::GetListForBlock to work with CBlockIndex. * Refactor CDeterministicMNManager::GetListForBlock to require CBlockIndex* Instead of requiring a block hash. This allows us to remove blockHash and prevBlockHash from CDeterministicMNListDiff without the use of cs_main locks in GetListForBlock. * Remove prevBlockHash, blockHash and nHeight from CDeterministicMNListDiff * Remove access to determinisitcMNManager in CMasternodeMetaMan::ToString() The deterministic MN manager is not fully initialized yet at the time this is called, which results in an empty list being returned everytime. * Better logic to determine if an upgrade is needed Reuse the "best block" logic to figure out if an upgrade is needed. Also use it to ensure that older nodes are unable to start after the upgrade was performed. * Return null block hash if it was requested with getmnlistdiff * bump CGovernanceManager::SERIALIZATION_VERSION_STRING * Check SERIALIZATION_VERSION_STRING before deserializing anything else * Invoke Clear() before deserializing just to be sure
2019-07-09 07:59:57 +02:00
private:
void CleanupCache(int nHeight);
};
Merge #11043: Use std::unique_ptr (C++11) where possible a357293 Use MakeUnique<Db>(...) (practicalswift) 3e09b39 Use MakeUnique<T>(...) instead of std::unique_ptr<T>(new T(...)) (practicalswift) 8617989 Add MakeUnique (substitute for C++14 std::make_unique) (practicalswift) d223bc9 Use unique_ptr for pcoinscatcher/pcoinsdbview/pcoinsTip/pblocktree (practicalswift) b45c597 Use unique_ptr for pdbCopy (Db) and fix potential memory leak (practicalswift) 29ab96d Use unique_ptr for dbenv (DbEnv) (practicalswift) f72cbf9 Use unique_ptr for pfilter (CBloomFilter) (practicalswift) 8ccf1bb Use unique_ptr for sem{Addnode,Outbound} (CSemaphore) (practicalswift) 73db063 Use unique_ptr for upnp_thread (boost::thread) (practicalswift) 0024531 Use unique_ptr for dbw (CDBWrapper) (practicalswift) fa6d122 Use unique_ptr:s for {fee,short,long}Stats (TxConfirmStats) (practicalswift) 5a6f768 Use unique_ptr for httpRPCTimerInterface (HTTPRPCTimerInterface) (practicalswift) 860e912 Use unique_ptr for pwalletMain (CWallet) (practicalswift) Pull request description: Use `std::unique_ptr` (C++11) where possible. Rationale: 1. Avoid resource leaks (specifically: forgetting to `delete` an object created using `new`) 2. Avoid undefined behaviour (specifically: double `delete`:s) **Note to reviewers:** Please let me know if I've missed any obvious `std::unique_ptr` candidates. Hopefully this PR should cover all the trivial cases. Tree-SHA512: 9fbeb47b800ab8ff4e0be9f2a22ab63c23d5c613a0c6716d9183db8d22ddbbce592fb8384a8b7874bf7375c8161efb13ca2197ad6f24b75967148037f0f7b20c
2017-11-09 21:22:08 +01:00
extern std::unique_ptr<CDeterministicMNManager> deterministicMNManager;
#endif // BITCOIN_EVO_DETERMINISTICMNS_H