From 05e5ee1be662532e9a93f971d47775e0b87a6ff5 Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <6098974-kittywhiskers@users.noreply.gitlab.com> Date: Wed, 26 May 2021 15:04:39 +0300 Subject: [PATCH 01/14] partial merge #8149: import OverrideStream from "BIP144: Serialization, hashes, relay (sender side)" --- src/streams.h | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/streams.h b/src/streams.h index 21db4ba6dd..5d0be5ab9c 100644 --- a/src/streams.h +++ b/src/streams.h @@ -22,6 +22,33 @@ #include #include +template +class OverrideStream +{ + Stream* stream; +public: + const int nType; + const int nVersion; + + OverrideStream(Stream* stream_, int nType_, int nVersion_) : stream(stream_), nType(nType_), nVersion(nVersion_) {} + + template + OverrideStream& operator<<(const T& obj) + { + // Serialize to this stream + ::Serialize(*this->stream, obj); + return (*this); + } + + template + OverrideStream& operator>>(T& obj) + { + // Unserialize from this stream + ::Unserialize(*this->stream, obj); + return (*this); + } +}; + /* Minimal stream for overwriting and/or appending to an existing byte vector * * The referenced vector will grow as necessary From 95209acdc34ae35a00c2d08d78e276867b26b541 Mon Sep 17 00:00:00 2001 From: UdjinM6 Date: Wed, 26 May 2021 15:14:27 +0300 Subject: [PATCH 02/14] partial merge #8149: `ser_vector` changes from `[qa] p2p segwit tests` commit --- test/functional/test_framework/messages.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/test/functional/test_framework/messages.py b/test/functional/test_framework/messages.py index 5d3df24c7c..b413d3b5f9 100755 --- a/test/functional/test_framework/messages.py +++ b/test/functional/test_framework/messages.py @@ -127,10 +127,16 @@ def deser_vector(f, c): return r -def ser_vector(l): +# ser_function_name: Allow for an alternate serialization function on the +# entries in the vector (we use this for serializing the vector of transactions +# for a witness block). +def ser_vector(l, ser_function_name=None): r = ser_compact_size(len(l)) for i in l: - r += i.serialize() + if ser_function_name: + r += getattr(i, ser_function_name)() + else: + r += i.serialize() return r From ade894eb0f0b29a1b6fbbdf6018632bacfa601c8 Mon Sep 17 00:00:00 2001 From: UdjinM6 Date: Wed, 26 May 2021 15:24:20 +0300 Subject: [PATCH 03/14] partial merge #9039: `OverrideStream` changes from `Make nType and nVersion private and sometimes const` commit --- src/streams.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/streams.h b/src/streams.h index 5d0be5ab9c..e549d4f811 100644 --- a/src/streams.h +++ b/src/streams.h @@ -26,10 +26,11 @@ template class OverrideStream { Stream* stream; -public: + const int nType; const int nVersion; +public: OverrideStream(Stream* stream_, int nType_, int nVersion_) : stream(stream_), nType(nType_), nVersion(nVersion_) {} template From cb0eeee0dfa7d8eb65880bf004da1440a06709a2 Mon Sep 17 00:00:00 2001 From: UdjinM6 Date: Wed, 26 May 2021 15:28:44 +0300 Subject: [PATCH 04/14] partial merge #9039: `OverrideStream` changes from `Get rid of nType and nVersion` commit --- src/streams.h | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/src/streams.h b/src/streams.h index e549d4f811..b7c3cbf21c 100644 --- a/src/streams.h +++ b/src/streams.h @@ -37,7 +37,7 @@ public: OverrideStream& operator<<(const T& obj) { // Serialize to this stream - ::Serialize(*this->stream, obj); + ::Serialize(*this, obj); return (*this); } @@ -45,9 +45,22 @@ public: OverrideStream& operator>>(T& obj) { // Unserialize from this stream - ::Unserialize(*this->stream, obj); + ::Unserialize(*this, obj); return (*this); } + + void write(const char* pch, size_t nSize) + { + stream->write(pch, nSize); + } + + void read(char* pch, size_t nSize) + { + stream->read(pch, nSize); + } + + int GetVersion() const { return nVersion; } + int GetType() const { return nType; } }; /* Minimal stream for overwriting and/or appending to an existing byte vector From ac863d395526509f199e5a0dc738062480bae8f8 Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <6098974-kittywhiskers@users.noreply.gitlab.com> Date: Tue, 25 May 2021 19:13:32 +0530 Subject: [PATCH 05/14] partial merge #16670: Add Join helper to join a list of strings --- src/Makefile.am | 2 ++ src/test/util_tests.cpp | 8 ++++++++ src/utilstring.cpp | 5 +++++ src/utilstring.h | 35 +++++++++++++++++++++++++++++++++++ 4 files changed, 50 insertions(+) create mode 100644 src/utilstring.cpp create mode 100644 src/utilstring.h diff --git a/src/Makefile.am b/src/Makefile.am index c85d1b5c2b..c284f9800d 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -262,6 +262,7 @@ BITCOIN_CORE_H = \ utilasmap.h \ utilmemory.h \ utilmoneystr.h \ + utilstring.h \ utiltime.h \ validation.h \ validationinterface.h \ @@ -587,6 +588,7 @@ libdash_util_a_SOURCES = \ utilmoneystr.cpp \ utilstrencodings.cpp \ utiltime.cpp \ + utilstring.cpp \ $(BITCOIN_CORE_H) if GLIBC_BACK_COMPAT diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index f30dc17bd0..84ba262f9b 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include @@ -95,6 +96,13 @@ BOOST_AUTO_TEST_CASE(util_HexStr) ); } +BOOST_AUTO_TEST_CASE(util_Join) +{ + // Normal version + BOOST_CHECK_EQUAL(Join({}, ", "), ""); + BOOST_CHECK_EQUAL(Join({"foo"}, ", "), "foo"); + BOOST_CHECK_EQUAL(Join({"foo", "bar"}, ", "), "foo, bar"); +} BOOST_AUTO_TEST_CASE(util_FormatISO8601DateTime) { diff --git a/src/utilstring.cpp b/src/utilstring.cpp new file mode 100644 index 0000000000..28b95ff75b --- /dev/null +++ b/src/utilstring.cpp @@ -0,0 +1,5 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include diff --git a/src/utilstring.h b/src/utilstring.h new file mode 100644 index 0000000000..6d0edcbcf9 --- /dev/null +++ b/src/utilstring.h @@ -0,0 +1,35 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTILSTRING_H +#define BITCOIN_UTILSTRING_H + +#include +#include +#include + +/** + * Join a list of items + * + * @param list The list to join + * @param separator The separator + * @param unary_op Apply this operator to each item in the list + */ +template +std::string Join(const std::vector& list, const std::string& separator, UnaryOp unary_op) +{ + std::string ret; + for (size_t i = 0; i < list.size(); ++i) { + if (i > 0) ret += separator; + ret += unary_op(list.at(i)); + } + return ret; +} + +inline std::string Join(const std::vector& list, const std::string& separator) +{ + return Join(list, separator, [](const std::string& i) { return i; }); +} + +#endif // BITCOIN_UTILSTRING_H From 50f95ca817089a6f6ee81089f22f55a96405a570 Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <6098974-kittywhiskers@users.noreply.gitlab.com> Date: Wed, 11 Dec 2019 11:00:52 +0000 Subject: [PATCH 06/14] merge #17721: Don't allow Base58 decoding of non-Base58 strings. Add Base58 tests. --- src/base58.cpp | 8 ++++++++ src/test/base58_tests.cpp | 12 ++++++++++++ src/utilstrencodings.cpp | 3 ++- src/utilstring.h | 12 +++++++++++- 4 files changed, 33 insertions(+), 2 deletions(-) diff --git a/src/base58.cpp b/src/base58.cpp index a05917aedf..f2f0f1373e 100644 --- a/src/base58.cpp +++ b/src/base58.cpp @@ -6,6 +6,8 @@ #include #include +#include +#include #include #include @@ -127,6 +129,9 @@ std::string EncodeBase58(const std::vector& vch) bool DecodeBase58(const std::string& str, std::vector& vchRet) { + if (!ValidAsCString(str)) { + return false; + } return DecodeBase58(str.c_str(), vchRet); } @@ -158,6 +163,9 @@ bool DecodeBase58Check(const char* psz, std::vector& vchRet) bool DecodeBase58Check(const std::string& str, std::vector& vchRet) { + if (!ValidAsCString(str)) { + return false; + } return DecodeBase58Check(str.c_str(), vchRet); } diff --git a/src/test/base58_tests.cpp b/src/test/base58_tests.cpp index 2a713a3fff..04cde66219 100644 --- a/src/test/base58_tests.cpp +++ b/src/test/base58_tests.cpp @@ -58,12 +58,24 @@ BOOST_AUTO_TEST_CASE(base58_DecodeBase58) } BOOST_CHECK(!DecodeBase58("invalid", result)); + BOOST_CHECK(!DecodeBase58(std::string("invalid"), result)); + BOOST_CHECK(!DecodeBase58(std::string("\0invalid", 8), result)); + + BOOST_CHECK(DecodeBase58(std::string("good", 4), result)); + BOOST_CHECK(!DecodeBase58(std::string("bad0IOl", 7), result)); + BOOST_CHECK(!DecodeBase58(std::string("goodbad0IOl", 11), result)); + BOOST_CHECK(!DecodeBase58(std::string("good\0bad0IOl", 12), result)); // check that DecodeBase58 skips whitespace, but still fails with unexpected non-whitespace at the end. BOOST_CHECK(!DecodeBase58(" \t\n\v\f\r skip \r\f\v\n\t a", result)); BOOST_CHECK( DecodeBase58(" \t\n\v\f\r skip \r\f\v\n\t ", result)); std::vector expected = ParseHex("971a55"); BOOST_CHECK_EQUAL_COLLECTIONS(result.begin(), result.end(), expected.begin(), expected.end()); + + BOOST_CHECK(DecodeBase58Check(std::string("3vQB7B6MrGQZaxCuFg4oh", 21), result)); + BOOST_CHECK(!DecodeBase58Check(std::string("3vQB7B6MrGQZaxCuFg4oi", 21), result)); + BOOST_CHECK(!DecodeBase58Check(std::string("3vQB7B6MrGQZaxCuFg4oh0IOl", 25), result)); + BOOST_CHECK(!DecodeBase58Check(std::string("3vQB7B6MrGQZaxCuFg4oh\00IOl", 26), result)); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/utilstrencodings.cpp b/src/utilstrencodings.cpp index e2c5e5913a..c8d75130d4 100644 --- a/src/utilstrencodings.cpp +++ b/src/utilstrencodings.cpp @@ -4,6 +4,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include +#include #include @@ -267,7 +268,7 @@ NODISCARD static bool ParsePrechecks(const std::string& str) return false; if (str.size() >= 1 && (isspace(str[0]) || isspace(str[str.size()-1]))) // No padding allowed return false; - if (str.size() != strlen(str.c_str())) // No embedded NUL characters allowed + if (!ValidAsCString(str)) // No embedded NUL characters allowed return false; return true; } diff --git a/src/utilstring.h b/src/utilstring.h index 6d0edcbcf9..0c8a4f4410 100644 --- a/src/utilstring.h +++ b/src/utilstring.h @@ -5,7 +5,9 @@ #ifndef BITCOIN_UTILSTRING_H #define BITCOIN_UTILSTRING_H -#include +#include + +#include #include #include @@ -32,4 +34,12 @@ inline std::string Join(const std::vector& list, const std::string& return Join(list, separator, [](const std::string& i) { return i; }); } +/** + * Check if a string does not contain any embedded NUL (\0) characters + */ +NODISCARD inline bool ValidAsCString(const std::string& str) noexcept +{ + return str.size() == strlen(str.c_str()); +} + #endif // BITCOIN_UTILSTRING_H From 45b894103b251522288206fb918980385a91f09b Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <6098974-kittywhiskers@users.noreply.gitlab.com> Date: Wed, 11 Dec 2019 09:55:00 +0000 Subject: [PATCH 07/14] partial merge #17229: util: Add TrimString(...). Introduce default pattern, NODISCARD --- src/utilstring.h | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/utilstring.h b/src/utilstring.h index 0c8a4f4410..9b8230f492 100644 --- a/src/utilstring.h +++ b/src/utilstring.h @@ -11,6 +11,16 @@ #include #include +NODISCARD inline std::string TrimString(const std::string& str, const std::string& pattern = " \f\n\r\t\v") +{ + std::string::size_type front = str.find_first_not_of(pattern); + if (front == std::string::npos) { + return std::string(); + } + std::string::size_type end = str.find_last_not_of(pattern); + return str.substr(front, end - front + 1); +} + /** * Join a list of items * From 6a36c4e4b1c485d091d6988ef40a298fb085d673 Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Wed, 20 May 2020 07:30:21 -0400 Subject: [PATCH 08/14] partial merge #18317: Serialization improvements step 6 (all except wallet/gui) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit f9ee0f37c28f604bc82dab502ce229c66ef5b3b9 Add comments to CustomUintFormatter (Pieter Wuille) 4eb5643e3538863c9d2ff261f49a9a1b248de243 Convert everything except wallet/qt to new serialization (Pieter Wuille) 2b1f85e8c52c8bc5a17eae4c809eaf61d724af98 Convert blockencodings_tests to new serialization (Pieter Wuille) 73747afbbeb013669faf4c4d2c0903cec4526fb0 Convert merkleblock to new serialization (Pieter Wuille) d06fedd1bc26bf5bf2b203d4445aeaebccca780e Add SER_READ and SER_WRITE for read/write-dependent statements (Russell Yanofsky) 6f9a1e5ad0a270d3b5a715f3e3ea0911193bf244 Extend CustomUintFormatter to support enums (Russell Yanofsky) 769ee5fa0011ae658770586442715452a656559d Merge BigEndian functionality into CustomUintFormatter (Pieter Wuille) Pull request description: The next step of changes from #10785. This: * Adds support for enum serialization to `CustomUintFormatter`, used in `CAddress` for service flags. * Merges `BigEndian` into `CustomUintFormatter`, used in `CNetAddr` for port numbers. * Converts everything (except wallet and gui) to use the new serialization framework. ACKs for top commit: MarcoFalke: re-ACK f9ee0f37c2, only change is new documentation commit for CustomUintFormatter 📂 ryanofsky: Code review ACK f9ee0f37c28f604bc82dab502ce229c66ef5b3b9. Just new commit adding comment since last review jonatack: Code review re-ACK f9ee0f37c28f604bc82dab502ce229c6 only change since last review is an additional commit adding Doxygen documentation for `CustomUintFormatter`. Tree-SHA512: e7a0a36afae592d5a4ff8c81ae04d858ac409388e361f2bc197d9a78abca45134218497ab2dfd6d031e0cce0ca586cf857077b7c6ce17fccf67e2d367c1b6cd4 --- src/bench/prevector.cpp | 4 +- src/blockencodings.h | 3 +- src/bloom.h | 10 +--- src/merkleblock.cpp | 18 ++++++ src/merkleblock.h | 41 +++++--------- src/netaddress.h | 29 +--------- src/policy/feerate.h | 7 +-- src/primitives/block.h | 30 +++------- src/primitives/transaction.h | 25 +-------- src/protocol.h | 42 ++++---------- src/rest.cpp | 9 +-- src/script/script.h | 7 +-- src/serialize.h | 93 +++++++++++++++++-------------- src/test/blockencodings_tests.cpp | 19 +------ src/test/dbwrapper_tests.cpp | 34 +++++------ src/test/serialize_tests.cpp | 23 ++++---- src/txdb.cpp | 14 +---- 17 files changed, 144 insertions(+), 264 deletions(-) diff --git a/src/bench/prevector.cpp b/src/bench/prevector.cpp index 18ca9fb298..fd13505f43 100644 --- a/src/bench/prevector.cpp +++ b/src/bench/prevector.cpp @@ -12,9 +12,7 @@ struct nontrivial_t { int x; nontrivial_t() :x(-1) {} - ADD_SERIALIZE_METHODS - template - inline void SerializationOp(Stream& s, Operation ser_action) {READWRITE(x);} + SERIALIZE_METHODS(nontrivial_t, obj) { READWRITE(obj.x); } }; static_assert(!IS_TRIVIALLY_CONSTRUCTIBLE::value, "expected nontrivial_t to not be trivially constructible"); diff --git a/src/blockencodings.h b/src/blockencodings.h index 84d722adc1..954055488a 100644 --- a/src/blockencodings.h +++ b/src/blockencodings.h @@ -93,12 +93,13 @@ private: friend class PartiallyDownloadedBlock; - static const int SHORTTXIDS_LENGTH = 6; protected: std::vector shorttxids; std::vector prefilledtxn; public: + static constexpr int SHORTTXIDS_LENGTH = 6; + CBlockHeader header; // Dummy for deserialization diff --git a/src/bloom.h b/src/bloom.h index 33c7d7227a..ecb3c75599 100644 --- a/src/bloom.h +++ b/src/bloom.h @@ -76,15 +76,7 @@ public: CBloomFilter(const unsigned int nElements, const double nFPRate, const unsigned int nTweak, unsigned char nFlagsIn); CBloomFilter() : isFull(true), isEmpty(false), nHashFuncs(0), nTweak(0), nFlags(0) {} - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(vData); - READWRITE(nHashFuncs); - READWRITE(nTweak); - READWRITE(nFlags); - } + SERIALIZE_METHODS(CBloomFilter, obj) { READWRITE(obj.vData, obj.nHashFuncs, obj.nTweak, obj.nFlags); } void insert(const std::vector& vKey); void insert(const COutPoint& outpoint); diff --git a/src/merkleblock.cpp b/src/merkleblock.cpp index 46a8228f94..0731ceba8a 100644 --- a/src/merkleblock.cpp +++ b/src/merkleblock.cpp @@ -10,6 +10,24 @@ #include +std::vector BitsToBytes(const std::vector& bits) +{ + std::vector ret((bits.size() + 7) / 8); + for (unsigned int p = 0; p < bits.size(); p++) { + ret[p / 8] |= bits[p] << (p % 8); + } + return ret; +} + +std::vector BytesToBits(const std::vector& bytes) +{ + std::vector ret(bytes.size() * 8); + for (unsigned int p = 0; p < ret.size(); p++) { + ret[p] = (bytes[p / 8] & (1 << (p % 8))) != 0; + } + return ret; +} + CMerkleBlock::CMerkleBlock(const CBlock& block, CBloomFilter* filter, const std::set* txids) { header = block.GetBlockHeader(); diff --git a/src/merkleblock.h b/src/merkleblock.h index 3207022fb2..275e0373cd 100644 --- a/src/merkleblock.h +++ b/src/merkleblock.h @@ -13,6 +13,10 @@ #include +// Helper functions for serialization. +std::vector BitsToBytes(const std::vector& bits); +std::vector BytesToBits(const std::vector& bytes); + /** Data structure that represents a partial merkle tree. * * It represents a subset of the txid's of a known block, in a way that @@ -81,27 +85,14 @@ protected: public: - /** serialization implementation */ - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(nTransactions); - READWRITE(vHash); - std::vector vBytes; - if (ser_action.ForRead()) { - READWRITE(vBytes); - CPartialMerkleTree &us = *(const_cast(this)); - us.vBits.resize(vBytes.size() * 8); - for (unsigned int p = 0; p < us.vBits.size(); p++) - us.vBits[p] = (vBytes[p / 8] & (1 << (p % 8))) != 0; - us.fBad = false; - } else { - vBytes.resize((vBits.size()+7)/8); - for (unsigned int p = 0; p < vBits.size(); p++) - vBytes[p / 8] |= vBits[p] << (p % 8); - READWRITE(vBytes); - } + SERIALIZE_METHODS(CPartialMerkleTree, obj) + { + READWRITE(obj.nTransactions, obj.vHash); + std::vector bytes; + SER_WRITE(obj, bytes = BitsToBytes(obj.vBits)); + READWRITE(bytes); + SER_READ(obj, obj.vBits = BytesToBits(bytes)); + SER_READ(obj, obj.fBad = false); } /** Construct a partial merkle tree from a list of transaction ids, and a mask that selects a subset of them */ @@ -157,13 +148,7 @@ public: CMerkleBlock() {} - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(header); - READWRITE(txn); - } + SERIALIZE_METHODS(CMerkleBlock, obj) { READWRITE(obj.header, obj.txn); } private: // Combined constructor to consolidate code diff --git a/src/netaddress.h b/src/netaddress.h index c1af100d12..8ba6a34daf 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -316,14 +316,7 @@ class CSubNet friend bool operator!=(const CSubNet& a, const CSubNet& b) { return !(a == b); } friend bool operator<(const CSubNet& a, const CSubNet& b); - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(network); - READWRITE(netmask); - READWRITE(valid); - } + SERIALIZE_METHODS(CSubNet, obj) { READWRITE(obj.network, obj.netmask, obj.valid); } }; /** A combination of a network address (CNetAddr) and a (TCP) port */ @@ -352,25 +345,7 @@ class CService : public CNetAddr CService(const struct in6_addr& ipv6Addr, unsigned short port); explicit CService(const struct sockaddr_in6& addr); - /** - * Serialize to a stream. - */ - template - void Serialize(Stream& s) const - { - SerializeV1Stream(s); - s << WrapBigEndian(port); - } - - /** - * Unserialize from a stream. - */ - template - void Unserialize(Stream& s) - { - UnserializeV1Stream(s); - s >> WrapBigEndian(port); - } + SERIALIZE_METHODS(CService, obj) { READWRITE(obj.ip, Using>(obj.port)); } }; #endif // BITCOIN_NETADDRESS_H diff --git a/src/policy/feerate.h b/src/policy/feerate.h index 35b1b23786..fd5b581710 100644 --- a/src/policy/feerate.h +++ b/src/policy/feerate.h @@ -48,12 +48,7 @@ public: CFeeRate& operator+=(const CFeeRate& a) { nSatoshisPerK += a.nSatoshisPerK; return *this; } std::string ToString() const; - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(nSatoshisPerK); - } + SERIALIZE_METHODS(CFeeRate, obj) { READWRITE(obj.nSatoshisPerK); } }; #endif // BITCOIN_POLICY_FEERATE_H diff --git a/src/primitives/block.h b/src/primitives/block.h index 9551b544c1..324806b916 100644 --- a/src/primitives/block.h +++ b/src/primitives/block.h @@ -33,17 +33,7 @@ public: SetNull(); } - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(this->nVersion); - READWRITE(hashPrevBlock); - READWRITE(hashMerkleRoot); - READWRITE(nTime); - READWRITE(nBits); - READWRITE(nNonce); - } + SERIALIZE_METHODS(CBlockHeader, obj) { READWRITE(obj.nVersion, obj.hashPrevBlock, obj.hashMerkleRoot, obj.nTime, obj.nBits, obj.nNonce); } void SetNull() { @@ -89,12 +79,10 @@ public: *(static_cast(this)) = header; } - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITEAS(CBlockHeader, *this); - READWRITE(vtx); + SERIALIZE_METHODS(CBlock, obj) + { + READWRITEAS(CBlockHeader, obj); + READWRITE(obj.vtx); } void SetNull() @@ -132,14 +120,12 @@ struct CBlockLocator explicit CBlockLocator(const std::vector& vHaveIn) : vHave(vHaveIn) {} - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { + SERIALIZE_METHODS(CBlockLocator, obj) + { int nVersion = s.GetVersion(); if (!(s.GetType() & SER_GETHASH)) READWRITE(nVersion); - READWRITE(vHave); + READWRITE(obj.vHave); } void SetNull() diff --git a/src/primitives/transaction.h b/src/primitives/transaction.h index a5ab6f0338..e1a9e0ff22 100644 --- a/src/primitives/transaction.h +++ b/src/primitives/transaction.h @@ -32,13 +32,7 @@ public: COutPoint(): n((uint32_t) -1) { } COutPoint(const uint256& hashIn, uint32_t nIn): hash(hashIn), n(nIn) { } - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(hash); - READWRITE(n); - } + SERIALIZE_METHODS(COutPoint, obj) { READWRITE(obj.hash, obj.n); } void SetNull() { hash.SetNull(); n = (uint32_t) -1; } bool IsNull() const { return (hash.IsNull() && n == (uint32_t) -1); } @@ -109,14 +103,7 @@ public: explicit CTxIn(COutPoint prevoutIn, CScript scriptSigIn=CScript(), uint32_t nSequenceIn=SEQUENCE_FINAL); CTxIn(uint256 hashPrevTx, uint32_t nOut, CScript scriptSigIn=CScript(), uint32_t nSequenceIn=SEQUENCE_FINAL); - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(prevout); - READWRITE(scriptSig); - READWRITE(nSequence); - } + SERIALIZE_METHODS(CTxIn, obj) { READWRITE(obj.prevout, obj.scriptSig, obj.nSequence); } friend bool operator==(const CTxIn& a, const CTxIn& b) { @@ -154,13 +141,7 @@ public: CTxOut(const CAmount& nValueIn, CScript scriptPubKeyIn); - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(nValue); - READWRITE(scriptPubKey); - } + SERIALIZE_METHODS(CTxOut, obj) { READWRITE(obj.nValue, obj.scriptPubKey); } void SetNull() { diff --git a/src/protocol.h b/src/protocol.h index 9ab34c6488..759c13a739 100644 --- a/src/protocol.h +++ b/src/protocol.h @@ -43,16 +43,7 @@ public: std::string GetCommand() const; bool IsValid(const MessageStartChars& messageStart) const; - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) - { - READWRITE(pchMessageStart); - READWRITE(pchCommand); - READWRITE(nMessageSize); - READWRITE(pchChecksum); - } + SERIALIZE_METHODS(CMessageHeader, obj) { READWRITE(obj.pchMessageStart, obj.pchCommand, obj.nMessageSize, obj.pchChecksum); } char pchMessageStart[MESSAGE_START_SIZE]; char pchCommand[COMMAND_SIZE]; @@ -365,23 +356,19 @@ public: void Init(); - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CAddress, obj) { - if (ser_action.ForRead()) - Init(); + SER_READ(obj, obj.Init()); int nVersion = s.GetVersion(); - if (s.GetType() & SER_DISK) + if (s.GetType() & SER_DISK) { READWRITE(nVersion); + } if ((s.GetType() & SER_DISK) || - (nVersion >= CADDR_TIME_VERSION && !(s.GetType() & SER_GETHASH))) - READWRITE(nTime); - uint64_t nServicesInt = nServices; - READWRITE(nServicesInt); - nServices = static_cast(nServicesInt); - READWRITEAS(CService, *this); + (nVersion >= CADDR_TIME_VERSION && !(s.GetType() & SER_GETHASH))) { + READWRITE(obj.nTime); + } + READWRITE(Using>(obj.nServices)); + READWRITEAS(CService, obj); } // TODO: make private (improves encapsulation) @@ -434,14 +421,7 @@ public: CInv(); CInv(int typeIn, const uint256& hashIn); - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) - { - READWRITE(type); - READWRITE(hash); - } + SERIALIZE_METHODS(CInv, obj) { READWRITE(obj.type, obj.hash); } friend bool operator<(const CInv& a, const CInv& b); diff --git a/src/rest.cpp b/src/rest.cpp index 008dbf9a7b..313528b251 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -46,18 +46,13 @@ struct CCoin { uint32_t nHeight; CTxOut out; - ADD_SERIALIZE_METHODS; - CCoin() : nHeight(0) {} explicit CCoin(Coin&& in) : nHeight(in.nHeight), out(std::move(in.out)) {} - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CCoin, obj) { uint32_t nTxVerDummy = 0; - READWRITE(nTxVerDummy); - READWRITE(nHeight); - READWRITE(out); + READWRITE(nTxVerDummy, obj.nHeight, obj.out); } }; diff --git a/src/script/script.h b/src/script/script.h index c10d1251f7..4a2a506f56 100644 --- a/src/script/script.h +++ b/src/script/script.h @@ -414,12 +414,7 @@ public: CScript(std::vector::const_iterator pbegin, std::vector::const_iterator pend) : CScriptBase(pbegin, pend) { } CScript(const unsigned char* pbegin, const unsigned char* pend) : CScriptBase(pbegin, pend) { } - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITEAS(CScriptBase, *this); - } + SERIALIZE_METHODS(CScript, obj) { READWRITEAS(CScriptBase, obj); } CScript& operator+=(const CScript& b) { diff --git a/src/serialize.h b/src/serialize.h index 0bc1e51d8a..6cdbbd067e 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -189,6 +189,8 @@ template const X& ReadWriteAsHelper(const X& x) { return x; } #define READWRITE(...) (::SerReadWriteMany(s, ser_action, __VA_ARGS__)) #define READWRITEAS(type, obj) (::SerReadWriteMany(s, ser_action, ReadWriteAsHelper(obj))) +#define SER_READ(obj, code) ::SerRead(s, ser_action, obj, [&](Stream& s, typename std::remove_const::type& obj) { code; }) +#define SER_WRITE(obj, code) ::SerWrite(s, ser_action, obj, [&](Stream& s, const Type& obj) { code; }) /** * Implement three methods for serializable objects. These are actually wrappers over @@ -678,7 +680,16 @@ struct VarIntFormatter } }; -template +/** Serialization wrapper class for custom integers and enums. + * + * It permits specifying the serialized size (1 to 8 bytes) and endianness. + * + * Use the big endian mode for values that are stored in memory in native + * byte order, but serialized in big endian notation. This is only intended + * to implement serializers that are compatible with existing formats, and + * its use is not recommended for new data structures. + */ +template struct CustomUintFormatter { static_assert(Bytes > 0 && Bytes <= 8, "CustomUintFormatter Bytes out of range"); @@ -687,52 +698,31 @@ struct CustomUintFormatter template void Ser(Stream& s, I v) { if (v < 0 || v > MAX) throw std::ios_base::failure("CustomUintFormatter value out of range"); - uint64_t raw = htole64(v); - s.write((const char*)&raw, Bytes); + if (BigEndian) { + uint64_t raw = htobe64(v); + s.write(((const char*)&raw) + 8 - Bytes, Bytes); + } else { + uint64_t raw = htole64(v); + s.write((const char*)&raw, Bytes); + } } template void Unser(Stream& s, I& v) { - static_assert(std::numeric_limits::max() >= MAX && std::numeric_limits::min() <= 0, "CustomUintFormatter type too small"); + using U = typename std::conditional::value, std::underlying_type, std::common_type>::type::type; + static_assert(std::numeric_limits::max() >= MAX && std::numeric_limits::min() <= 0, "Assigned type too small"); uint64_t raw = 0; - s.read((char*)&raw, Bytes); - v = le64toh(raw); + if (BigEndian) { + s.read(((char*)&raw) + 8 - Bytes, Bytes); + v = static_cast(be64toh(raw)); + } else { + s.read((char*)&raw, Bytes); + v = static_cast(le64toh(raw)); + } } }; -/** Serialization wrapper class for big-endian integers. - * - * Use this wrapper around integer types that are stored in memory in native - * byte order, but serialized in big endian notation. This is only intended - * to implement serializers that are compatible with existing formats, and - * its use is not recommended for new data structures. - * - * Only 16-bit types are supported for now. - */ -template -class BigEndian -{ -protected: - I& m_val; -public: - explicit BigEndian(I& val) : m_val(val) - { - static_assert(std::is_unsigned::value, "BigEndian type must be unsigned integer"); - static_assert(sizeof(I) == 2 && std::numeric_limits::min() == 0 && std::numeric_limits::max() == std::numeric_limits::max(), "Unsupported BigEndian size"); - } - - template - void Serialize(Stream& s) const - { - ser_writedata16be(s, m_val); - } - - template - void Unserialize(Stream& s) - { - m_val = ser_readdata16be(s); - } -}; +template using BigEndianFormatter = CustomUintFormatter; /** Formatter for integers in CompactSize format. */ struct CompactSizeFormatter @@ -786,9 +776,6 @@ public: } }; -template -BigEndian WrapBigEndian(I& n) { return BigEndian(n); } - /** Formatter to serialize/deserialize vector elements using another formatter * * Example: @@ -1444,6 +1431,28 @@ inline void SerReadWriteMany(Stream& s, CSerActionUnserialize ser_action, Args&& ::UnserializeMany(s, args...); } +template +inline void SerRead(Stream& s, CSerActionSerialize ser_action, Type&&, Fn&&) +{ +} + +template +inline void SerRead(Stream& s, CSerActionUnserialize ser_action, Type&& obj, Fn&& fn) +{ + fn(s, std::forward(obj)); +} + +template +inline void SerWrite(Stream& s, CSerActionSerialize ser_action, Type&& obj, Fn&& fn) +{ + fn(s, std::forward(obj)); +} + +template +inline void SerWrite(Stream& s, CSerActionUnserialize ser_action, Type&&, Fn&&) +{ +} + template inline void WriteVarInt(CSizeComputer &s, I n) { diff --git a/src/test/blockencodings_tests.cpp b/src/test/blockencodings_tests.cpp index f7c7994556..59b027cc3a 100644 --- a/src/test/blockencodings_tests.cpp +++ b/src/test/blockencodings_tests.cpp @@ -136,24 +136,7 @@ public: return base.GetShortID(txhash); } - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(header); - READWRITE(nonce); - size_t shorttxids_size = shorttxids.size(); - READWRITE(VARINT(shorttxids_size)); - shorttxids.resize(shorttxids_size); - for (size_t i = 0; i < shorttxids.size(); i++) { - uint32_t lsb = shorttxids[i] & 0xffffffff; - uint16_t msb = (shorttxids[i] >> 32) & 0xffff; - READWRITE(lsb); - READWRITE(msb); - shorttxids[i] = (uint64_t(msb) << 32) | uint64_t(lsb); - } - READWRITE(prefilledtxn); - } + SERIALIZE_METHODS(TestHeaderAndShortIDs, obj) { READWRITE(obj.header, obj.nonce, Using>>(obj.shorttxids), obj.prefilledtxn); } }; BOOST_AUTO_TEST_CASE(NonCoinbasePreforwardRTTest) diff --git a/src/test/dbwrapper_tests.cpp b/src/test/dbwrapper_tests.cpp index 82e9ed6c5f..14979c081f 100644 --- a/src/test/dbwrapper_tests.cpp +++ b/src/test/dbwrapper_tests.cpp @@ -251,24 +251,26 @@ struct StringContentsSerializer { } StringContentsSerializer& operator+=(const StringContentsSerializer& s) { return *this += s.str; } - ADD_SERIALIZE_METHODS; + template + void Serialize(Stream& s) const + { + for (size_t i = 0; i < str.size(); i++) { + s << str[i]; + } + } - template - inline void SerializationOp(Stream& s, Operation ser_action) { - if (ser_action.ForRead()) { - str.clear(); - char c = 0; - while (true) { - try { - READWRITE(c); - str.push_back(c); - } catch (const std::ios_base::failure& e) { - break; - } + template + void Unserialize(Stream& s) + { + str.clear(); + char c = 0; + while (true) { + try { + s >> c; + str.push_back(c); + } catch (const std::ios_base::failure&) { + break; } - } else { - for (size_t i = 0; i < str.size(); i++) - READWRITE(str[i]); } } }; diff --git a/src/test/serialize_tests.cpp b/src/test/serialize_tests.cpp index c1fbaec319..346255b87e 100644 --- a/src/test/serialize_tests.cpp +++ b/src/test/serialize_tests.cpp @@ -28,15 +28,13 @@ public: memcpy(charstrval, charstrvalin, sizeof(charstrval)); } - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(intval); - READWRITE(boolval); - READWRITE(stringval); - READWRITE(charstrval); - READWRITE(txval); + SERIALIZE_METHODS(CSerializeMethodsTestSingle, obj) + { + READWRITE(obj.intval); + READWRITE(obj.boolval); + READWRITE(obj.stringval); + READWRITE(obj.charstrval); + READWRITE(obj.txval); } bool operator==(const CSerializeMethodsTestSingle& rhs) @@ -53,11 +51,10 @@ class CSerializeMethodsTestMany : public CSerializeMethodsTestSingle { public: using CSerializeMethodsTestSingle::CSerializeMethodsTestSingle; - ADD_SERIALIZE_METHODS; - template - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(intval, boolval, stringval, charstrval, txval); + SERIALIZE_METHODS(CSerializeMethodsTestMany, obj) + { + READWRITE(obj.intval, obj.boolval, obj.stringval, obj.charstrval, obj.txval); } }; diff --git a/src/txdb.cpp b/src/txdb.cpp index 1beb79e834..0ac3eb2332 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -41,19 +41,7 @@ struct CoinEntry { char key; explicit CoinEntry(const COutPoint* ptr) : outpoint(const_cast(ptr)), key(DB_COIN) {} - template - void Serialize(Stream &s) const { - s << key; - s << outpoint->hash; - s << VARINT(outpoint->n); - } - - template - void Unserialize(Stream& s) { - s >> key; - s >> outpoint->hash; - s >> VARINT(outpoint->n); - } + SERIALIZE_METHODS(CoinEntry, obj) { READWRITE(obj.key, obj.outpoint->hash, VARINT(obj.outpoint->n)); } }; } From 33f09d0b35575e432842cba2c18d288b98471d26 Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <6098974-kittywhiskers@users.noreply.gitlab.com> Date: Sat, 9 May 2020 14:45:47 +0300 Subject: [PATCH 09/14] partial merge #18922: Enhance Join() --- src/utilstring.h | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/utilstring.h b/src/utilstring.h index 9b8230f492..8d9fff6c9d 100644 --- a/src/utilstring.h +++ b/src/utilstring.h @@ -28,10 +28,11 @@ NODISCARD inline std::string TrimString(const std::string& str, const std::strin * @param separator The separator * @param unary_op Apply this operator to each item in the list */ -template -std::string Join(const std::vector& list, const std::string& separator, UnaryOp unary_op) +template +auto Join(const std::vector& list, const BaseType& separator, UnaryOp unary_op) + -> decltype(unary_op(list.at(0))) { - std::string ret; + decltype(unary_op(list.at(0))) ret; for (size_t i = 0; i < list.size(); ++i) { if (i > 0) ret += separator; ret += unary_op(list.at(i)); @@ -39,9 +40,16 @@ std::string Join(const std::vector& list, const std::string& separator, Unary return ret; } +template +T Join(const std::vector& list, const T& separator) +{ + return Join(list, separator, [](const T& i) { return i; }); +} + +// Explicit overload needed for c_str arguments, which would otherwise cause a substitution failure in the template above. inline std::string Join(const std::vector& list, const std::string& separator) { - return Join(list, separator, [](const std::string& i) { return i; }); + return Join(list, separator); } /** From 6e5210adeb6631ed85efe34998bd4bcc6264c90d Mon Sep 17 00:00:00 2001 From: UdjinM6 Date: Wed, 26 May 2021 15:58:42 +0300 Subject: [PATCH 10/14] partial merge #19360 / revert 78328c1898e52565d7da2b2815e32c6a3cf98b76 --- src/netaddress.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/netaddress.h b/src/netaddress.h index 8ba6a34daf..78f9591338 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -345,7 +345,11 @@ class CService : public CNetAddr CService(const struct in6_addr& ipv6Addr, unsigned short port); explicit CService(const struct sockaddr_in6& addr); - SERIALIZE_METHODS(CService, obj) { READWRITE(obj.ip, Using>(obj.port)); } + SERIALIZE_METHODS(CService, obj) + { + READWRITEAS(CNetAddr, obj); + READWRITE(Using>(obj.port)); + } }; #endif // BITCOIN_NETADDRESS_H From 9029448233a92e2954056260b031f52b2013ceca Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Thu, 27 May 2021 17:53:41 +0300 Subject: [PATCH 11/14] partial merge #19032: Serialization improvements: final step MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 71f016c6eb42e1ac2c905e04ba4d20c2009e533f Remove old serialization primitives (Pieter Wuille) 92beff15d3ae2646c00bd78146d7592a7097ce9c Convert LimitedString to formatter (Pieter Wuille) ef17c03e074b6c3f185afa4eff572ba687c2a171 Convert wallet to new serialization (Pieter Wuille) 65c589e45e8b8914698a0fd25cd5aafdda30869c Convert Qt to new serialization (Pieter Wuille) Pull request description: This is the final step 🥳 of the serialization improvements extracted from #10785. It converts the LimitedString wrapper to a new-style formatter, and updates the wallet and Qt code to use the new serialization framework. Finally all remaining old primitives are removed. ACKs for top commit: jonatack: ACK 71f016c6eb42e1ac2 reviewed diff, builds/tests/re-fuzzed. laanwj: Code review ACK 71f016c6eb42e1ac2c905e04ba4d20c2009e533f Tree-SHA512: d952194bc73259f6510bd4ab1348a1febbbf9862af30f905991812fb0e1f23f15948cdb3fc662be54d648e8f6d95b11060055d2e7a8c2cb5bf008224870b1ea1 --- src/qt/recentrequeststablemodel.h | 18 +++------- src/qt/walletmodel.h | 42 ++++++++++------------ src/serialize.h | 60 +++++-------------------------- src/wallet/crypter.h | 12 ++----- src/wallet/wallet.h | 41 +++++++++++---------- src/wallet/walletdb.h | 9 ++--- 6 files changed, 59 insertions(+), 123 deletions(-) diff --git a/src/qt/recentrequeststablemodel.h b/src/qt/recentrequeststablemodel.h index 7a520d8c93..037f510825 100644 --- a/src/qt/recentrequeststablemodel.h +++ b/src/qt/recentrequeststablemodel.h @@ -22,19 +22,11 @@ public: QDateTime date; SendCoinsRecipient recipient; - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - unsigned int nDate = date.toTime_t(); - - READWRITE(this->nVersion); - READWRITE(id); - READWRITE(nDate); - READWRITE(recipient); - - if (ser_action.ForRead()) - date = QDateTime::fromTime_t(nDate); + SERIALIZE_METHODS(RecentRequestEntry, obj) { + unsigned int date_timet; + SER_WRITE(obj, date_timet = obj.date.toTime_t()); + READWRITE(obj.nVersion, obj.id, date_timet, obj.recipient); + SER_READ(obj, obj.date = QDateTime::fromTime_t(date_timet)); } }; diff --git a/src/qt/walletmodel.h b/src/qt/walletmodel.h index ab40489b82..6aeef530df 100644 --- a/src/qt/walletmodel.h +++ b/src/qt/walletmodel.h @@ -70,34 +70,28 @@ public: static const int CURRENT_VERSION = 1; int nVersion; - ADD_SERIALIZE_METHODS; + SERIALIZE_METHODS(SendCoinsRecipient, obj) + { + std::string address_str, label_str, message_str, auth_merchant_str, sPaymentRequest; + PaymentRequestPlus paymentRequest; - template - inline void SerializationOp(Stream& s, Operation ser_action) { - std::string sAddress = address.toStdString(); - std::string sLabel = label.toStdString(); - std::string sMessage = message.toStdString(); - std::string sPaymentRequest; - if (!ser_action.ForRead() && paymentRequest.IsInitialized()) + SER_WRITE(obj, address_str = obj.address.toStdString()); + SER_WRITE(obj, label_str = obj.label.toStdString()); + SER_WRITE(obj, message_str = obj.message.toStdString()); + SER_WRITE(obj, auth_merchant_str = obj.authenticatedMerchant.toStdString()); + SER_WRITE(obj, paymentRequest = obj.paymentRequest); + if (paymentRequest.IsInitialized()) { paymentRequest.SerializeToString(&sPaymentRequest); - std::string sAuthenticatedMerchant = authenticatedMerchant.toStdString(); + } - READWRITE(this->nVersion); - READWRITE(sAddress); - READWRITE(sLabel); - READWRITE(amount); - READWRITE(sMessage); - READWRITE(sPaymentRequest); - READWRITE(sAuthenticatedMerchant); + READWRITE(obj.nVersion, address_str, label_str, obj.amount, message_str, sPaymentRequest, auth_merchant_str); - if (ser_action.ForRead()) - { - address = QString::fromStdString(sAddress); - label = QString::fromStdString(sLabel); - message = QString::fromStdString(sMessage); - if (!sPaymentRequest.empty()) - paymentRequest.parse(QByteArray::fromRawData(sPaymentRequest.data(), sPaymentRequest.size())); - authenticatedMerchant = QString::fromStdString(sAuthenticatedMerchant); + SER_READ(obj, obj.address = QString::fromStdString(address_str)); + SER_READ(obj, obj.label = QString::fromStdString(label_str)); + SER_READ(obj, obj.message = QString::fromStdString(message_str)); + SER_READ(obj, obj.authenticatedMerchant = QString::fromStdString(auth_merchant_str)); + if (!sPaymentRequest.empty()) { + SER_READ(obj, obj.paymentRequest.parse(QByteArray::fromRawData(sPaymentRequest.data(), sPaymentRequest.size()))); } } }; diff --git a/src/serialize.h b/src/serialize.h index 6cdbbd067e..3dad1ff64f 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -46,26 +46,6 @@ static const unsigned int MAX_VECTOR_ALLOCATE = 5000000; struct deserialize_type {}; constexpr deserialize_type deserialize {}; -/** - * Used to bypass the rule against non-const reference to temporary - * where it makes sense with wrappers. - */ -template -inline T& REF(const T& val) -{ - return const_cast(val); -} - -/** - * Used to acquire a non-const pointer "this" to generate bodies - * of const serialization operations from a template - */ -template -inline T* NCONST_PTR(const T* val) -{ - return const_cast(val); -} - //! Safely convert odd char pointer types to standard ones. inline char* CharCast(char* c) { return c; } inline char* CharCast(unsigned char* c) { return (char*)c; } @@ -192,22 +172,6 @@ template const X& ReadWriteAsHelper(const X& x) { return x; } #define SER_READ(obj, code) ::SerRead(s, ser_action, obj, [&](Stream& s, typename std::remove_const::type& obj) { code; }) #define SER_WRITE(obj, code) ::SerWrite(s, ser_action, obj, [&](Stream& s, const Type& obj) { code; }) -/** - * Implement three methods for serializable objects. These are actually wrappers over - * "SerializationOp" template, which implements the body of each class' serialization - * code. Adding "ADD_SERIALIZE_METHODS" in the body of the class causes these wrappers to be - * added as members. - */ -#define ADD_SERIALIZE_METHODS \ - template \ - void Serialize(Stream& s) const { \ - NCONST_PTR(this)->SerializationOp(s, CSerActionSerialize()); \ - } \ - template \ - void Unserialize(Stream& s) { \ - SerializationOp(s, CSerActionUnserialize()); \ - } - /** * Implement the Ser and Unser methods needed for implementing a formatter (see Using below). * @@ -507,7 +471,7 @@ static inline Wrapper Using(T&& t) { return Wrapper>(obj) #define COMPACTSIZE(obj) Using(obj) -#define LIMITED_STRING(obj,n) LimitedString< n >(REF(obj)) +#define LIMITED_STRING(obj,n) Using>(obj) class CFixedBitSet { @@ -748,31 +712,23 @@ struct CompactSizeFormatter }; template -class LimitedString +struct LimitedStringFormatter { -protected: - std::string& string; -public: - explicit LimitedString(std::string& _string) : string(_string) {} - template - void Unserialize(Stream& s) + void Unser(Stream& s, std::string& v) { size_t size = ReadCompactSize(s); if (size > Limit) { throw std::ios_base::failure("String length limit exceeded"); } - string.resize(size); - if (size != 0) - s.read((char*)string.data(), size); + v.resize(size); + if (size != 0) s.read((char*)v.data(), size); } template - void Serialize(Stream& s) const + void Ser(Stream& s, const std::string& v) { - WriteCompactSize(s, string.size()); - if (!string.empty()) - s.write((char*)string.data(), string.size()); + s << v; } }; @@ -1330,7 +1286,7 @@ void Unserialize(Stream& is, std::shared_ptr& p) /** - * Support for ADD_SERIALIZE_METHODS and READWRITE macro + * Support for SERIALIZE_METHODS and READWRITE macro. */ struct CSerActionSerialize { diff --git a/src/wallet/crypter.h b/src/wallet/crypter.h index 8e61b8db56..9f8fb3b4c1 100644 --- a/src/wallet/crypter.h +++ b/src/wallet/crypter.h @@ -44,15 +44,9 @@ public: //! such as the various parameters to scrypt std::vector vchOtherDerivationParameters; - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(vchCryptedKey); - READWRITE(vchSalt); - READWRITE(nDerivationMethod); - READWRITE(nDeriveIterations); - READWRITE(vchOtherDerivationParameters); + SERIALIZE_METHODS(CMasterKey, obj) + { + READWRITE(obj.vchCryptedKey, obj.vchSalt, obj.nDerivationMethod, obj.nDeriveIterations, obj.vchOtherDerivationParameters); } CMasterKey() diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index b71b27327f..52a330b45c 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -116,27 +116,30 @@ public: CKeyPool(); CKeyPool(const CPubKey& vchPubKeyIn, bool fInternalIn); - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { + template + void Serialize(Stream& s) const + { int nVersion = s.GetVersion(); - if (!(s.GetType() & SER_GETHASH)) - READWRITE(nVersion); - READWRITE(nTime); - READWRITE(vchPubKey); - if (ser_action.ForRead()) { - try { - READWRITE(fInternal); - } - catch (std::ios_base::failure&) { - /* flag as external address if we can't read the internal boolean - (this will be the case for any wallet before the HD chain split version) */ - fInternal = false; - } + if (!(s.GetType() & SER_GETHASH)) { + s << nVersion; } - else { - READWRITE(fInternal); + s << nTime << vchPubKey << fInternal; + } + + template + void Unserialize(Stream& s) + { + int nVersion = s.GetVersion(); + if (!(s.GetType() & SER_GETHASH)) { + s >> nVersion; + } + s >> nTime >> vchPubKey; + try { + s >> fInternal; + } catch (std::ios_base::failure&) { + /* flag as external address if we can't read the internal boolean + (this will be the case for any wallet before the HD chain split version) */ + fInternal = false; } } }; diff --git a/src/wallet/walletdb.h b/src/wallet/walletdb.h index 79f38c88cc..777e4fd5c7 100644 --- a/src/wallet/walletdb.h +++ b/src/wallet/walletdb.h @@ -74,12 +74,9 @@ public: nCreateTime = nCreateTime_; } - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(this->nVersion); - READWRITE(nCreateTime); + SERIALIZE_METHODS(CKeyMetadata, obj) + { + READWRITE(obj.nVersion, obj.nCreateTime); } void SetNull() From 64d4a48a09387569dc0f3a0c759b4cb5e5d2bbfa Mon Sep 17 00:00:00 2001 From: UdjinM6 Date: Thu, 27 May 2021 17:59:09 +0300 Subject: [PATCH 12/14] Some missing non-Dash-specifc serializations --- src/chain.h | 9 +++------ src/streams.h | 2 +- src/txdb.h | 10 ++++------ src/undo.h | 4 ++-- src/wallet/wallet.h | 30 +++++++++--------------------- 5 files changed, 19 insertions(+), 36 deletions(-) diff --git a/src/chain.h b/src/chain.h index bb2763a1b1..5962f335f4 100644 --- a/src/chain.h +++ b/src/chain.h @@ -85,12 +85,9 @@ struct CDiskBlockPos int nFile; unsigned int nPos; - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(VARINT(nFile, VarIntMode::NONNEGATIVE_SIGNED)); - READWRITE(VARINT(nPos)); + SERIALIZE_METHODS(CDiskBlockPos, obj) + { + READWRITE(VARINT(obj.nFile, VarIntMode::NONNEGATIVE_SIGNED), VARINT(obj.nPos)); } CDiskBlockPos() { diff --git a/src/streams.h b/src/streams.h index b7c3cbf21c..2505f07390 100644 --- a/src/streams.h +++ b/src/streams.h @@ -42,7 +42,7 @@ public: } template - OverrideStream& operator>>(T& obj) + OverrideStream& operator>>(T&& obj) { // Unserialize from this stream ::Unserialize(*this, obj); diff --git a/src/txdb.h b/src/txdb.h index 34f293e7b7..59fe68f242 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -46,12 +46,10 @@ struct CDiskTxPos : public CDiskBlockPos { unsigned int nTxOffset; // after header - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITEAS(CDiskBlockPos, *this); - READWRITE(VARINT(nTxOffset)); + SERIALIZE_METHODS(CDiskTxPos, obj) + { + READWRITEAS(CDiskBlockPos, obj); + READWRITE(VARINT(obj.nTxOffset)); } CDiskTxPos(const CDiskBlockPos &blockIn, unsigned int nTxOffsetIn) : CDiskBlockPos(blockIn.nFile, blockIn.nPos), nTxOffset(nTxOffsetIn) { diff --git a/src/undo.h b/src/undo.h index 3fd3ebc1bc..636f748867 100644 --- a/src/undo.h +++ b/src/undo.h @@ -28,7 +28,7 @@ struct TxInUndoFormatter // Required to maintain compatibility with older undo format. ::Serialize(s, (unsigned char)0); } - ::Serialize(s, Using(REF(txout.out))); + ::Serialize(s, Using(txout.out)); } template @@ -44,7 +44,7 @@ struct TxInUndoFormatter unsigned int nVersionDummy; ::Unserialize(s, VARINT(nVersionDummy)); } - ::Unserialize(s, Using(REF(txout.out))); + ::Unserialize(s, Using(txout.out)); } }; diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 52a330b45c..8b9b2f1857 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -236,15 +236,10 @@ public: tx = std::move(arg); } - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { + SERIALIZE_METHODS(CMerkleTx, obj) + { std::vector vMerkleBranch; // For compatibility with older versions. - READWRITE(tx); - READWRITE(hashBlock); - READWRITE(vMerkleBranch); - READWRITE(nIndex); + READWRITE(obj.tx, obj.hashBlock, vMerkleBranch, obj.nIndex); } void SetMerkleBranch(const CBlockIndex* pIndex, int posInBlock); @@ -581,17 +576,12 @@ public: explicit CWalletKey(int64_t nExpires=0); - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { + SERIALIZE_METHODS(CWalletKey, obj) + { int nVersion = s.GetVersion(); if (!(s.GetType() & SER_GETHASH)) READWRITE(nVersion); - READWRITE(vchPrivKey); - READWRITE(nTimeCreated); - READWRITE(nTimeExpires); - READWRITE(LIMITED_STRING(strComment, 65536)); + READWRITE(obj.vchPrivKey, obj.nTimeCreated, obj.nTimeExpires, LIMITED_STRING(obj.strComment, 65536)); } }; @@ -1320,14 +1310,12 @@ public: vchPubKey = CPubKey(); } - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { + SERIALIZE_METHODS(CAccount, obj) + { int nVersion = s.GetVersion(); if (!(s.GetType() & SER_GETHASH)) READWRITE(nVersion); - READWRITE(vchPubKey); + READWRITE(obj.vchPubKey); } }; From d60a6d172965963882b1f6fb1f4d2babf80bc929 Mon Sep 17 00:00:00 2001 From: UdjinM6 Date: Thu, 27 May 2021 18:04:17 +0300 Subject: [PATCH 13/14] Convert bitset serializationis to use formatters --- src/llmq/quorums_commitment.h | 25 ++- src/llmq/quorums_dkgsession.h | 42 +++-- src/llmq/quorums_signing_shares.h | 15 +- src/serialize.h | 286 ++++++++++++++++-------------- 4 files changed, 188 insertions(+), 180 deletions(-) diff --git a/src/llmq/quorums_commitment.h b/src/llmq/quorums_commitment.h index 835dcb07f9..c4fb8abb0b 100644 --- a/src/llmq/quorums_commitment.h +++ b/src/llmq/quorums_commitment.h @@ -54,20 +54,19 @@ public: bool VerifySizes(const Consensus::LLMQParams& params) const; public: - ADD_SERIALIZE_METHODS - - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CFinalCommitment, obj) { - READWRITE(nVersion); - READWRITE(llmqType); - READWRITE(quorumHash); - READWRITE(DYNBITSET(signers)); - READWRITE(DYNBITSET(validMembers)); - READWRITE(quorumPublicKey); - READWRITE(quorumVvecHash); - READWRITE(quorumSig); - READWRITE(membersSig); + READWRITE( + obj.nVersion, + obj.llmqType, + obj.quorumHash, + DYNBITSET(obj.signers), + DYNBITSET(obj.validMembers), + obj.quorumPublicKey, + obj.quorumVvecHash, + obj.quorumSig, + obj.membersSig + ); } public: diff --git a/src/llmq/quorums_dkgsession.h b/src/llmq/quorums_dkgsession.h index 7ccde6e5a8..a08b50a854 100644 --- a/src/llmq/quorums_dkgsession.h +++ b/src/llmq/quorums_dkgsession.h @@ -95,17 +95,16 @@ public: CDKGComplaint() = default; explicit CDKGComplaint(const Consensus::LLMQParams& params); - ADD_SERIALIZE_METHODS - - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CDKGComplaint, obj) { - READWRITE(llmqType); - READWRITE(quorumHash); - READWRITE(proTxHash); - READWRITE(DYNBITSET(badMembers)); - READWRITE(DYNBITSET(complainForMembers)); - READWRITE(sig); + READWRITE( + obj.llmqType, + obj.quorumHash, + obj.proTxHash, + DYNBITSET(obj.badMembers), + DYNBITSET(obj.complainForMembers), + obj.sig + ); } uint256 GetSignHash() const @@ -174,19 +173,18 @@ public: } public: - ADD_SERIALIZE_METHODS - - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CDKGPrematureCommitment, obj) { - READWRITE(llmqType); - READWRITE(quorumHash); - READWRITE(proTxHash); - READWRITE(DYNBITSET(validMembers)); - READWRITE(quorumPublicKey); - READWRITE(quorumVvecHash); - READWRITE(quorumSig); - READWRITE(sig); + READWRITE( + obj.llmqType, + obj.quorumHash, + obj.proTxHash, + DYNBITSET(obj.validMembers), + obj.quorumPublicKey, + obj.quorumVvecHash, + obj.quorumSig, + obj.sig + ); } uint256 GetSignHash() const diff --git a/src/llmq/quorums_signing_shares.h b/src/llmq/quorums_signing_shares.h index bb4b5fec1f..0146458ce8 100644 --- a/src/llmq/quorums_signing_shares.h +++ b/src/llmq/quorums_signing_shares.h @@ -98,16 +98,13 @@ public: std::vector inv; public: - ADD_SERIALIZE_METHODS - - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CSigSharesInv, obj) { - uint64_t invSize = inv.size(); - - READWRITE(VARINT(sessionId)); - READWRITE(COMPACTSIZE(invSize)); - READWRITE(AUTOBITSET(inv, (size_t)invSize)); + uint64_t invSize = obj.inv.size(); + READWRITE(VARINT(obj.sessionId), COMPACTSIZE(invSize)); + autobitset_t bitset = std::make_pair(obj.inv, (size_t)invSize); + READWRITE(AUTOBITSET(bitset)); + SER_READ(obj, obj.inv = bitset.first); } void Init(size_t size); diff --git a/src/serialize.h b/src/serialize.h index 3dad1ff64f..399f660075 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -439,6 +439,143 @@ I ReadVarInt(Stream& is) } } +/** TODO: describe FixedBitSet */ +inline unsigned int GetSizeOfFixedBitSet(size_t size) +{ + return (size + 7) / 8; +} + +template +void WriteFixedBitSet(Stream& s, const std::vector& vec, size_t size) +{ + std::vector vBytes((size + 7) / 8); + size_t ms = std::min(size, vec.size()); + for (size_t p = 0; p < ms; p++) + vBytes[p / 8] |= vec[p] << (p % 8); + s.write((char*)vBytes.data(), vBytes.size()); +} + +template +void ReadFixedBitSet(Stream& s, std::vector& vec, size_t size) +{ + vec.resize(size); + + std::vector vBytes((size + 7) / 8); + s.read((char*)vBytes.data(), vBytes.size()); + for (size_t p = 0; p < size; p++) + vec[p] = (vBytes[p / 8] & (1 << (p % 8))) != 0; + if (vBytes.size() * 8 != size) { + size_t rem = vBytes.size() * 8 - size; + uint8_t m = ~(uint8_t)(0xff >> rem); + if (vBytes[vBytes.size() - 1] & m) { + throw std::ios_base::failure("Out-of-range bits set"); + } + } +} + +/** + * Stores a fixed size bitset as a series of VarInts. Each VarInt is an offset from the last entry and the sum of the + * last entry and the offset gives an index into the bitset for a set bit. The series of VarInts ends with a 0. + */ +template +void WriteFixedVarIntsBitSet(Stream& s, const std::vector& vec, size_t size) +{ + int32_t last = -1; + for (int32_t i = 0; i < (int32_t)vec.size(); i++) { + if (vec[i]) { + WriteVarInt(s, (uint32_t)(i - last)); + last = i; + } + } + WriteVarInt(s, 0); // stopper +} + +template +void ReadFixedVarIntsBitSet(Stream& s, std::vector& vec, size_t size) +{ + vec.assign(size, false); + + int32_t last = -1; + while(true) { + uint32_t offset = ReadVarInt(s); + if (offset == 0) { + break; + } + int32_t idx = last + offset; + if (idx >= size) { + throw std::ios_base::failure("out of bounds index"); + } + if (last != -1 && idx <= last) { + throw std::ios_base::failure("offset overflow"); + } + vec[idx] = true; + last = idx; + } +} + +/** + * Serializes either as a CFixedBitSet or CFixedVarIntsBitSet, depending on which would give a smaller size + */ +typedef std::pair, size_t> autobitset_t; + +struct CFixedBitSet +{ + const std::vector& vec; + size_t size; + CFixedBitSet(const std::vector& vecIn, size_t sizeIn) : vec(vecIn), size(sizeIn) {} + template + void Serialize(Stream& s) const { WriteFixedBitSet(s, vec, size); } +}; + +struct CFixedVarIntsBitSet +{ + const std::vector& vec; + size_t size; + CFixedVarIntsBitSet(const std::vector& vecIn, size_t sizeIn) : vec(vecIn), size(sizeIn) {} + template + void Serialize(Stream& s) const { WriteFixedVarIntsBitSet(s, vec, vec.size()); } +}; + +template +void WriteAutoBitSet(Stream& s, const autobitset_t& item) +{ + auto& vec = item.first; + auto& size = item.second; + + assert(vec.size() == size); + + size_t size1 = ::GetSerializeSize(s, CFixedBitSet(vec, size)); + size_t size2 = ::GetSerializeSize(s, CFixedVarIntsBitSet(vec, size)); + + assert(size1 == GetSizeOfFixedBitSet(size)); + + if (size1 < size2) { + ser_writedata8(s, 0); + WriteFixedBitSet(s, vec, vec.size()); + } else { + ser_writedata8(s, 1); + WriteFixedVarIntsBitSet(s, vec, vec.size()); + } +} + +template +void ReadAutoBitSet(Stream& s, autobitset_t& item) +{ + uint8_t isVarInts = ser_readdata8(s); + if (isVarInts != 0 && isVarInts != 1) { + throw std::ios_base::failure("invalid value for isVarInts byte"); + } + + auto& vec = item.first; + auto& size = item.second; + + if (!isVarInts) { + ReadFixedBitSet(s, vec, size); + } else { + ReadFixedVarIntsBitSet(s, vec, size); + } +} + /** Simple wrapper class to serialize objects using a formatter; used by Using(). */ template class Wrapper @@ -465,167 +602,44 @@ public: template static inline Wrapper Using(T&& t) { return Wrapper(t); } -#define FIXEDBITSET(obj, size) CFixedBitSet(REF(obj), (size)) -#define DYNBITSET(obj) CDynamicBitSet(REF(obj)) -#define FIXEDVARINTSBITSET(obj, size) CFixedVarIntsBitSet(REF(obj), (size)) -#define AUTOBITSET(obj, size) CAutoBitSet(REF(obj), (size)) +#define DYNBITSET(obj) Using(obj) +#define AUTOBITSET(obj) Using(obj) #define VARINT(obj, ...) Using>(obj) #define COMPACTSIZE(obj) Using(obj) #define LIMITED_STRING(obj,n) Using>(obj) -class CFixedBitSet +/** TODO: describe DynamicBitSet */ +struct DynamicBitSetFormatter { -protected: - std::vector& vec; - size_t size; - -public: - CFixedBitSet(std::vector& vecIn, size_t sizeIn) : vec(vecIn), size(sizeIn) {} - template - void Serialize(Stream& s) const - { - std::vector vBytes((size + 7) / 8); - size_t ms = std::min(size, vec.size()); - for (size_t p = 0; p < ms; p++) - vBytes[p / 8] |= vec[p] << (p % 8); - s.write((char*)vBytes.data(), vBytes.size()); - } - - template - void Unserialize(Stream& s) - { - vec.resize(size); - - std::vector vBytes((size + 7) / 8); - s.read((char*)vBytes.data(), vBytes.size()); - for (size_t p = 0; p < size; p++) - vec[p] = (vBytes[p / 8] & (1 << (p % 8))) != 0; - if (vBytes.size() * 8 != size) { - size_t rem = vBytes.size() * 8 - size; - uint8_t m = ~(uint8_t)(0xff >> rem); - if (vBytes[vBytes.size() - 1] & m) { - throw std::ios_base::failure("Out-of-range bits set"); - } - } - } -}; - -class CDynamicBitSet -{ -protected: - std::vector& vec; - -public: - explicit CDynamicBitSet(std::vector& vecIn) : vec(vecIn) {} - - template - void Serialize(Stream& s) const + void Ser(Stream& s, const std::vector& vec) const { WriteCompactSize(s, vec.size()); - CFixedBitSet(REF(vec), vec.size()).Serialize(s); + WriteFixedBitSet(s, vec, vec.size()); } template - void Unserialize(Stream& s) + void Unser(Stream& s, std::vector& vec) { - vec.resize(ReadCompactSize(s)); - CFixedBitSet(vec, vec.size()).Unserialize(s); - } -}; - -/** - * Stores a fixed size bitset as a series of VarInts. Each VarInt is an offset from the last entry and the sum of the - * last entry and the offset gives an index into the bitset for a set bit. The series of VarInts ends with a 0. - */ -class CFixedVarIntsBitSet -{ -protected: - std::vector& vec; - size_t size; - -public: - CFixedVarIntsBitSet(std::vector& vecIn, size_t sizeIn) : vec(vecIn), size(sizeIn) {} - - template - void Serialize(Stream& s) const - { - int32_t last = -1; - for (int32_t i = 0; i < (int32_t)vec.size(); i++) { - if (vec[i]) { - WriteVarInt(s, (uint32_t)(i - last)); - last = i; - } - } - WriteVarInt(s, 0); // stopper - } - - template - void Unserialize(Stream& s) - { - vec.assign(size, false); - - int32_t last = -1; - while(true) { - uint32_t offset = ReadVarInt(s); - if (offset == 0) { - break; - } - int32_t idx = last + offset; - if (idx >= size) { - throw std::ios_base::failure("out of bounds index"); - } - if (last != -1 && idx <= last) { - throw std::ios_base::failure("offset overflow"); - } - vec[idx] = true; - last = idx; - } + ReadFixedBitSet(s, vec, ReadCompactSize(s)); } }; /** * Serializes either as a CFixedBitSet or CFixedVarIntsBitSet, depending on which would give a smaller size */ -class CAutoBitSet +struct AutoBitSetFormatter { -protected: - std::vector& vec; - size_t size; - -public: - explicit CAutoBitSet(std::vector& vecIn, size_t sizeIn) : vec(vecIn), size(sizeIn) {} - template - void Serialize(Stream& s) const + void Ser(Stream& s, const autobitset_t& item) const { - assert(vec.size() == size); - - size_t size1 = ::GetSerializeSize(s, CFixedBitSet(vec, size)); - size_t size2 = ::GetSerializeSize(s, CFixedVarIntsBitSet(vec, size)); - - if (size1 < size2) { - ser_writedata8(s, 0); - s << FIXEDBITSET(vec, vec.size()); - } else { - ser_writedata8(s, 1); - s << FIXEDVARINTSBITSET(vec, vec.size()); - } + WriteAutoBitSet(s, item); } template - void Unserialize(Stream& s) + void Unser(Stream& s, autobitset_t& item) { - uint8_t isVarInts = ser_readdata8(s); - if (isVarInts != 0 && isVarInts != 1) { - throw std::ios_base::failure("invalid value for isVarInts byte"); - } - - if (!isVarInts) { - s >> FIXEDBITSET(vec, size); - } else { - s >> FIXEDVARINTSBITSET(vec, size); - } + ReadAutoBitSet(s, item); } }; From 011a178c6084c69e89cd7c456b8c6ad860f39be5 Mon Sep 17 00:00:00 2001 From: UdjinM6 Date: Thu, 27 May 2021 18:17:29 +0300 Subject: [PATCH 14/14] Other Dash-specific serializations --- src/bls/bls_ies.h | 18 ++----- src/cachemap.h | 20 ++------ src/cachemultimap.h | 12 ++--- src/coinjoin/coinjoin.h | 54 ++++++-------------- src/evo/cbtx.h | 13 ++--- src/evo/deterministicmns.h | 48 +++++++++--------- src/evo/mnauth.h | 8 +-- src/evo/providertx.h | 79 ++++++++++++------------------ src/evo/simplifiedmns.h | 42 ++++++---------- src/governance/governance-object.h | 53 ++++++++------------ src/governance/governance-vote.h | 16 ++---- src/governance/governance-votedb.h | 12 ++--- src/governance/governance.h | 63 ++++++++++++------------ src/hdchain.h | 39 ++++++--------- src/llmq/quorums.h | 22 ++++----- src/llmq/quorums_chainlocks.h | 9 +--- src/llmq/quorums_commitment.h | 9 +--- src/llmq/quorums_dkgsession.h | 11 +---- src/llmq/quorums_instantsend.h | 9 +--- src/llmq/quorums_signing.h | 15 ++---- src/llmq/quorums_signing_shares.h | 38 ++++---------- src/masternode/masternode-meta.h | 74 +++++++++++++--------------- src/netfulfilledman.h | 12 ++--- src/primitives/transaction.h | 23 ++++----- src/spentindex.h | 32 ++++-------- src/spork.h | 49 +++++++++--------- src/stacktraces.cpp | 28 +++-------- src/test/serialize_tests.cpp | 29 ++++++----- 28 files changed, 302 insertions(+), 535 deletions(-) diff --git a/src/bls/bls_ies.h b/src/bls/bls_ies.h index 05bee91317..c2a7bd2f40 100644 --- a/src/bls/bls_ies.h +++ b/src/bls/bls_ies.h @@ -18,14 +18,9 @@ public: uint256 GetIV(size_t idx) const; public: - ADD_SERIALIZE_METHODS - - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CBLSIESEncryptedBlob, obj) { - READWRITE(ephemeralPubKey); - READWRITE(ivSeed); - READWRITE(data); + READWRITE(obj.ephemeralPubKey, obj.ivSeed, obj.data); } public: @@ -98,14 +93,9 @@ public: bool Decrypt(size_t idx, const CBLSSecretKey& sk, Blob& blobRet) const; public: - ADD_SERIALIZE_METHODS - - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CBLSIESMultiRecipientBlobs, obj) { - READWRITE(ephemeralPubKey); - READWRITE(ivSeed); - READWRITE(blobs); + READWRITE(obj.ephemeralPubKey, obj.ivSeed, obj.blobs); } }; diff --git a/src/cachemap.h b/src/cachemap.h index 94c32d8176..af5474cb1b 100644 --- a/src/cachemap.h +++ b/src/cachemap.h @@ -28,13 +28,9 @@ struct CacheItem K key; V value; - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CacheItem, obj) { - READWRITE(key); - READWRITE(value); + READWRITE(obj.key, obj.value); } }; @@ -154,16 +150,10 @@ public: return *this; } - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CacheMap, obj) { - READWRITE(nMaxSize); - READWRITE(listItems); - if(ser_action.ForRead()) { - RebuildIndex(); - } + READWRITE(obj.nMaxSize, obj.listItems); + SER_READ(obj, obj.RebuildIndex()); } private: diff --git a/src/cachemultimap.h b/src/cachemultimap.h index ee6b367dca..935c9a0f62 100644 --- a/src/cachemultimap.h +++ b/src/cachemultimap.h @@ -192,16 +192,10 @@ public: return *this; } - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CacheMultiMap, obj) { - READWRITE(nMaxSize); - READWRITE(listItems); - if(ser_action.ForRead()) { - RebuildIndex(); - } + READWRITE(obj.nMaxSize, obj.listItems); + SER_READ(obj, obj.RebuildIndex()); } private: diff --git a/src/coinjoin/coinjoin.h b/src/coinjoin/coinjoin.h index 218577a310..2a9c955456 100644 --- a/src/coinjoin/coinjoin.h +++ b/src/coinjoin/coinjoin.h @@ -96,18 +96,13 @@ public: nStatusUpdate(nStatusUpdate), nMessageID(nMessageID) {}; - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CCoinJoinStatusUpdate, obj) { - READWRITE(nSessionID); - READWRITE(nState); + READWRITE(obj.nSessionID, obj.nState); if (s.GetVersion() <= 702015) { - READWRITE(nEntriesCount); + READWRITE(obj.nEntriesCount); } - READWRITE(nStatusUpdate); - READWRITE(nMessageID); + READWRITE(obj.nStatusUpdate, obj.nMessageID); } }; @@ -152,13 +147,9 @@ public: nDenom(nDenom), txCollateral(txCollateral){}; - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CCoinJoinAccept, obj) { - READWRITE(nDenom); - READWRITE(txCollateral); + READWRITE(obj.nDenom, obj.txCollateral); } friend bool operator==(const CCoinJoinAccept& a, const CCoinJoinAccept& b) @@ -193,14 +184,9 @@ public: { } - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CCoinJoinEntry, obj) { - READWRITE(vecTxDSIn); - READWRITE(txCollateral); - READWRITE(vecTxOut); + READWRITE(obj.vecTxDSIn, obj.txCollateral, obj.vecTxOut); } bool AddScriptSig(const CTxIn& txin); @@ -241,17 +227,11 @@ public: { } - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CCoinJoinQueue, obj) { - READWRITE(nDenom); - READWRITE(masternodeOutpoint); - READWRITE(nTime); - READWRITE(fReady); + READWRITE(obj.nDenom, obj.masternodeOutpoint, obj.nTime, obj.fReady); if (!(s.GetType() & SER_GETHASH)) { - READWRITE(vchSig); + READWRITE(obj.vchSig); } } @@ -317,17 +297,13 @@ public: { } - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CCoinJoinBroadcastTx, obj) { - READWRITE(tx); - READWRITE(masternodeOutpoint); + READWRITE(obj.tx, obj.masternodeOutpoint); if (!(s.GetType() & SER_GETHASH)) { - READWRITE(vchSig); + READWRITE(obj.vchSig); } - READWRITE(sigTime); + READWRITE(obj.sigTime); } friend bool operator==(const CCoinJoinBroadcastTx& a, const CCoinJoinBroadcastTx& b) diff --git a/src/evo/cbtx.h b/src/evo/cbtx.h index 6b74f95bae..a88247c01b 100644 --- a/src/evo/cbtx.h +++ b/src/evo/cbtx.h @@ -26,17 +26,12 @@ public: uint256 merkleRootQuorums; public: - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CCbTx, obj) { - READWRITE(nVersion); - READWRITE(nHeight); - READWRITE(merkleRootMNList); + READWRITE(obj.nVersion, obj.nHeight, obj.merkleRootMNList); - if (nVersion >= 2) { - READWRITE(merkleRootQuorums); + if (obj.nVersion >= 2) { + READWRITE(obj.merkleRootQuorums); } } diff --git a/src/evo/deterministicmns.h b/src/evo/deterministicmns.h index 5536d39262..2e554203a7 100644 --- a/src/evo/deterministicmns.h +++ b/src/evo/deterministicmns.h @@ -68,25 +68,24 @@ public: s >> *this; } - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CDeterministicMNState, obj) { - READWRITE(nRegisteredHeight); - READWRITE(nLastPaidHeight); - READWRITE(nPoSePenalty); - READWRITE(nPoSeRevivedHeight); - READWRITE(nPoSeBanHeight); - READWRITE(nRevocationReason); - READWRITE(confirmedHash); - READWRITE(confirmedHashWithProRegTxHash); - READWRITE(keyIDOwner); - READWRITE(pubKeyOperator); - READWRITE(keyIDVoting); - READWRITE(addr); - READWRITE(scriptPayout); - READWRITE(scriptOperatorPayout); + READWRITE( + obj.nRegisteredHeight, + obj.nLastPaidHeight, + obj.nPoSePenalty, + obj.nPoSeRevivedHeight, + obj.nPoSeBanHeight, + obj.nRevocationReason, + obj.confirmedHash, + obj.confirmedHashWithProRegTxHash, + obj.keyIDOwner, + obj.pubKeyOperator, + obj.keyIDVoting, + obj.addr, + obj.scriptPayout, + obj.scriptOperatorPayout + ); } void ResetOperatorFields() @@ -182,13 +181,10 @@ public: #undef DMN_STATE_DIFF_LINE } - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CDeterministicMNStateDiff, obj) { - READWRITE(VARINT(fields)); -#define DMN_STATE_DIFF_LINE(f) if (fields & Field_##f) READWRITE(state.f); + READWRITE(VARINT(obj.fields)); +#define DMN_STATE_DIFF_LINE(f) if (obj.fields & Field_##f) READWRITE(obj.state.f); DMN_STATE_DIFF_ALL_FIELDS #undef DMN_STATE_DIFF_LINE } @@ -247,7 +243,7 @@ public: template void Serialize(Stream& s) const { - NCONST_PTR(this)->SerializationOp(s, CSerActionSerialize(), false); + const_cast(this)->SerializationOp(s, CSerActionSerialize(), false); } template @@ -338,7 +334,7 @@ public: template void Serialize(Stream& s) const { - NCONST_PTR(this)->SerializationOpBase(s, CSerActionSerialize()); + const_cast(this)->SerializationOpBase(s, CSerActionSerialize()); // Serialize the map as a vector WriteCompactSize(s, mnMap.size()); for (const auto& p : mnMap) { diff --git a/src/evo/mnauth.h b/src/evo/mnauth.h index 95bfffb3cd..260daf43be 100644 --- a/src/evo/mnauth.h +++ b/src/evo/mnauth.h @@ -40,13 +40,9 @@ public: CBLSSignature sig; public: - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CMNAuth, obj) { - READWRITE(proRegTxHash); - READWRITE(sig); + READWRITE(obj.proRegTxHash, obj.sig); } static void PushMNAUTH(CNode* pnode, CConnman& connman); diff --git a/src/evo/providertx.h b/src/evo/providertx.h index 390f45b213..6ee94fe824 100644 --- a/src/evo/providertx.h +++ b/src/evo/providertx.h @@ -37,24 +37,23 @@ public: std::vector vchSig; public: - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CProRegTx, obj) { - READWRITE(nVersion); - READWRITE(nType); - READWRITE(nMode); - READWRITE(collateralOutpoint); - READWRITE(addr); - READWRITE(keyIDOwner); - READWRITE(pubKeyOperator); - READWRITE(keyIDVoting); - READWRITE(nOperatorReward); - READWRITE(scriptPayout); - READWRITE(inputsHash); + READWRITE( + obj.nVersion, + obj.nType, + obj.nMode, + obj.collateralOutpoint, + obj.addr, + obj.keyIDOwner, + obj.pubKeyOperator, + obj.keyIDVoting, + obj.nOperatorReward, + obj.scriptPayout, + obj.inputsHash + ); if (!(s.GetType() & SER_GETHASH)) { - READWRITE(vchSig); + READWRITE(obj.vchSig); } } @@ -100,18 +99,11 @@ public: CBLSSignature sig; public: - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CProUpServTx, obj) { - READWRITE(nVersion); - READWRITE(proTxHash); - READWRITE(addr); - READWRITE(scriptOperatorPayout); - READWRITE(inputsHash); + READWRITE(obj.nVersion, obj.proTxHash, obj.addr, obj.scriptOperatorPayout, obj.inputsHash); if (!(s.GetType() & SER_GETHASH)) { - READWRITE(sig); + READWRITE(obj.sig); } } @@ -149,20 +141,19 @@ public: std::vector vchSig; public: - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CProUpRegTx, obj) { - READWRITE(nVersion); - READWRITE(proTxHash); - READWRITE(nMode); - READWRITE(pubKeyOperator); - READWRITE(keyIDVoting); - READWRITE(scriptPayout); - READWRITE(inputsHash); + READWRITE( + obj.nVersion, + obj.proTxHash, + obj.nMode, + obj.pubKeyOperator, + obj.keyIDVoting, + obj.scriptPayout, + obj.inputsHash + ); if (!(s.GetType() & SER_GETHASH)) { - READWRITE(vchSig); + READWRITE(obj.vchSig); } } @@ -207,17 +198,11 @@ public: CBLSSignature sig; public: - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CProUpRevTx, obj) { - READWRITE(nVersion); - READWRITE(proTxHash); - READWRITE(nReason); - READWRITE(inputsHash); + READWRITE(obj.nVersion, obj.proTxHash, obj.nReason, obj.inputsHash); if (!(s.GetType() & SER_GETHASH)) { - READWRITE(sig); + READWRITE(obj.sig); } } diff --git a/src/evo/simplifiedmns.h b/src/evo/simplifiedmns.h index 1c30273b02..8eb43fc60a 100644 --- a/src/evo/simplifiedmns.h +++ b/src/evo/simplifiedmns.h @@ -47,17 +47,16 @@ public: } public: - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CSimplifiedMNListEntry, obj) { - READWRITE(proRegTxHash); - READWRITE(confirmedHash); - READWRITE(service); - READWRITE(pubKeyOperator); - READWRITE(keyIDVoting); - READWRITE(isValid); + READWRITE( + obj.proRegTxHash, + obj.confirmedHash, + obj.service, + obj.pubKeyOperator, + obj.keyIDVoting, + obj.isValid + ); } public: @@ -89,13 +88,9 @@ public: uint256 blockHash; public: - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CGetSimplifiedMNListDiff, obj) { - READWRITE(baseBlockHash); - READWRITE(blockHash); + READWRITE(obj.baseBlockHash, obj.blockHash); } }; @@ -114,21 +109,12 @@ public: std::vector newQuorums; public: - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CSimplifiedMNListDiff, obj) { - READWRITE(baseBlockHash); - READWRITE(blockHash); - READWRITE(cbTxMerkleTree); - READWRITE(cbTx); - READWRITE(deletedMNs); - READWRITE(mnList); + READWRITE(obj.baseBlockHash, obj.blockHash, obj.cbTxMerkleTree, obj.cbTx, obj.deletedMNs, obj.mnList); if (s.GetVersion() >= LLMQS_PROTO_VERSION) { - READWRITE(deletedQuorums); - READWRITE(newQuorums); + READWRITE(obj.deletedQuorums, obj.newQuorums); } } diff --git a/src/governance/governance-object.h b/src/governance/governance-object.h index 756aafba92..a8fed35142 100644 --- a/src/governance/governance-object.h +++ b/src/governance/governance-object.h @@ -61,18 +61,12 @@ struct vote_instance_t { { } - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(vote_instance_t, obj) { - int nOutcome = int(eOutcome); - READWRITE(nOutcome); - READWRITE(nTime); - READWRITE(nCreationTime); - if (ser_action.ForRead()) { - eOutcome = vote_outcome_enum_t(nOutcome); - } + int nOutcome; + SER_WRITE(obj, nOutcome = int(obj.eOutcome)); + READWRITE(nOutcome, obj.nTime, obj.nCreationTime); + SER_READ(obj, obj.eOutcome = vote_outcome_enum_t(nOutcome)); } }; @@ -81,12 +75,9 @@ typedef std::map vote_instance_m_t; struct vote_rec_t { vote_instance_m_t mapInstances; - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(vote_rec_t, obj) { - READWRITE(mapInstances); + READWRITE(obj.mapInstances); } }; @@ -292,30 +283,26 @@ public: // SERIALIZER - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CGovernanceObject, obj) { // SERIALIZE DATA FOR SAVING/LOADING OR NETWORK FUNCTIONS - READWRITE(nHashParent); - READWRITE(nRevision); - READWRITE(nTime); - READWRITE(nCollateralHash); - READWRITE(vchData); - READWRITE(nObjectType); - READWRITE(masternodeOutpoint); + READWRITE( + obj.nHashParent, + obj.nRevision, + obj.nTime, + obj.nCollateralHash, + obj.vchData, + obj.nObjectType, + obj.masternodeOutpoint + ); if (!(s.GetType() & SER_GETHASH)) { - READWRITE(vchSig); + READWRITE(obj.vchSig); } if (s.GetType() & SER_DISK) { // Only include these for the disk file format LogPrint(BCLog::GOBJECT, "CGovernanceObject::SerializationOp Reading/writing votes from/to disk\n"); - READWRITE(nDeletionTime); - READWRITE(fExpired); - READWRITE(mapCurrentMNVotes); - READWRITE(fileVotes); - LogPrint(BCLog::GOBJECT, "CGovernanceObject::SerializationOp hash = %s, vote count = %d\n", GetHash().ToString(), fileVotes.GetVoteCount()); + READWRITE(obj.nDeletionTime, obj.fExpired, obj.mapCurrentMNVotes, obj.fileVotes); + LogPrint(BCLog::GOBJECT, "CGovernanceObject::SerializationOp hash = %s, vote count = %d\n", obj.GetHash().ToString(), obj.fileVotes.GetVoteCount()); } // AFTER DESERIALIZATION OCCURS, CACHED VARIABLES MUST BE CALCULATED MANUALLY diff --git a/src/governance/governance-vote.h b/src/governance/governance-vote.h index b1f4b38e87..4ea872a6e7 100644 --- a/src/governance/governance-vote.h +++ b/src/governance/governance-vote.h @@ -117,21 +117,13 @@ public: std::string ToString() const; - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CGovernanceVote, obj) { - READWRITE(masternodeOutpoint); - READWRITE(nParentHash); - READWRITE(nVoteOutcome); - READWRITE(nVoteSignal); - READWRITE(nTime); + READWRITE(obj.masternodeOutpoint, obj.nParentHash, obj.nVoteOutcome, obj.nVoteSignal, obj.nTime); if (!(s.GetType() & SER_GETHASH)) { - READWRITE(vchSig); + READWRITE(obj.vchSig); } - if (ser_action.ForRead()) - UpdateHash(); + SER_READ(obj, obj.UpdateHash()); } }; diff --git a/src/governance/governance-votedb.h b/src/governance/governance-votedb.h index 6b62cc197d..b88ffdc3e9 100644 --- a/src/governance/governance-votedb.h +++ b/src/governance/governance-votedb.h @@ -65,16 +65,10 @@ public: void RemoveVotesFromMasternode(const COutPoint& outpointMasternode); std::set RemoveInvalidVotes(const COutPoint& outpointMasternode, bool fProposal); - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CGovernanceObjectVoteFile, obj) { - READWRITE(nMemoryVotes); - READWRITE(listVotes); - if (ser_action.ForRead()) { - RebuildIndex(); - } + READWRITE(obj.nMemoryVotes, obj.listVotes); + SER_READ(obj, obj.RebuildIndex()); } private: diff --git a/src/governance/governance.h b/src/governance/governance.h index d694ca2989..1f6b8dba00 100644 --- a/src/governance/governance.h +++ b/src/governance/governance.h @@ -114,15 +114,9 @@ public: return double(nCount) / double(nMax - nMin); } - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CRateCheckBuffer, obj) { - READWRITE(vecTimestamps); - READWRITE(nDataStart); - READWRITE(nDataEnd); - READWRITE(fBufferEmpty); + READWRITE(obj.vecTimestamps, obj.nDataStart, obj.nDataEnd, obj.fBufferEmpty); } }; @@ -141,13 +135,9 @@ public: // Types { } - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(last_object_rec, obj) { - READWRITE(triggerBuffer); - READWRITE(fStatusOK); + READWRITE(obj.triggerBuffer, obj.fStatusOK); } CRateCheckBuffer triggerBuffer; @@ -274,30 +264,37 @@ public: std::string ToString() const; UniValue ToJson() const; - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) + template + void Serialize(Stream &s) const { LOCK(cs); + s << SERIALIZATION_VERSION_STRING + << mapErasedGovernanceObjects + << cmapInvalidVotes + << cmmapOrphanVotes + << mapObjects + << mapLastMasternodeObject + << *lastMNListForVotingKeys; + } + + template + void Unserialize(Stream &s) + { + LOCK(cs); + Clear(); + std::string strVersion; - if (ser_action.ForRead()) { - Clear(); - READWRITE(strVersion); - if (strVersion != SERIALIZATION_VERSION_STRING) { - return; - } - } else { - strVersion = SERIALIZATION_VERSION_STRING; - READWRITE(strVersion); + s >> strVersion; + if (strVersion != SERIALIZATION_VERSION_STRING) { + return; } - READWRITE(mapErasedGovernanceObjects); - READWRITE(cmapInvalidVotes); - READWRITE(cmmapOrphanVotes); - READWRITE(mapObjects); - READWRITE(mapLastMasternodeObject); - READWRITE(*lastMNListForVotingKeys); + s >> mapErasedGovernanceObjects + >> cmapInvalidVotes + >> cmmapOrphanVotes + >> mapObjects + >> mapLastMasternodeObject + >> *lastMNListForVotingKeys; } void UpdatedBlockTip(const CBlockIndex* pindex, CConnman& connman); diff --git a/src/hdchain.h b/src/hdchain.h index a1f455c468..d1ee445151 100644 --- a/src/hdchain.h +++ b/src/hdchain.h @@ -15,12 +15,9 @@ public: CHDAccount() : nExternalChainCounter(0), nInternalChainCounter(0) {} - ADD_SERIALIZE_METHODS; - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CHDAccount, obj) { - READWRITE(nExternalChainCounter); - READWRITE(nInternalChainCounter); + READWRITE(obj.nExternalChainCounter, obj.nInternalChainCounter); } }; @@ -56,18 +53,18 @@ public: mapAccounts(other.mapAccounts) {} - ADD_SERIALIZE_METHODS; - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CHDChain, obj) { - LOCK(cs); - READWRITE(this->nVersion); - READWRITE(id); - READWRITE(fCrypted); - READWRITE(vchSeed); - READWRITE(vchMnemonic); - READWRITE(vchMnemonicPassphrase); - READWRITE(mapAccounts); + LOCK(obj.cs); + READWRITE( + obj.nVersion, + obj.id, + obj.fCrypted, + obj.vchSeed, + obj.vchMnemonic, + obj.vchMnemonicPassphrase, + obj.mapAccounts + ); } void swap(CHDChain& first, CHDChain& second) // nothrow @@ -133,15 +130,9 @@ public: CHDPubKey() : nVersion(CHDPubKey::CURRENT_VERSION), nAccountIndex(0), nChangeIndex(0) {} - ADD_SERIALIZE_METHODS; - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CHDPubKey, obj) { - READWRITE(this->nVersion); - READWRITE(extPubKey); - READWRITE(hdchainID); - READWRITE(nAccountIndex); - READWRITE(nChangeIndex); + READWRITE(obj.nVersion, obj.extPubKey, obj.hdchainID, obj.nAccountIndex, obj.nChangeIndex); } std::string GetKeyPath() const; diff --git a/src/llmq/quorums.h b/src/llmq/quorums.h index 1d8e84818a..1bf6d5c8e2 100644 --- a/src/llmq/quorums.h +++ b/src/llmq/quorums.h @@ -79,23 +79,19 @@ public: nTime(GetTime()), fProcessed(false) {} - ADD_SERIALIZE_METHODS - - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CQuorumDataRequest, obj) { - READWRITE(llmqType); - READWRITE(quorumHash); - READWRITE(nDataMask); - READWRITE(proTxHash); - if (ser_action.ForRead()) { + bool fRead{false}; + SER_READ(obj, fRead = true); + READWRITE(obj.llmqType, obj.quorumHash, obj.nDataMask, obj.proTxHash); + if (fRead) { try { - READWRITE(nError); + READWRITE(obj.nError); } catch (...) { - nError = UNDEFINED; + SER_READ(obj, obj.nError = UNDEFINED); } - } else if (nError != UNDEFINED) { - READWRITE(nError); + } else if (obj.nError != UNDEFINED) { + READWRITE(obj.nError); } } diff --git a/src/llmq/quorums_chainlocks.h b/src/llmq/quorums_chainlocks.h index a0172c513c..3f283a0bfc 100644 --- a/src/llmq/quorums_chainlocks.h +++ b/src/llmq/quorums_chainlocks.h @@ -31,14 +31,9 @@ public: CBLSSignature sig; public: - ADD_SERIALIZE_METHODS - - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CChainLockSig, obj) { - READWRITE(nHeight); - READWRITE(blockHash); - READWRITE(sig); + READWRITE(obj.nHeight, obj.blockHash, obj.sig); } bool IsNull() const; diff --git a/src/llmq/quorums_commitment.h b/src/llmq/quorums_commitment.h index c4fb8abb0b..15f0f7f6e3 100644 --- a/src/llmq/quorums_commitment.h +++ b/src/llmq/quorums_commitment.h @@ -114,14 +114,9 @@ public: CFinalCommitment commitment; public: - ADD_SERIALIZE_METHODS - - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CFinalCommitmentTxPayload, obj) { - READWRITE(nVersion); - READWRITE(nHeight); - READWRITE(commitment); + READWRITE(obj.nVersion, obj.nHeight, obj.commitment); } void ToJson(UniValue& obj) const diff --git a/src/llmq/quorums_dkgsession.h b/src/llmq/quorums_dkgsession.h index a08b50a854..191e51146c 100644 --- a/src/llmq/quorums_dkgsession.h +++ b/src/llmq/quorums_dkgsession.h @@ -125,16 +125,9 @@ public: CBLSSignature sig; public: - ADD_SERIALIZE_METHODS - - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CDKGJustification, obj) { - READWRITE(llmqType); - READWRITE(quorumHash); - READWRITE(proTxHash); - READWRITE(contributions); - READWRITE(sig); + READWRITE(obj.llmqType, obj.quorumHash, obj.proTxHash, obj.contributions, obj.sig); } uint256 GetSignHash() const diff --git a/src/llmq/quorums_instantsend.h b/src/llmq/quorums_instantsend.h index 7b963ab36d..6fbdbf4d16 100644 --- a/src/llmq/quorums_instantsend.h +++ b/src/llmq/quorums_instantsend.h @@ -26,14 +26,9 @@ public: CBLSLazySignature sig; public: - ADD_SERIALIZE_METHODS - - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CInstantSendLock, obj) { - READWRITE(inputs); - READWRITE(txid); - READWRITE(sig); + READWRITE(obj.inputs, obj.txid, obj.sig); } uint256 GetRequestId() const; diff --git a/src/llmq/quorums_signing.h b/src/llmq/quorums_signing.h index 69bca65bf0..68fd46afd0 100644 --- a/src/llmq/quorums_signing.h +++ b/src/llmq/quorums_signing.h @@ -46,19 +46,10 @@ public: public: - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CRecoveredSig, obj) { - READWRITE(llmqType); - READWRITE(quorumHash); - READWRITE(id); - READWRITE(msgHash); - READWRITE(sig); - if (ser_action.ForRead()) { - UpdateHash(); - } + READWRITE(obj.llmqType, obj.quorumHash, obj.id, obj.msgHash, obj.sig); + SER_READ(obj, obj.UpdateHash()); } void UpdateHash() diff --git a/src/llmq/quorums_signing_shares.h b/src/llmq/quorums_signing_shares.h index 0146458ce8..55e73bd666 100644 --- a/src/llmq/quorums_signing_shares.h +++ b/src/llmq/quorums_signing_shares.h @@ -48,20 +48,10 @@ public: return key.first; } - ADD_SERIALIZE_METHODS - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(llmqType); - READWRITE(quorumHash); - READWRITE(quorumMember); - READWRITE(id); - READWRITE(msgHash); - READWRITE(sigShare); - - if (ser_action.ForRead()) { - UpdateKey(); - } + SERIALIZE_METHODS(CSigShare, obj) + { + READWRITE(obj.llmqType, obj.quorumHash, obj.quorumMember, obj.id, obj.msgHash, obj.sigShare); + SER_READ(obj, obj.UpdateKey()); } }; @@ -77,15 +67,9 @@ public: uint256 id; uint256 msgHash; - ADD_SERIALIZE_METHODS - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(VARINT(sessionId)); - READWRITE(llmqType); - READWRITE(quorumHash); - READWRITE(id); - READWRITE(msgHash); + SERIALIZE_METHODS(CSigSesAnn, obj) + { + READWRITE(VARINT(obj.sessionId), obj.llmqType, obj.quorumHash, obj.id, obj.msgHash); } std::string ToString() const; @@ -125,13 +109,9 @@ public: std::vector> sigShares; public: - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CBatchedSigShares, obj) { - READWRITE(VARINT(sessionId)); - READWRITE(sigShares); + READWRITE(VARINT(obj.sessionId), obj.sigShares); } std::string ToInvString() const; diff --git a/src/masternode/masternode-meta.h b/src/masternode/masternode-meta.h index 1b5d14a7ed..eda960c207 100644 --- a/src/masternode/masternode-meta.h +++ b/src/masternode/masternode-meta.h @@ -50,17 +50,17 @@ public: { } - ADD_SERIALIZE_METHODS - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(CMasternodeMetaInfo, obj) { - LOCK(cs); - READWRITE(proTxHash); - READWRITE(nLastDsq); - READWRITE(nMixingTxCount); - READWRITE(mapGovernanceObjectsVotedOn); - READWRITE(lastOutboundAttempt); - READWRITE(lastOutboundSuccess); + LOCK(obj.cs); + READWRITE( + obj.proTxHash, + obj.nLastDsq, + obj.nMixingTxCount, + obj.mapGovernanceObjectsVotedOn, + obj.lastOutboundAttempt, + obj.lastOutboundSuccess + ); } UniValue ToJson() const; @@ -89,7 +89,7 @@ class CMasternodeMetaMan private: static const std::string SERIALIZATION_VERSION_STRING; - CCriticalSection cs; + mutable CCriticalSection cs; std::map metaInfos; std::vector vecDirtyGovernanceObjectHashes; @@ -98,41 +98,33 @@ private: int64_t nDsqCount = 0; public: - ADD_SERIALIZE_METHODS - - template - inline void SerializationOp(Stream& s, Operation ser_action) + template + void Serialize(Stream &s) const { LOCK(cs); - - std::string strVersion; - if(ser_action.ForRead()) { - Clear(); - READWRITE(strVersion); - if (strVersion != SERIALIZATION_VERSION_STRING) { - return; - } - } - else { - strVersion = SERIALIZATION_VERSION_STRING; - READWRITE(strVersion); - } - std::vector tmpMetaInfo; - if (ser_action.ForRead()) { - READWRITE(tmpMetaInfo); - metaInfos.clear(); - for (auto& mm : tmpMetaInfo) { - metaInfos.emplace(mm.GetProTxHash(), std::make_shared(std::move(mm))); - } - } else { - for (auto& p : metaInfos) { - tmpMetaInfo.emplace_back(*p.second); - } - READWRITE(tmpMetaInfo); + for (auto& p : metaInfos) { + tmpMetaInfo.emplace_back(*p.second); } + s << SERIALIZATION_VERSION_STRING << tmpMetaInfo << nDsqCount; + } - READWRITE(nDsqCount); + template + void Unserialize(Stream &s) + { + LOCK(cs); + Clear(); + std::string strVersion; + s >> strVersion; + if (strVersion != SERIALIZATION_VERSION_STRING) { + return; + } + std::vector tmpMetaInfo; + s >> tmpMetaInfo >> nDsqCount; + metaInfos.clear(); + for (auto& mm : tmpMetaInfo) { + metaInfos.emplace(mm.GetProTxHash(), std::make_shared(std::move(mm))); + } } public: diff --git a/src/netfulfilledman.h b/src/netfulfilledman.h index 2204f7e73c..e1e13cd586 100644 --- a/src/netfulfilledman.h +++ b/src/netfulfilledman.h @@ -22,19 +22,17 @@ private: //keep track of what node has/was asked for and when fulfilledreqmap_t mapFulfilledRequests; - CCriticalSection cs_mapFulfilledRequests; + mutable CCriticalSection cs_mapFulfilledRequests; void RemoveFulfilledRequest(const CService& addr, const std::string& strRequest); public: CNetFulfilledRequestManager() {} - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - LOCK(cs_mapFulfilledRequests); - READWRITE(mapFulfilledRequests); + SERIALIZE_METHODS(CNetFulfilledRequestManager, obj) + { + LOCK(obj.cs_mapFulfilledRequests); + READWRITE(obj.mapFulfilledRequests); } void AddFulfilledRequest(const CService& addr, const std::string& strRequest); diff --git a/src/primitives/transaction.h b/src/primitives/transaction.h index e1a9e0ff22..70c9886358 100644 --- a/src/primitives/transaction.h +++ b/src/primitives/transaction.h @@ -278,21 +278,16 @@ struct CMutableTransaction CMutableTransaction(); CMutableTransaction(const CTransaction& tx); - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - int32_t n32bitVersion = this->nVersion | (this->nType << 16); + SERIALIZE_METHODS(CMutableTransaction, obj) + { + int32_t n32bitVersion; + SER_WRITE(obj, n32bitVersion = obj.nVersion | (obj.nType << 16)); READWRITE(n32bitVersion); - if (ser_action.ForRead()) { - this->nVersion = (int16_t) (n32bitVersion & 0xffff); - this->nType = (int16_t) ((n32bitVersion >> 16) & 0xffff); - } - READWRITE(vin); - READWRITE(vout); - READWRITE(nLockTime); - if (this->nVersion == 3 && this->nType != TRANSACTION_NORMAL) { - READWRITE(vExtraPayload); + SER_READ(obj, obj.nVersion = (int16_t) (n32bitVersion & 0xffff)); + SER_READ(obj, obj.nType = (int16_t) ((n32bitVersion >> 16) & 0xffff)); + READWRITE(obj.vin, obj.vout, obj.nLockTime); + if (obj.nVersion == 3 && obj.nType != TRANSACTION_NORMAL) { + READWRITE(obj.vExtraPayload); } } diff --git a/src/spentindex.h b/src/spentindex.h index 3b90956154..2de0df571d 100644 --- a/src/spentindex.h +++ b/src/spentindex.h @@ -15,12 +15,9 @@ struct CSpentIndexKey { uint256 txid; unsigned int outputIndex; - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(txid); - READWRITE(outputIndex); + SERIALIZE_METHODS(CSpentIndexKey, obj) + { + READWRITE(obj.txid, obj.outputIndex); } CSpentIndexKey(uint256 t, unsigned int i) { @@ -47,16 +44,9 @@ struct CSpentIndexValue { int addressType; uint160 addressHash; - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(txid); - READWRITE(inputIndex); - READWRITE(blockHeight); - READWRITE(satoshis); - READWRITE(addressType); - READWRITE(addressHash); + SERIALIZE_METHODS(CSpentIndexValue, obj) + { + READWRITE(obj.txid, obj.inputIndex, obj.blockHeight, obj.satoshis, obj.addressType, obj.addressHash); } CSpentIndexValue(uint256 t, unsigned int i, int h, CAmount s, int type, uint160 a) { @@ -211,13 +201,9 @@ struct CAddressUnspentValue { CScript script; int blockHeight; - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(satoshis); - READWRITE(script); - READWRITE(blockHeight); + SERIALIZE_METHODS(CAddressUnspentValue, obj) + { + READWRITE(obj.satoshis, obj.script, obj.blockHeight); } CAddressUnspentValue(CAmount sats, CScript scriptPubKey, int height) { diff --git a/src/spork.h b/src/spork.h index 57ec391143..b7d92c3f9e 100644 --- a/src/spork.h +++ b/src/spork.h @@ -94,14 +94,9 @@ public: {} - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(nSporkID); - READWRITE(nValue); - READWRITE(nTimeSigned); - READWRITE(vchSig); + SERIALIZE_METHODS(CSporkMessage, obj) + { + READWRITE(obj.nSporkID, obj.nValue, obj.nTimeSigned, obj.vchSig); } /** @@ -176,27 +171,27 @@ public: CSporkManager(); - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - std::string strVersion; - if(ser_action.ForRead()) { - READWRITE(strVersion); - if (strVersion != SERIALIZATION_VERSION_STRING) { - return; - } - } else { - strVersion = SERIALIZATION_VERSION_STRING; - READWRITE(strVersion); - } - // we don't serialize pubkey ids because pubkeys should be + template + void Serialize(Stream &s) const + { + // We don't serialize pubkey ids because pubkeys should be // hardcoded or be set with cmdline or options, should - // not reuse pubkeys from previous dashd run + // not reuse pubkeys from previous dashd run. + // We don't serialize private key to prevent its leakage. LOCK(cs); - READWRITE(mapSporksByHash); - READWRITE(mapSporksActive); - // we don't serialize private key to prevent its leakage + s << SERIALIZATION_VERSION_STRING << mapSporksByHash << mapSporksActive; + } + + template + void Unserialize(Stream &s) + { + LOCK(cs); + std::string strVersion; + s >> strVersion; + if (strVersion != SERIALIZATION_VERSION_STRING) { + return; + } + s >> mapSporksByHash >> mapSporksActive; } /** diff --git a/src/stacktraces.cpp b/src/stacktraces.cpp index cb6cdb75e6..2ce4be2d67 100644 --- a/src/stacktraces.cpp +++ b/src/stacktraces.cpp @@ -310,15 +310,9 @@ struct stackframe_info { int lineno{-1}; std::string function; - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(stackframe_info, obj) { - READWRITE(pc); - READWRITE(filename); - READWRITE(lineno); - READWRITE(function); + READWRITE(obj.pc, obj.filename, obj.lineno, obj.function); } }; @@ -376,14 +370,9 @@ struct crash_info_header uint16_t version; std::string exeFileName; - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(crash_info_header, obj) { - READWRITE(magic); - READWRITE(version); - READWRITE(exeFileName); + READWRITE(obj.magic, obj.version, obj.exeFileName); } }; @@ -393,14 +382,9 @@ struct crash_info std::vector stackframes; std::vector stackframeInfos; - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) + SERIALIZE_METHODS(crash_info, obj) { - READWRITE(crashDescription); - READWRITE(stackframes); - READWRITE(stackframeInfos); + READWRITE(obj.crashDescription, obj.stackframes, obj.stackframeInfos); } void ConvertAddresses(int64_t offset) diff --git a/src/test/serialize_tests.cpp b/src/test/serialize_tests.cpp index 346255b87e..1378e77e09 100644 --- a/src/test/serialize_tests.cpp +++ b/src/test/serialize_tests.cpp @@ -354,29 +354,32 @@ struct old_version { int field1; - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(field1); + SERIALIZE_METHODS(old_version, obj) + { + READWRITE(obj.field1); } -};\ +}; + struct new_version { int field1; int field2; - ADD_SERIALIZE_METHODS; + template + void Serialize(Stream &s) const + { + s << field1 << field2; + } - template - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITE(field1); - if(ser_action.ForRead() && (s.size() == 0)) - { + template + void Unserialize(Stream &s) + { + s >> field1; + if (s.size() == 0) { field2 = 0; return; } - READWRITE(field2); + s >> field2; } };