sikkienl/dash
1
0
Fork 0
mirror of https://github.com/dashpay/dash.git synced 2024-12-27 04:52:59 +01:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge #18112: Convert blockencodings.h to new serialization framework

Browse Source 
353f376277

Partial #18112: Add CustomUintFormatter

e574fff53e

Partial #18112: Add DifferenceFormatter

10633398f2

Partial #18112: Make VectorFormatter support stateful formatters

56dd9f04c7

Partial #18112: Convert CCompactSize to proper formatter

3ca574cef0
This commit is contained in:
Kittywhiskers Van Gogh 2020-02-03 19:49:10 -08:00 committed by pasta
parent bfe5971359
commit 97caca3033
No known key found for this signature in database
GPG Key ID: 52527BEDABE87984
3 changed files with 87 additions and 130 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

144
src/blockencodings.h
View File

@ -11,18 +11,29 @@
class CTxMemPool; class CTxMemPool;
// Dumb helper to handle CTransaction compression at serialize-time // Transaction compression schemes for compact block relay can be introduced by writing
struct TransactionCompressor { // an actual formatter here.
private: using TransactionCompression = DefaultFormatter;
CTransactionRef& tx;
class DifferenceFormatter
{
uint64_t m_shift = 0;
public: public:
explicit TransactionCompressor(CTransactionRef& txIn) : tx(txIn) {} template<typename Stream, typename I>
void Ser(Stream& s, I v)
ADD_SERIALIZE_METHODS; {
if (v < m_shift || v >= std::numeric_limits<uint64_t>::max()) throw std::ios_base::failure("differential value overflow");
template <typename Stream, typename Operation> WriteCompactSize(s, v - m_shift);
inline void SerializationOp(Stream& s, Operation ser_action) { m_shift = uint64_t(v) + 1;
READWRITE(tx); //TODO: Compress tx encoding }
template<typename Stream, typename I>
void Unser(Stream& s, I& v)
{
uint64_t n = ReadCompactSize(s);
m_shift += n;
if (m_shift < n || m_shift >= std::numeric_limits<uint64_t>::max() || m_shift < std::numeric_limits<I>::min() || m_shift > std::numeric_limits<I>::max()) throw std::ios_base::failure("differential value overflow");
v = I(m_shift++);
} }
}; };
@ -32,39 +43,9 @@ public:
uint256 blockhash; uint256 blockhash;
std::vector<uint16_t> indexes; std::vector<uint16_t> indexes;
ADD_SERIALIZE_METHODS; SERIALIZE_METHODS(BlockTransactionsRequest, obj)
{
template <typename Stream, typename Operation> READWRITE(obj.blockhash, Using<VectorFormatter<DifferenceFormatter>>(obj.indexes));
inline void SerializationOp(Stream& s, Operation ser_action) {
READWRITE(blockhash);
uint64_t indexes_size = (uint64_t)indexes.size();
READWRITE(COMPACTSIZE(indexes_size));
if (ser_action.ForRead()) {
size_t i = 0;
while (indexes.size() < indexes_size) {
indexes.resize(std::min((uint64_t)(1000 + indexes.size()), indexes_size));
for (; i < indexes.size(); i++) {
uint64_t index = 0;
READWRITE(COMPACTSIZE(index));
if (index > std::numeric_limits<uint16_t>::max())
throw std::ios_base::failure("index overflowed 16 bits");
indexes[i] = index;
}
}
int32_t offset = 0;
for (size_t j = 0; j < indexes.size(); j++) {
if (int32_t(indexes[j]) + offset > std::numeric_limits<uint16_t>::max())
throw std::ios_base::failure("indexes overflowed 16 bits");
indexes[j] = indexes[j] + offset;
offset = int32_t(indexes[j]) + 1;
}
} else {
for (size_t i = 0; i < indexes.size(); i++) {
uint64_t index = indexes[i] - (i == 0 ? 0 : (indexes[i - 1] + 1));
READWRITE(COMPACTSIZE(index));
}
}
} }
}; };
@ -78,24 +59,9 @@ public:
explicit BlockTransactions(const BlockTransactionsRequest& req) : explicit BlockTransactions(const BlockTransactionsRequest& req) :
blockhash(req.blockhash), txn(req.indexes.size()) {} blockhash(req.blockhash), txn(req.indexes.size()) {}
ADD_SERIALIZE_METHODS; SERIALIZE_METHODS(BlockTransactions, obj)
{
template <typename Stream, typename Operation> READWRITE(obj.blockhash, Using<VectorFormatter<TransactionCompression>>(obj.txn));
inline void SerializationOp(Stream& s, Operation ser_action) {
READWRITE(blockhash);
uint64_t txn_size = (uint64_t)txn.size();
READWRITE(COMPACTSIZE(txn_size));
if (ser_action.ForRead()) {
size_t i = 0;
while (txn.size() < txn_size) {
txn.resize(std::min((uint64_t)(1000 + txn.size()), txn_size));
for (; i < txn.size(); i++)
READWRITE(TransactionCompressor(txn[i]));
}
} else {
for (size_t i = 0; i < txn.size(); i++)
READWRITE(TransactionCompressor(txn[i]));
}
} }
}; };
@ -106,17 +72,7 @@ struct PrefilledTransaction {
uint16_t index; uint16_t index;
CTransactionRef tx; CTransactionRef tx;
ADD_SERIALIZE_METHODS; SERIALIZE_METHODS(PrefilledTransaction, obj) { READWRITE(COMPACTSIZE(obj.index), Using<TransactionCompression>(obj.tx)); }
template <typename Stream, typename Operation>
inline void SerializationOp(Stream& s, Operation ser_action) {
uint64_t idx = index;
READWRITE(COMPACTSIZE(idx));
if (idx > std::numeric_limits<uint16_t>::max())
throw std::ios_base::failure("index overflowed 16-bits");
index = idx;
READWRITE(TransactionCompressor(tx));
}
}; };
typedef enum ReadStatus_t typedef enum ReadStatus_t
@ -154,43 +110,15 @@ public:
size_t BlockTxCount() const { return shorttxids.size() + prefilledtxn.size(); } size_t BlockTxCount() const { return shorttxids.size() + prefilledtxn.size(); }
ADD_SERIALIZE_METHODS; SERIALIZE_METHODS(CBlockHeaderAndShortTxIDs, obj)
{
template <typename Stream, typename Operation> READWRITE(obj.header, obj.nonce, Using<VectorFormatter<CustomUintFormatter<SHORTTXIDS_LENGTH>>>(obj.shorttxids), obj.prefilledtxn);
inline void SerializationOp(Stream& s, Operation ser_action) {
READWRITE(header);
READWRITE(nonce);
uint64_t shorttxids_size = (uint64_t)shorttxids.size();
READWRITE(COMPACTSIZE(shorttxids_size));
if (ser_action.ForRead()) { if (ser_action.ForRead()) {
size_t i = 0; if (obj.BlockTxCount() > std::numeric_limits<uint16_t>::max()) {
while (shorttxids.size() < shorttxids_size) {
shorttxids.resize(std::min((uint64_t)(1000 + shorttxids.size()), shorttxids_size));
for (; i < shorttxids.size(); i++) {
uint32_t lsb = 0; uint16_t msb = 0;
READWRITE(lsb);
READWRITE(msb);
shorttxids[i] = (uint64_t(msb) << 32) | uint64_t(lsb);
static_assert(SHORTTXIDS_LENGTH == 6, "shorttxids serialization assumes 6-byte shorttxids");
}
}
} else {
for (size_t i = 0; i < shorttxids.size(); i++) {
uint32_t lsb = shorttxids[i] & 0xffffffff;
uint16_t msb = (shorttxids[i] >> 32) & 0xffff;
READWRITE(lsb);
READWRITE(msb);
}
}
READWRITE(prefilledtxn);
if (BlockTxCount() > std::numeric_limits<uint16_t>::max())
throw std::ios_base::failure("indexes overflowed 16 bits"); throw std::ios_base::failure("indexes overflowed 16 bits");
}
if (ser_action.ForRead()) obj.FillShortTxIDSelector();
FillShortTxIDSelector(); }
} }
}; };

9
src/prevector.h
View File

@ -452,15 +452,20 @@ public:
return first; return first;
} }
void push_back(const T& value) { template<typename... Args>
void emplace_back(Args&&... args) {
size_type new_size = size() + 1; size_type new_size = size() + 1;
if (capacity() < new_size) { if (capacity() < new_size) {
change_capacity(new_size + (new_size >> 1)); change_capacity(new_size + (new_size >> 1));
} }
new(item_ptr(size())) T(value); new(item_ptr(size())) T(std::forward<Args>(args)...);
_size++; _size++;
} }
void push_back(const T& value) {
emplace_back(value);
}
void pop_back() { void pop_back() {
erase(end() - 1, end()); erase(end() - 1, end());
} }

66
src/serialize.h
View File

@ -504,7 +504,7 @@ static inline Wrapper<Formatter, T&> Using(T&& t) { return Wrapper<Formatter, T&
#define FIXEDVARINTSBITSET(obj, size) CFixedVarIntsBitSet(REF(obj), (size)) #define FIXEDVARINTSBITSET(obj, size) CFixedVarIntsBitSet(REF(obj), (size))
#define AUTOBITSET(obj, size) CAutoBitSet(REF(obj), (size)) #define AUTOBITSET(obj, size) CAutoBitSet(REF(obj), (size))
#define VARINT(obj, ...) Using<VarIntFormatter<__VA_ARGS__>>(obj) #define VARINT(obj, ...) Using<VarIntFormatter<__VA_ARGS__>>(obj)
#define COMPACTSIZE(obj) CCompactSize(REF(obj)) #define COMPACTSIZE(obj) Using<CompactSizeFormatter>(obj)
#define LIMITED_STRING(obj,n) LimitedString< n >(REF(obj)) #define LIMITED_STRING(obj,n) LimitedString< n >(REF(obj))
class CFixedBitSet class CFixedBitSet
@ -678,6 +678,28 @@ struct VarIntFormatter
} }
}; };
template<int Bytes>
struct CustomUintFormatter
{
static_assert(Bytes > 0 && Bytes <= 8, "CustomUintFormatter Bytes out of range");
static constexpr uint64_t MAX = 0xffffffffffffffff >> (8 * (8 - Bytes));
template <typename Stream, typename I> void Ser(Stream& s, I v)
{
if (v < 0 || v > MAX) throw std::ios_base::failure("CustomUintFormatter value out of range");
uint64_t raw = htole64(v);
s.write((const char*)&raw, Bytes);
}
template <typename Stream, typename I> void Unser(Stream& s, I& v)
{
static_assert(std::numeric_limits<I>::max() >= MAX && std::numeric_limits<I>::min() <= 0, "CustomUintFormatter type too small");
uint64_t raw = 0;
s.read((char*)&raw, Bytes);
v = le64toh(raw);
}
};
/** Serialization wrapper class for big-endian integers. /** Serialization wrapper class for big-endian integers.
* *
* Use this wrapper around integer types that are stored in memory in native * Use this wrapper around integer types that are stored in memory in native
@ -712,25 +734,26 @@ public:
} }
}; };
class CCompactSize /** Formatter for integers in CompactSize format. */
struct CompactSizeFormatter
{ {
protected: template<typename Stream, typename I>
uint64_t &n; void Unser(Stream& s, I& v)
public: {
explicit CCompactSize(uint64_t& nIn) : n(nIn) { } uint64_t n = ReadCompactSize<Stream>(s);
if (n < std::numeric_limits<I>::min() || n > std::numeric_limits<I>::max()) {
unsigned int GetSerializeSize() const { throw std::ios_base::failure("CompactSize exceeds limit of type");
return GetSizeOfCompactSize(n); }
v = n;
} }
template<typename Stream> template<typename Stream, typename I>
void Serialize(Stream &s) const { void Ser(Stream& s, I v)
WriteCompactSize<Stream>(s, n); {
} static_assert(std::is_unsigned<I>::value, "CompactSize only supported for unsigned integers");
static_assert(std::numeric_limits<I>::max() <= std::numeric_limits<uint64_t>::max(), "CompactSize only supports 64-bit integers and below");
template<typename Stream> WriteCompactSize<Stream>(s, v);
void Unserialize(Stream& s) {
n = ReadCompactSize<Stream>(s);
} }
}; };
@ -777,7 +800,7 @@ BigEndian<I> WrapBigEndian(I& n) { return BigEndian<I>(n); }
* as a vector of VarInt-encoded integers. * as a vector of VarInt-encoded integers.
* *
* V is not required to be an std::vector type. It works for any class that * V is not required to be an std::vector type. It works for any class that
* exposes a value_type, size, reserve, push_back, and const iterators. * exposes a value_type, size, reserve, emplace_back, back, and const iterators.
*/ */
template<class Formatter> template<class Formatter>
struct VectorFormatter struct VectorFormatter
@ -785,15 +808,17 @@ struct VectorFormatter
template<typename Stream, typename V> template<typename Stream, typename V>
void Ser(Stream& s, const V& v) void Ser(Stream& s, const V& v)
{ {
Formatter formatter;
WriteCompactSize(s, v.size()); WriteCompactSize(s, v.size());
for (const typename V::value_type& elem : v) { for (const typename V::value_type& elem : v) {
s << Using<Formatter>(elem); formatter.Ser(s, elem);
} }
} }
template<typename Stream, typename V> template<typename Stream, typename V>
void Unser(Stream& s, V& v) void Unser(Stream& s, V& v)
{ {
Formatter formatter;
v.clear(); v.clear();
size_t size = ReadCompactSize(s); size_t size = ReadCompactSize(s);
size_t allocated = 0; size_t allocated = 0;
@ -805,9 +830,8 @@ struct VectorFormatter
allocated = std::min(size, allocated + MAX_VECTOR_ALLOCATE / sizeof(typename V::value_type)); allocated = std::min(size, allocated + MAX_VECTOR_ALLOCATE / sizeof(typename V::value_type));
v.reserve(allocated); v.reserve(allocated);
while (v.size() < allocated) { while (v.size() < allocated) {
typename V::value_type val; v.emplace_back();
s >> Using<Formatter>(val); formatter.Unser(s, v.back());
v.push_back(std::move(val));
} }
} }
}; };