feat: store protx version in CSimplifiedMNListEntry and use it to ser/deser pubKeyOperator (#5397)

## Issue being fixed or feature implemented
Mobile wallets would have to convert 4k+ pubkeys at the V19 fork point
and it's a pretty hard job for them that can easily take 10-15 seconds
if not more. Also after the HF, if a masternode list is requested from
before the HF, the operator keys come in basic scheme, but the
merkelroot was calculated with legacy. From mobile team work it wasn't
possible to convert all operator keys to legacy and then calculate the
correct merkleroot.

~This PR builds on top of ~#5392~ #5403 (changes that belong to this PR:
26f7e966500bdea4c604f1d16716b40b366fc707 and
4b42dc8fcee3354afd82ce7e3a72ebe1659f5f22) and aims to solve both of
these issues.~

cc @hashengineering @QuantumExplorer 

## What was done?
Introduce `nVersion` on p2p level for every CSimplifiedMNListEntry. Set
`nVersion` to the same value we have it in CDeterministicMNState i.e.
pubkey serialization would not be via basic scheme only after the V19
fork, it would match the way it’s serialized on-chain/in
CDeterministicMNState for that specific MN.

## How Has This Been Tested?
run tests

## Breaking Changes
NOTE: `testnet` is going to re-fork at v19 forkpoint because
`merkleRootMNList` is not going to match

## Checklist:
- [x] I have performed a self-review of my own code
- [ ] I have commented my code, particularly in hard-to-understand areas
- [ ] I have added or updated relevant unit/integration/functional/e2e
tests
- [ ] I have made corresponding changes to the documentation
- [ ] I have assigned this pull request to a milestone _(for repository
code-owners and collaborators only)_

src/evo/cbtx.cpp

@@ -128,8 +128,7 @@ bool CalcCbTxMerkleRootMNList(const CBlock& block, const CBlockIndex* pindexPrev
         int64_t nTime2 = GetTimeMicros(); nTimeDMN += nTime2 - nTime1;
         LogPrint(BCLog::BENCHMARK, "            - BuildNewListFromBlock: %.2fms [%.2fs]\n", 0.001 * (nTime2 - nTime1), nTimeDMN * 0.000001);
 
-        bool v19active = llmq::utils::IsV19Active(pindexPrev);
+        CSimplifiedMNList sml(tmpMNList);
-        CSimplifiedMNList sml(tmpMNList, v19active);
 
         int64_t nTime3 = GetTimeMicros(); nTimeSMNL += nTime3 - nTime2;
         LogPrint(BCLog::BENCHMARK, "            - CSimplifiedMNList: %.2fms [%.2fs]\n", 0.001 * (nTime3 - nTime2), nTimeSMNL * 0.000001);

src/evo/simplifiedmns.cpp

@@ -33,6 +33,7 @@ CSimplifiedMNListEntry::CSimplifiedMNListEntry(const CDeterministicMN& dmn) :
     isValid(!dmn.pdmnState->IsBanned()),
     scriptPayout(dmn.pdmnState->scriptPayout),
     scriptOperatorPayout(dmn.pdmnState->scriptOperatorPayout),
+    nVersion(dmn.pdmnState->nVersion == CProRegTx::LEGACY_BLS_VERSION ? LEGACY_BLS_VERSION : BASIC_BLS_VERSION),
     nType(dmn.nType),
     platformHTTPPort(dmn.pdmnState->platformHTTPPort),
     platformNodeID(dmn.pdmnState->platformNodeID)
@@ -70,7 +71,7 @@ void CSimplifiedMNListEntry::ToJson(UniValue& obj, bool extended) const
     obj.pushKV("proRegTxHash", proRegTxHash.ToString());
     obj.pushKV("confirmedHash", confirmedHash.ToString());
     obj.pushKV("service", service.ToString(false));
-    obj.pushKV("pubKeyOperator", pubKeyOperator.Get().ToString());
+    obj.pushKV("pubKeyOperator", pubKeyOperator.ToString());
     obj.pushKV("votingAddress", EncodeDestination(PKHash(keyIDVoting)));
     obj.pushKV("isValid", isValid);
     obj.pushKV("nVersion", nVersion);
@@ -104,15 +105,13 @@ CSimplifiedMNList::CSimplifiedMNList(const std::vector<CSimplifiedMNListEntry>&
     });
 }
 
-CSimplifiedMNList::CSimplifiedMNList(const CDeterministicMNList& dmnList, bool isV19Active)
+CSimplifiedMNList::CSimplifiedMNList(const CDeterministicMNList& dmnList)
 {
     mnList.resize(dmnList.GetAllMNsCount());
 
     size_t i = 0;
-    dmnList.ForEachMN(false, [this, &i, isV19Active](auto& dmn) {
+    dmnList.ForEachMN(false, [this, &i](auto& dmn) {
-        auto sme = std::make_unique<CSimplifiedMNListEntry>(dmn);
+        mnList[i++] = std::make_unique<CSimplifiedMNListEntry>(dmn);
-        sme->nVersion = isV19Active ? CSimplifiedMNListEntry::BASIC_BLS_VERSION : CSimplifiedMNListEntry::LEGACY_BLS_VERSION;
-        mnList[i++] = std::move(sme);
     });
 
     std::sort(mnList.begin(), mnList.end(), [&](const std::unique_ptr<CSimplifiedMNListEntry>& a, const std::unique_ptr<CSimplifiedMNListEntry>& b) {
@@ -239,23 +238,18 @@ void CSimplifiedMNListDiff::ToJson(UniValue& obj, bool extended) const
 
 CSimplifiedMNListDiff BuildSimplifiedDiff(const CDeterministicMNList& from, const CDeterministicMNList& to, bool extended)
 {
-    bool v19active = llmq::utils::IsV19Active(::ChainActive().Tip());
     CSimplifiedMNListDiff diffRet;
     diffRet.baseBlockHash = from.GetBlockHash();
     diffRet.blockHash = to.GetBlockHash();
-    diffRet.nVersion = v19active ? CSimplifiedMNListDiff::BASIC_BLS_VERSION : CSimplifiedMNListDiff::LEGACY_BLS_VERSION;
 
     to.ForEachMN(false, [&](const auto& toPtr) {
         auto fromPtr = from.GetMN(toPtr.proTxHash);
         if (fromPtr == nullptr) {
             CSimplifiedMNListEntry sme(toPtr);
-            sme.nVersion = diffRet.nVersion;
             diffRet.mnList.push_back(std::move(sme));
         } else {
             CSimplifiedMNListEntry sme1(toPtr);
             CSimplifiedMNListEntry sme2(*fromPtr);
-            sme1.nVersion = diffRet.nVersion;
-            sme2.nVersion = diffRet.nVersion;
             if ((sme1 != sme2) ||
                 (extended && (sme1.scriptPayout != sme2.scriptPayout || sme1.scriptOperatorPayout != sme2.scriptOperatorPayout))) {
                     diffRet.mnList.push_back(std::move(sme1));

src/evo/simplifiedmns.h

@@ -39,7 +39,7 @@ public:
     uint160 platformNodeID{};
     CScript scriptPayout; // mem-only
     CScript scriptOperatorPayout; // mem-only
-    uint16_t nVersion{LEGACY_BLS_VERSION}; // mem-only
+    uint16_t nVersion{LEGACY_BLS_VERSION};
 
     CSimplifiedMNListEntry() = default;
     explicit CSimplifiedMNListEntry(const CDeterministicMN& dmn);
@@ -65,6 +65,9 @@ public:
 
     SERIALIZE_METHODS(CSimplifiedMNListEntry, obj)
     {
+        if ((s.GetType() & SER_NETWORK) && s.GetVersion() >= SMNLE_VERSIONED_PROTO_VERSION) {
+            READWRITE(obj.nVersion);
+        }
         READWRITE(
                 obj.proRegTxHash,
                 obj.confirmedHash,
@@ -98,7 +101,7 @@ public:
 
     CSimplifiedMNList() = default;
     explicit CSimplifiedMNList(const std::vector<CSimplifiedMNListEntry>& smlEntries);
-    explicit CSimplifiedMNList(const CDeterministicMNList& dmnList, bool isV19Active);
+    explicit CSimplifiedMNList(const CDeterministicMNList& dmnList);
 
     uint256 CalcMerkleRoot(bool* pmutated = nullptr) const;
     bool operator==(const CSimplifiedMNList& rhs) const;
@@ -121,8 +124,7 @@ public:
 class CSimplifiedMNListDiff
 {
 public:
-    static constexpr uint16_t LEGACY_BLS_VERSION = 1;
+    static constexpr uint16_t CURRENT_VERSION = 1;
-    static constexpr uint16_t BASIC_BLS_VERSION = 2;
 
     uint256 baseBlockHash;
     uint256 blockHash;
@@ -130,7 +132,7 @@ public:
     CTransactionRef cbTx;
     std::vector<uint256> deletedMNs;
     std::vector<CSimplifiedMNListEntry> mnList;
-    uint16_t nVersion{LEGACY_BLS_VERSION};
+    uint16_t nVersion{CURRENT_VERSION};
 
     std::vector<std::pair<uint8_t, uint256>> deletedQuorums; // p<LLMQType, quorumHash>
     std::vector<llmq::CFinalCommitment> newQuorums;

src/version.h

@@ -11,7 +11,7 @@
  */
 
 
-static const int PROTOCOL_VERSION = 70227;
+static const int PROTOCOL_VERSION = 70228;
 
 //! initial proto version, to be increased after version/verack negotiation
 static const int INIT_PROTO_VERSION = 209;
@@ -49,6 +49,9 @@ static const int COINJOIN_PROTX_HASH_PROTO_VERSION = 70226;
 //! Masternode type was introduced in this version
 static const int DMN_TYPE_PROTO_VERSION = 70227;
 
+//! Versioned Simplified Masternode List Entries were introduced in this version
+static const int SMNLE_VERSIONED_PROTO_VERSION = 70228;
+
 // Make sure that none of the values above collide with `ADDRV2_FORMAT`.
 
 #endif // BITCOIN_VERSION_H

test/functional/feature_dip3_v19.py

@@ -166,7 +166,7 @@ class DIP3V19Test(DashTestFramework):
         # Verify that the merkle root matches what we locally calculate
         hashes = []
         for mn in sorted(new_mn_list.values(), key=lambda mn: ser_uint256(mn.proRegTxHash)):
-            hashes.append(hash256(mn.serialize()))
+            hashes.append(hash256(mn.serialize(with_version = False)))
         merkle_root = CBlock.get_merkle_root(hashes)
         assert_equal(merkle_root, cbtx.merkleRootMNList)
 

test/functional/feature_dip4_coinbasemerkleroots.py

@@ -180,7 +180,7 @@ class LLMQCoinbaseCommitmentsTest(DashTestFramework):
         # Verify that the merkle root matches what we locally calculate
         hashes = []
         for mn in sorted(newMNList.values(), key=lambda mn: ser_uint256(mn.proRegTxHash)):
-            hashes.append(hash256(mn.serialize()))
+            hashes.append(hash256(mn.serialize(with_version = False)))
         merkleRoot = CBlock.get_merkle_root(hashes)
         assert_equal(merkleRoot, cbtx.merkleRootMNList)
 

test/functional/feature_llmq_hpmn.py

@@ -253,7 +253,7 @@ class LLMQHPMNTest(DashTestFramework):
         # Verify that the merkle root matches what we locally calculate
         hashes = []
         for mn in sorted(newMNList.values(), key=lambda mn: ser_uint256(mn.proRegTxHash)):
-            hashes.append(hash256(mn.serialize()))
+            hashes.append(hash256(mn.serialize(with_version = False)))
         merkleRoot = CBlock.get_merkle_root(hashes)
         assert_equal(merkleRoot, cbtx.merkleRootMNList)
 

test/functional/test_framework/messages.py

@@ -32,7 +32,7 @@ from test_framework.util import hex_str_to_bytes, assert_equal
 import dash_hash
 
 MIN_VERSION_SUPPORTED = 60001
-MY_VERSION = 70227  # DMN_TYPE_PROTO_VERSION
+MY_VERSION = 70228  # SMNLE_VERSIONED_PROTO_VERSION
 MY_SUBVERSION = b"/python-mininode-tester:0.0.3%s/"
 MY_RELAY = 1 # from version 70001 onwards, fRelay should be appended to version messages (BIP37)
 
@@ -1095,7 +1095,7 @@ class CCbTx:
 
 
 class CSimplifiedMNListEntry:
-    __slots__ = ("proRegTxHash", "confirmedHash", "service", "pubKeyOperator", "keyIDVoting", "isValid", "version", "type", "platformHTTPPort", "platformNodeID")
+    __slots__ = ("proRegTxHash", "confirmedHash", "service", "pubKeyOperator", "keyIDVoting", "isValid", "nVersion", "type", "platformHTTPPort", "platformNodeID")
 
     def __init__(self):
         self.set_null()
@@ -1107,34 +1107,36 @@ class CSimplifiedMNListEntry:
         self.pubKeyOperator = b'\x00' * 48
         self.keyIDVoting = 0
         self.isValid = False
-        self.version = 0
+        self.nVersion = 0
         self.type = 0
         self.platformHTTPPort = 0
         self.platformNodeID = b'\x00' * 20
 
-    def deserialize(self, f, version):
+    def deserialize(self, f):
-        self.version = version # memory only
+        self.nVersion = struct.unpack("<H", f.read(2))[0]
         self.proRegTxHash = deser_uint256(f)
         self.confirmedHash = deser_uint256(f)
         self.service.deserialize(f)
         self.pubKeyOperator = f.read(48)
         self.keyIDVoting = f.read(20)
         self.isValid = struct.unpack("<?", f.read(1))[0]
-        if self.version == 2:
+        if self.nVersion == 2:
             self.type = struct.unpack("<H", f.read(2))[0]
             if self.type == 1:
                 self.platformHTTPPort = struct.unpack("<H", f.read(2))[0]
                 self.platformNodeID = f.read(20)
 
-    def serialize(self):
+    def serialize(self, with_version = True):
         r = b""
+        if with_version:
+            r += struct.pack("<H", self.nVersion)
         r += ser_uint256(self.proRegTxHash)
         r += ser_uint256(self.confirmedHash)
         r += self.service.serialize()
         r += self.pubKeyOperator
         r += self.keyIDVoting
         r += struct.pack("<?", self.isValid)
-        if self.version == 2:
+        if self.nVersion == 2:
             r += struct.pack("<H", self.type)
             if self.type == 1:
                 r += struct.pack("<H", self.platformHTTPPort)
@@ -2021,7 +2023,7 @@ class msg_getmnlistd:
 QuorumId = namedtuple('QuorumId', ['llmqType', 'quorumHash'])
 
 class msg_mnlistdiff:
-    __slots__ = ("baseBlockHash", "blockHash", "merkleProof", "cbTx", "version", "deletedMNs", "mnList", "deletedQuorums", "newQuorums",)
+    __slots__ = ("baseBlockHash", "blockHash", "merkleProof", "cbTx", "nVersion", "deletedMNs", "mnList", "deletedQuorums", "newQuorums",)
     command = b"mnlistdiff"
 
     def __init__(self):
@@ -2029,7 +2031,7 @@ class msg_mnlistdiff:
         self.blockHash = 0
         self.merkleProof = CPartialMerkleTree()
         self.cbTx = None
-        self.version = 0
+        self.nVersion = 0
         self.deletedMNs = []
         self.mnList = []
         self.deletedQuorums = []
@@ -2042,12 +2044,12 @@ class msg_mnlistdiff:
         self.cbTx = CTransaction()
         self.cbTx.deserialize(f)
         self.cbTx.rehash()
-        self.version = struct.unpack("<H", f.read(2))[0]
+        self.nVersion = struct.unpack("<H", f.read(2))[0]
         self.deletedMNs = deser_uint256_vector(f)
         self.mnList = []
         for i in range(deser_compact_size(f)):
             e = CSimplifiedMNListEntry()
-            e.deserialize(f, self.version)
+            e.deserialize(f)
             self.mnList.append(e)
 
         self.deletedQuorums = []