From 6a25151bbd4feb29c86bb5c99428c14a846a3fec Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <6098974-kittywhiskers@users.noreply.gitlab.com> Date: Mon, 26 Aug 2019 11:32:47 -0700 Subject: [PATCH 1/4] Merge #16730: Support serialization of std::vector --- src/serialize.h | 13 +++++++++++++ src/test/serialize_tests.cpp | 8 ++++++++ 2 files changed, 21 insertions(+) diff --git a/src/serialize.h b/src/serialize.h index d7ace2ff92..0bc1e51d8a 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -863,6 +863,7 @@ template inline void Unserialize(St * vectors of unsigned char are a special case and are intended to be serialized as a single opaque blob. */ template void Serialize_impl(Stream& os, const std::vector& v, const unsigned char&); +template void Serialize_impl(Stream& os, const std::vector& v, const bool&); template void Serialize_impl(Stream& os, const std::vector& v, const V&); template inline void Serialize(Stream& os, const std::vector& v); template void Unserialize_impl(Stream& is, std::vector& v, const unsigned char&); @@ -1061,6 +1062,18 @@ void Serialize_impl(Stream& os, const std::vector& v, const unsigned char& os.write((char*)v.data(), v.size() * sizeof(T)); } +template +void Serialize_impl(Stream& os, const std::vector& v, const bool&) +{ + // A special case for std::vector, as dereferencing + // std::vector::const_iterator does not result in a const bool& + // due to std::vector's special casing for bool arguments. + WriteCompactSize(os, v.size()); + for (bool elem : v) { + ::Serialize(os, elem); + } +} + template void Serialize_impl(Stream& os, const std::vector& v, const V&) { diff --git a/src/test/serialize_tests.cpp b/src/test/serialize_tests.cpp index a12877ef27..c1fbaec319 100644 --- a/src/test/serialize_tests.cpp +++ b/src/test/serialize_tests.cpp @@ -257,6 +257,14 @@ static bool isCanonicalException(const std::ios_base::failure& ex) return strcmp(expectedException.what(), ex.what()) == 0; } +BOOST_AUTO_TEST_CASE(vector_bool) +{ + std::vector vec1{1, 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 1, 1, 1, 0, 1, 0, 0, 1}; + std::vector vec2{1, 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 1, 1, 1, 0, 1, 0, 0, 1}; + + BOOST_CHECK(vec1 == std::vector(vec2.begin(), vec2.end())); + BOOST_CHECK(SerializeHash(vec1) == SerializeHash(vec2)); +} BOOST_AUTO_TEST_CASE(noncanonical) { From 4b2b5f78d461d915577f3dfb02c57dc45a000be1 Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <6098974-kittywhiskers@users.noreply.gitlab.com> Date: Tue, 11 May 2021 20:41:07 +0530 Subject: [PATCH 2/4] Merge #16702: supplying and using asmap to improve IP bucketing --- src/Makefile.am | 2 + src/Makefile.test.include | 3 +- src/addrman.cpp | 58 +++++-- src/addrman.h | 103 +++++++++--- src/init.cpp | 24 +++ src/net.cpp | 11 +- src/net.h | 6 +- src/netaddress.cpp | 70 ++++++-- src/netaddress.h | 8 +- src/rpc/net.cpp | 4 + src/test/addrman_tests.cpp | 321 +++++++++++++++++++++++++++++++++---- src/test/data/asmap.raw | Bin 0 -> 59 bytes src/test/netbase_tests.cpp | 28 ++-- src/utilasmap.cpp | 97 +++++++++++ src/utilasmap.h | 10 ++ 15 files changed, 641 insertions(+), 104 deletions(-) create mode 100644 src/test/data/asmap.raw create mode 100644 src/utilasmap.cpp create mode 100644 src/utilasmap.h diff --git a/src/Makefile.am b/src/Makefile.am index 6decad0180..9832690d7d 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -259,6 +259,7 @@ BITCOIN_CORE_H = \ undo.h \ unordered_lru_cache.h \ util.h \ + utilasmap.h \ utilmemory.h \ utilmoneystr.h \ utiltime.h \ @@ -580,6 +581,7 @@ libdash_util_a_SOURCES = \ sync.cpp \ threadinterrupt.cpp \ util.cpp \ + utilasmap.cpp \ utilmoneystr.cpp \ utilstrencodings.cpp \ utiltime.cpp \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index a461a75f18..0ad3d24da0 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -21,7 +21,8 @@ JSON_TEST_FILES = \ test/data/tx_invalid.json \ test/data/tx_valid.json -RAW_TEST_FILES = +RAW_TEST_FILES = \ + test/data/asmap.raw GENERATED_TEST_FILES = $(JSON_TEST_FILES:.json=.json.h) $(RAW_TEST_FILES:.raw=.raw.h) diff --git a/src/addrman.cpp b/src/addrman.cpp index ac37931dde..a04265c382 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -8,20 +8,27 @@ #include #include #include +#include -int CAddrInfo::GetTriedBucket(const uint256& nKey) const +int CAddrInfo::GetTriedBucket(const uint256& nKey, const std::vector &asmap) const { uint64_t hash1 = (CHashWriter(SER_GETHASH, 0) << nKey << GetKey()).GetHash().GetCheapHash(); - uint64_t hash2 = (CHashWriter(SER_GETHASH, 0) << nKey << GetGroup() << (hash1 % ADDRMAN_TRIED_BUCKETS_PER_GROUP)).GetHash().GetCheapHash(); - return hash2 % ADDRMAN_TRIED_BUCKET_COUNT; + uint64_t hash2 = (CHashWriter(SER_GETHASH, 0) << nKey << GetGroup(asmap) << (hash1 % ADDRMAN_TRIED_BUCKETS_PER_GROUP)).GetHash().GetCheapHash(); + int tried_bucket = hash2 % ADDRMAN_TRIED_BUCKET_COUNT; + uint32_t mapped_as = GetMappedAS(asmap); + LogPrint(BCLog::NET, "IP %s mapped to AS%i belongs to tried bucket %i.\n", ToStringIP(), mapped_as, tried_bucket); + return tried_bucket; } -int CAddrInfo::GetNewBucket(const uint256& nKey, const CNetAddr& src) const +int CAddrInfo::GetNewBucket(const uint256& nKey, const CNetAddr& src, const std::vector &asmap) const { - std::vector vchSourceGroupKey = src.GetGroup(); - uint64_t hash1 = (CHashWriter(SER_GETHASH, 0) << nKey << GetGroup() << vchSourceGroupKey).GetHash().GetCheapHash(); + std::vector vchSourceGroupKey = src.GetGroup(asmap); + uint64_t hash1 = (CHashWriter(SER_GETHASH, 0) << nKey << GetGroup(asmap) << vchSourceGroupKey).GetHash().GetCheapHash(); uint64_t hash2 = (CHashWriter(SER_GETHASH, 0) << nKey << vchSourceGroupKey << (hash1 % ADDRMAN_NEW_BUCKETS_PER_SOURCE_GROUP)).GetHash().GetCheapHash(); - return hash2 % ADDRMAN_NEW_BUCKET_COUNT; + int new_bucket = hash2 % ADDRMAN_NEW_BUCKET_COUNT; + uint32_t mapped_as = GetMappedAS(asmap); + LogPrint(BCLog::NET, "IP %s mapped to AS%i belongs to new bucket %i.\n", ToStringIP(), mapped_as, new_bucket); + return new_bucket; } int CAddrInfo::GetBucketPosition(const uint256 &nKey, bool fNew, int nBucket) const @@ -169,7 +176,7 @@ void CAddrMan::MakeTried(CAddrInfo& info, int nId) assert(info.nRefCount == 0); // which tried bucket to move the entry to - int nKBucket = info.GetTriedBucket(nKey); + int nKBucket = info.GetTriedBucket(nKey, m_asmap); int nKBucketPos = info.GetBucketPosition(nKey, false, nKBucket); // first make space to add it (the existing tried entry there is moved to new, deleting whatever is there). @@ -185,7 +192,7 @@ void CAddrMan::MakeTried(CAddrInfo& info, int nId) nTried--; // find which new bucket it belongs to - int nUBucket = infoOld.GetNewBucket(nKey); + int nUBucket = infoOld.GetNewBucket(nKey, m_asmap); int nUBucketPos = infoOld.GetBucketPosition(nKey, true, nUBucket); ClearNew(nUBucket, nUBucketPos); assert(vvNew[nUBucket][nUBucketPos] == -1); @@ -249,7 +256,7 @@ void CAddrMan::Good_(const CService& addr, bool test_before_evict, int64_t nTime return; // which tried bucket to move the entry to - int tried_bucket = info.GetTriedBucket(nKey); + int tried_bucket = info.GetTriedBucket(nKey, m_asmap); int tried_bucket_pos = info.GetBucketPosition(nKey, false, tried_bucket); // Will moving this address into tried evict another entry? @@ -315,7 +322,7 @@ bool CAddrMan::Add_(const CAddress& addr, const CNetAddr& source, int64_t nTimeP fNew = true; } - int nUBucket = pinfo->GetNewBucket(nKey, source); + int nUBucket = pinfo->GetNewBucket(nKey, source, m_asmap); int nUBucketPos = pinfo->GetBucketPosition(nKey, true, nUBucket); if (vvNew[nUBucket][nUBucketPos] != nId) { bool fInsert = vvNew[nUBucket][nUBucketPos] == -1; @@ -453,7 +460,7 @@ int CAddrMan::Check_() if (vvTried[n][i] != -1) { if (!setTried.count(vvTried[n][i])) return -11; - if (mapInfo[vvTried[n][i]].GetTriedBucket(nKey) != n) + if (mapInfo[vvTried[n][i]].GetTriedBucket(nKey, m_asmap) != n) return -17; if (mapInfo[vvTried[n][i]].GetBucketPosition(nKey, false, n) != i) return -18; @@ -580,7 +587,7 @@ void CAddrMan::ResolveCollisions_() CAddrInfo& info_new = mapInfo[id_new]; // Which tried bucket to move the entry to. - int tried_bucket = info_new.GetTriedBucket(nKey); + int tried_bucket = info_new.GetTriedBucket(nKey, m_asmap); int tried_bucket_pos = info_new.GetBucketPosition(nKey, false, tried_bucket); if (!info_new.IsValid()) { // id_new may no longer map to a valid address erase_collision = true; @@ -637,10 +644,33 @@ CAddrInfo CAddrMan::SelectTriedCollision_() CAddrInfo& newInfo = mapInfo[id_new]; // which tried bucket to move the entry to - int tried_bucket = newInfo.GetTriedBucket(nKey); + int tried_bucket = newInfo.GetTriedBucket(nKey, m_asmap); int tried_bucket_pos = newInfo.GetBucketPosition(nKey, false, tried_bucket); int id_old = vvTried[tried_bucket][tried_bucket_pos]; return mapInfo[id_old]; } + +std::vector CAddrMan::DecodeAsmap(fs::path path) +{ + std::vector bits; + FILE *filestr = fsbridge::fopen(path, "rb"); + CAutoFile file(filestr, SER_DISK, CLIENT_VERSION); + if (file.IsNull()) { + LogPrintf("Failed to open asmap file from disk.\n"); + return bits; + } + fseek(filestr, 0, SEEK_END); + int length = ftell(filestr); + LogPrintf("Opened asmap file %s (%d bytes) from disk.\n", path, length); + fseek(filestr, 0, SEEK_SET); + char cur_byte; + for (int i = 0; i < length; ++i) { + file >> cur_byte; + for (int bit = 0; bit < 8; ++bit) { + bits.push_back((cur_byte >> bit) & 1); + } + } + return bits; +} diff --git a/src/addrman.h b/src/addrman.h index 7a4cd4ee64..0698e1c555 100644 --- a/src/addrman.h +++ b/src/addrman.h @@ -12,11 +12,17 @@ #include #include #include +#include #include #include #include #include +#include +#include +#include +#include + /** * Extended statistics about a CAddress @@ -83,15 +89,15 @@ public: } //! Calculate in which "tried" bucket this entry belongs - int GetTriedBucket(const uint256 &nKey) const; + int GetTriedBucket(const uint256 &nKey, const std::vector &asmap) const; //! Calculate in which "new" bucket this entry belongs, given a certain source - int GetNewBucket(const uint256 &nKey, const CNetAddr& src) const; + int GetNewBucket(const uint256 &nKey, const CNetAddr& src, const std::vector &asmap) const; //! Calculate in which "new" bucket this entry belongs, using its default source - int GetNewBucket(const uint256 &nKey) const + int GetNewBucket(const uint256 &nKey, const std::vector &asmap) const { - return GetNewBucket(nKey, source); + return GetNewBucket(nKey, source, asmap); } //! Calculate in which position of a bucket to store this entry. @@ -184,6 +190,9 @@ public: class CAddrMan { private: + friend class CAddrManTest; + +protected: //! critical section to protect the inner data structures mutable CCriticalSection cs; @@ -285,9 +294,29 @@ protected: CAddrInfo GetAddressInfo_(const CService& addr); public: + // Compressed IP->ASN mapping, loaded from a file when a node starts. + // Should be always empty if no file was provided. + // This mapping is then used for bucketing nodes in Addrman. + // + // If asmap is provided, nodes will be bucketed by + // AS they belong to, in order to make impossible for a node + // to connect to several nodes hosted in a single AS. + // This is done in response to Erebus attack, but also to generally + // diversify the connections every node creates, + // especially useful when a large fraction of nodes + // operate under a couple of cloud providers. + // + // If a new asmap was provided, the existing records + // would be re-bucketed accordingly. + std::vector m_asmap; + + // Read asmap from provided binary file + static std::vector DecodeAsmap(fs::path path); + + /** * serialized format: - * * version byte (currently 1) + * * version byte (1 for pre-asmap files, 2 for files including asmap version) * * 0x20 + nKey (serialized as if it were a vector, for backward compatibility) * * nNew * * nTried @@ -319,7 +348,7 @@ public: { LOCK(cs); - unsigned char nVersion = 1; + unsigned char nVersion = 2; s << nVersion; s << ((unsigned char)32); s << nKey; @@ -362,6 +391,13 @@ public: } } } + // Store asmap version after bucket entries so that it + // can be ignored by older clients for backward compatibility. + uint256 asmap_version; + if (m_asmap.size() != 0) { + asmap_version = SerializeHash(m_asmap); + } + s << asmap_version; } template @@ -370,7 +406,6 @@ public: LOCK(cs); Clear(); - unsigned char nVersion; s >> nVersion; unsigned char nKeySize; @@ -400,16 +435,6 @@ public: mapAddr[info] = n; info.nRandomPos = vRandom.size(); vRandom.push_back(n); - if (nVersion != 1 || nUBuckets != ADDRMAN_NEW_BUCKET_COUNT) { - // In case the new table data cannot be used (nVersion unknown, or bucket count wrong), - // immediately try to give them a reference based on their primary source address. - int nUBucket = info.GetNewBucket(nKey); - int nUBucketPos = info.GetBucketPosition(nKey, true, nUBucket); - if (vvNew[nUBucket][nUBucketPos] == -1) { - vvNew[nUBucket][nUBucketPos] = n; - info.nRefCount++; - } - } } nIdCount = nNew; @@ -418,7 +443,7 @@ public: for (int n = 0; n < nTried; n++) { CAddrInfo info; s >> info; - int nKBucket = info.GetTriedBucket(nKey); + int nKBucket = info.GetTriedBucket(nKey, m_asmap); int nKBucketPos = info.GetBucketPosition(nKey, false, nKBucket); if (vvTried[nKBucket][nKBucketPos] == -1) { info.nRandomPos = vRandom.size(); @@ -434,7 +459,9 @@ public: } nTried -= nLost; - // Deserialize positions in the new table (if possible). + // Store positions in the new table buckets to apply later (if possible). + std::map entryToBucket; // Represents which entry belonged to which bucket when serializing + for (int bucket = 0; bucket < nUBuckets; bucket++) { int nSize = 0; s >> nSize; @@ -442,12 +469,38 @@ public: int nIndex = 0; s >> nIndex; if (nIndex >= 0 && nIndex < nNew) { - CAddrInfo &info = mapInfo[nIndex]; - int nUBucketPos = info.GetBucketPosition(nKey, true, bucket); - if (nVersion == 1 && nUBuckets == ADDRMAN_NEW_BUCKET_COUNT && vvNew[bucket][nUBucketPos] == -1 && info.nRefCount < ADDRMAN_NEW_BUCKETS_PER_ADDRESS) { - info.nRefCount++; - vvNew[bucket][nUBucketPos] = nIndex; - } + entryToBucket[nIndex] = bucket; + } + } + } + + uint256 supplied_asmap_version; + if (m_asmap.size() != 0) { + supplied_asmap_version = SerializeHash(m_asmap); + } + uint256 serialized_asmap_version; + if (nVersion > 1) { + s >> serialized_asmap_version; + } + + for (int n = 0; n < nNew; n++) { + CAddrInfo &info = mapInfo[n]; + int bucket = entryToBucket[n]; + int nUBucketPos = info.GetBucketPosition(nKey, true, bucket); + if (nVersion == 2 && nUBuckets == ADDRMAN_NEW_BUCKET_COUNT && vvNew[bucket][nUBucketPos] == -1 && + info.nRefCount < ADDRMAN_NEW_BUCKETS_PER_ADDRESS && serialized_asmap_version == supplied_asmap_version) { + // Bucketing has not changed, using existing bucket positions for the new table + vvNew[bucket][nUBucketPos] = n; + info.nRefCount++; + } else { + // In case the new table data cannot be used (nVersion unknown, bucket count wrong or new asmap), + // try to give them a reference based on their primary source address. + LogPrint(BCLog::ADDRMAN, "Bucketing method was updated, re-bucketing addrman entries from disk\n"); + bucket = info.GetNewBucket(nKey, m_asmap); + nUBucketPos = info.GetBucketPosition(nKey, true, bucket); + if (vvNew[bucket][nUBucketPos] == -1) { + vvNew[bucket][nUBucketPos] = n; + info.nRefCount++; } } } diff --git a/src/init.cpp b/src/init.cpp index 1338de6532..da31b5e992 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -43,6 +44,7 @@ #include #include #include +#include #include #include @@ -137,6 +139,8 @@ static CDSNotificationInterface* pdsNotificationInterface = nullptr; static const char* FEE_ESTIMATES_FILENAME="fee_estimates.dat"; +static const char* DEFAULT_ASMAP_FILENAME="ip_asn.map"; + ////////////////////////////////////////////////////////////////////////////// // // Shutdown @@ -539,6 +543,7 @@ void SetupServerArgs() gArgs.AddArg("-timeout=", strprintf("Specify connection timeout in milliseconds (minimum: 1, default: %d)", DEFAULT_CONNECT_TIMEOUT), false, OptionsCategory::CONNECTION); gArgs.AddArg("-torcontrol=:", strprintf("Tor control port to use if onion listening enabled (default: %s)", DEFAULT_TOR_CONTROL), false, OptionsCategory::CONNECTION); gArgs.AddArg("-torpassword=", "Tor control port password (default: empty)", false, OptionsCategory::CONNECTION); + gArgs.AddArg("-asmap=", "Specify asn mapping used for bucketing of the peers. Path should be relative to the -datadir path.", false, OptionsCategory::CONNECTION); #ifdef USE_UPNP #if USE_UPNP gArgs.AddArg("-upnp", "Use UPnP to map the listening port (default: 1 when listening and no -proxy)", false, OptionsCategory::CONNECTION); @@ -2453,6 +2458,25 @@ bool AppInitMain() return false; } + // Read asmap file if configured + if (gArgs.IsArgSet("-asmap")) { + std::string asmap_file = gArgs.GetArg("-asmap", ""); + if (asmap_file.empty()) { + asmap_file = DEFAULT_ASMAP_FILENAME; + } + const fs::path asmap_path = GetDataDir() / asmap_file; + std::vector asmap = CAddrMan::DecodeAsmap(asmap_path); + if (asmap.size() == 0) { + InitError(strprintf(_("Could not find or parse specified asmap: '%s'"), asmap_path)); + return false; + } + g_connman->SetAsmap(asmap); + const uint256 asmap_version = SerializeHash(asmap); + LogPrintf("Using asmap version %s for IP bucketing.\n", asmap_version.ToString()); + } else { + LogPrintf("Using /16 prefix for IP bucketing.\n"); + } + // ********************************************************* Step 13: finished SetRPCWarmupFinished(); diff --git a/src/net.cpp b/src/net.cpp index 29cde6714b..99d4dc3adc 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -762,12 +762,13 @@ std::string CNode::GetLogString() const #undef X #define X(name) stats.name = name -void CNode::copyStats(CNodeStats &stats) +void CNode::copyStats(CNodeStats &stats, std::vector &m_asmap) { stats.nodeid = this->GetId(); X(nServices); X(addr); X(addrBind); + stats.m_mapped_as = addr.GetMappedAS(m_asmap); { LOCK(cs_filter); X(fRelayTxes); @@ -2406,7 +2407,7 @@ void CConnman::ThreadOpenConnections(const std::vector connect) // but inbound and addnode peers do not use our outbound slots. Inbound peers // also have the added issue that they're attacker controlled and could be used // to prevent us from connecting to particular hosts if we used them here. - setConnected.insert(pnode->addr.GetGroup()); + setConnected.insert(pnode->addr.GetGroup(addrman.m_asmap)); nOutbound++; } } @@ -2465,7 +2466,7 @@ void CConnman::ThreadOpenConnections(const std::vector connect) bool isMasternode = dmn != nullptr; // if we selected an invalid address, restart - if (!addr.IsValid() || setConnected.count(addr.GetGroup())) + if (!addr.IsValid() || setConnected.count(addr.GetGroup(addrman.m_asmap))) break; // don't try to connect to masternodes that we already have a connection to (most likely inbound) @@ -3655,7 +3656,7 @@ void CConnman::GetNodeStats(std::vector& vstats) continue; } vstats.emplace_back(); - pnode->copyStats(vstats.back()); + pnode->copyStats(vstats.back(), addrman.m_asmap); } } @@ -4079,7 +4080,7 @@ CSipHasher CConnman::GetDeterministicRandomizer(uint64_t id) const uint64_t CConnman::CalculateKeyedNetGroup(const CAddress& ad) const { - std::vector vchNetGroup(ad.GetGroup()); + std::vector vchNetGroup(ad.GetGroup(addrman.m_asmap)); return GetDeterministicRandomizer(RANDOMIZER_ID_NETGROUP).Write(vchNetGroup.data(), vchNetGroup.size()).Finalize(); } diff --git a/src/net.h b/src/net.h index ea1a640437..9eda4d67a7 100644 --- a/src/net.h +++ b/src/net.h @@ -176,6 +176,7 @@ public: std::vector m_specified_outgoing; std::vector m_added_nodes; SocketEventsMode socketEventsMode = SOCKETEVENTS_SELECT; + std::vector m_asmap; }; void Init(const Options& connOptions) { @@ -476,6 +477,8 @@ public: */ int64_t PoissonNextSendInbound(int64_t now, int average_interval_seconds); + void SetAsmap(std::vector asmap) { addrman.m_asmap = asmap; } + private: struct ListenSocket { SOCKET socket; @@ -774,6 +777,7 @@ public: CAddress addr; // Bind address of our side of the connection CAddress addrBind; + uint32_t m_mapped_as; // In case this is a verified MN, this value is the proTx of the MN uint256 verifiedProRegTxHash; // In case this is a verified MN, this value is the hashed operator pubkey of the MN @@ -1116,7 +1120,7 @@ public: void CloseSocketDisconnect(CConnman* connman); - void copyStats(CNodeStats &stats); + void copyStats(CNodeStats &stats, std::vector &m_asmap); ServiceFlags GetLocalServices() const { diff --git a/src/netaddress.cpp b/src/netaddress.cpp index 11714e6b66..dabdd8f90c 100644 --- a/src/netaddress.cpp +++ b/src/netaddress.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include @@ -338,58 +339,104 @@ bool CNetAddr::GetIn6Addr(struct in6_addr* pipv6Addr) const return true; } -// get canonical identifier of an address' group -// no two connections will be attempted to addresses with the same group -std::vector CNetAddr::GetGroup() const +uint32_t CNetAddr::GetNetClass() const { + uint32_t net_class = NET_IPV6; + if (IsLocal()) { + net_class = 255; + } + if (IsInternal()) { + net_class = NET_INTERNAL; + } else if (!IsRoutable()) { + net_class = NET_UNROUTABLE; + } else if (IsIPv4() || IsRFC6145() || IsRFC6052() || IsRFC3964() || IsRFC4380()) { + net_class = NET_IPV4; + } else if (IsTor()) { + net_class = NET_ONION; + } + return net_class; +} + +uint32_t CNetAddr::GetMappedAS(const std::vector &asmap) const { + uint32_t net_class = GetNetClass(); + if (asmap.size() == 0 || (net_class != NET_IPV4 && net_class != NET_IPV6)) { + return 0; // Indicates not found, safe because AS0 is reserved per RFC7607. + } + std::vector ip_bits(128); + for (int8_t byte_i = 0; byte_i < 16; ++byte_i) { + uint8_t cur_byte = GetByte(15 - byte_i); + for (uint8_t bit_i = 0; bit_i < 8; ++bit_i) { + ip_bits[byte_i * 8 + bit_i] = (cur_byte >> (7 - bit_i)) & 1; + } + } + uint32_t mapped_as = Interpret(asmap, ip_bits); + return mapped_as; +} + +/** + * Get the canonical identifier of our network group + * + * The groups are assigned in a way where it should be costly for an attacker to + * obtain addresses with many different group identifiers, even if it is cheap + * to obtain addresses with the same identifier. + * + * @note No two connections will be attempted to addresses with the same network + * group. + */ +std::vector CNetAddr::GetGroup(const std::vector &asmap) const { std::vector vchRet; - int nClass = NET_IPV6; + uint32_t net_class = GetNetClass(); + // If non-empty asmap is supplied and the address is IPv4/IPv6, + // return ASN to be used for bucketing. + uint32_t asn = GetMappedAS(asmap); + if (asn != 0) { // Either asmap was empty, or address has non-asmappable net class (e.g. TOR). + vchRet.push_back(NET_IPV6); // IPv4 and IPv6 with same ASN should be in the same bucket + for (int i = 0; i < 4; i++) { + vchRet.push_back((asn >> (8 * i)) & 0xFF); + } + return vchRet; + } + + vchRet.push_back(net_class); int nStartByte = 0; int nBits = 16; // all local addresses belong to the same group if (IsLocal()) { - nClass = 255; nBits = 0; } // all internal-usage addresses get their own group if (IsInternal()) { - nClass = NET_INTERNAL; nStartByte = sizeof(g_internal_prefix); nBits = (sizeof(ip) - sizeof(g_internal_prefix)) * 8; } // all other unroutable addresses belong to the same group else if (!IsRoutable()) { - nClass = NET_UNROUTABLE; nBits = 0; } // for IPv4 addresses, '1' + the 16 higher-order bits of the IP // includes mapped IPv4, SIIT translated IPv4, and the well-known prefix else if (IsIPv4() || IsRFC6145() || IsRFC6052()) { - nClass = NET_IPV4; nStartByte = 12; } // for 6to4 tunnelled addresses, use the encapsulated IPv4 address else if (IsRFC3964()) { - nClass = NET_IPV4; nStartByte = 2; } // for Teredo-tunnelled IPv6 addresses, use the encapsulated IPv4 address else if (IsRFC4380()) { - vchRet.push_back(NET_IPV4); vchRet.push_back(GetByte(3) ^ 0xFF); vchRet.push_back(GetByte(2) ^ 0xFF); return vchRet; } else if (IsTor()) { - nClass = NET_ONION; nStartByte = 6; nBits = 4; } @@ -400,7 +447,6 @@ std::vector CNetAddr::GetGroup() const else nBits = 32; - vchRet.push_back(nClass); while (nBits >= 8) { vchRet.push_back(GetByte(15 - nStartByte)); diff --git a/src/netaddress.h b/src/netaddress.h index 46a6ec3326..1e992d44c1 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -122,7 +122,13 @@ class CNetAddr unsigned int GetByte(int n) const; uint64_t GetHash() const; bool GetInAddr(struct in_addr* pipv4Addr) const; - std::vector GetGroup() const; + uint32_t GetNetClass() const; + + // The AS on the BGP path to the node we use to diversify + // peers in AddrMan bucketing based on the AS infrastructure. + // The ip->AS mapping depends on how asmap is constructed. + uint32_t GetMappedAS(const std::vector &asmap) const; + std::vector GetGroup(const std::vector &asmap) const; std::vector GetAddrBytes() const { return {std::begin(ip), std::end(ip)}; } int GetReachabilityFrom(const CNetAddr *paddrPartner = nullptr) const; diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index 2921c21658..e2092d67f9 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -79,6 +79,7 @@ UniValue getpeerinfo(const JSONRPCRequest& request) " \"addr\":\"host:port\", (string) The IP address and port of the peer\n" " \"addrbind\":\"ip:port\", (string) Bind address of the connection to the peer\n" " \"addrlocal\":\"ip:port\", (string) Local address as reported by the peer\n" + " \"mapped_as\":\"mapped_as\", (string) The AS in the BGP route to the peer used for diversifying peer selection\n" " \"services\":\"xxxxxxxxxxxxxxxx\", (string) The services offered\n" " \"verified_proregtx_hash\": h, (hex) Only present when the peer is a masternode and succesfully\n" " authenticated via MNAUTH. In this case, this field contains the\n" @@ -144,6 +145,9 @@ UniValue getpeerinfo(const JSONRPCRequest& request) obj.pushKV("addrlocal", stats.addrLocal); if (stats.addrBind.IsValid()) obj.pushKV("addrbind", stats.addrBind.ToString()); + if (stats.m_mapped_as != 0) { + obj.pushKV("mapped_as", uint64_t(stats.m_mapped_as)); + } obj.pushKV("services", strprintf("%016x", stats.nServices)); if (!stats.verifiedProRegTxHash.IsNull()) { obj.pushKV("verified_proregtx_hash", stats.verifiedProRegTxHash.ToString()); diff --git a/src/test/addrman_tests.cpp b/src/test/addrman_tests.cpp index 10fc7df7f8..0f62b4a467 100644 --- a/src/test/addrman_tests.cpp +++ b/src/test/addrman_tests.cpp @@ -5,6 +5,8 @@ #include #include #include +#include +#include #include #include @@ -12,10 +14,13 @@ class CAddrManTest : public CAddrMan { +private: + bool deterministic; uint64_t state; public: - explicit CAddrManTest(bool makeDeterministic = true) + explicit CAddrManTest(bool makeDeterministic = true, + std::vector asmap = std::vector()) { state = 1; @@ -23,6 +28,8 @@ public: // Set addrman addr placement to be deterministic. MakeDeterministic(); } + deterministic = makeDeterministic; + m_asmap = asmap; } //! Ensure that bucket placement is always the same for testing purposes. @@ -53,6 +60,21 @@ public: CAddrMan::Delete(nId); } + // Used to test deserialization + std::pair GetBucketAndEntry(const CAddress& addr) + { + LOCK(cs); + int nId = mapAddr[addr]; + for (int bucket = 0; bucket < ADDRMAN_NEW_BUCKET_COUNT; ++bucket) { + for (int entry = 0; entry < ADDRMAN_BUCKET_SIZE; ++entry) { + if (nId == vvNew[bucket][entry]) { + return std::pair(bucket, entry); + } + } + } + return std::pair(-1, -1); + } + // Simulates connection failure so that we can test eviction of offline nodes void SimConnFail(CService& addr) { @@ -63,6 +85,16 @@ public: int64_t nLastTry = GetAdjustedTime()-61; Attempt(addr, count_failure, nLastTry); } + + void Clear() + { + CAddrMan::Clear(); + if (deterministic) { + nKey.SetNull(); + insecure_rand = FastRandomContext(true); + } + } + }; static CNetAddr ResolveIP(const char* ip) @@ -89,6 +121,18 @@ static CService ResolveService(std::string ip, int port = 0) return ResolveService(ip.c_str(), port); } +static std::vector FromBytes(const unsigned char* source, int vector_size) { + std::vector result(vector_size); + for (int byte_i = 0; byte_i < vector_size / 8; ++byte_i) { + unsigned char cur_byte = source[byte_i]; + for (int bit_i = 0; bit_i < 8; ++bit_i) { + result[byte_i * 8 + bit_i] = (cur_byte >> bit_i) & 1; + } + } + return result; +} + + BOOST_FIXTURE_TEST_SUITE(addrman_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(addrman_simple) @@ -415,6 +459,139 @@ BOOST_AUTO_TEST_CASE(addrman_getaddr) } +BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket_legacy) +{ + CAddrManTest addrman; + + CAddress addr1 = CAddress(ResolveService("250.1.1.1", 8333), NODE_NONE); + CAddress addr2 = CAddress(ResolveService("250.1.1.1", 9999), NODE_NONE); + + CNetAddr source1 = ResolveIP("250.1.1.1"); + + + CAddrInfo info1 = CAddrInfo(addr1, source1); + + uint256 nKey1 = (uint256)(CHashWriter(SER_GETHASH, 0) << 1).GetHash(); + uint256 nKey2 = (uint256)(CHashWriter(SER_GETHASH, 0) << 2).GetHash(); + + std::vector asmap; // use /16 + + BOOST_CHECK_EQUAL(info1.GetTriedBucket(nKey1, asmap), 40); + + // Test: Make sure key actually randomizes bucket placement. A fail on + // this test could be a security issue. + BOOST_CHECK(info1.GetTriedBucket(nKey1, asmap) != info1.GetTriedBucket(nKey2, asmap)); + + // Test: Two addresses with same IP but different ports can map to + // different buckets because they have different keys. + CAddrInfo info2 = CAddrInfo(addr2, source1); + + BOOST_CHECK(info1.GetKey() != info2.GetKey()); + BOOST_CHECK(info1.GetTriedBucket(nKey1, asmap) != info2.GetTriedBucket(nKey1, asmap)); + + std::set buckets; + for (int i = 0; i < 255; i++) { + CAddrInfo infoi = CAddrInfo( + CAddress(ResolveService("250.1.1." + std::to_string(i)), NODE_NONE), + ResolveIP("250.1.1." + std::to_string(i))); + int bucket = infoi.GetTriedBucket(nKey1, asmap); + buckets.insert(bucket); + } + // Test: IP addresses in the same /16 prefix should + // never get more than 8 buckets with legacy grouping + BOOST_CHECK_EQUAL(buckets.size(), 8U); + + buckets.clear(); + for (int j = 0; j < 255; j++) { + CAddrInfo infoj = CAddrInfo( + CAddress(ResolveService("250." + std::to_string(j) + ".1.1"), NODE_NONE), + ResolveIP("250." + std::to_string(j) + ".1.1")); + int bucket = infoj.GetTriedBucket(nKey1, asmap); + buckets.insert(bucket); + } + // Test: IP addresses in the different /16 prefix should map to more than + // 8 buckets with legacy grouping + BOOST_CHECK_EQUAL(buckets.size(), 160U); +} + +BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket_legacy) +{ + CAddrManTest addrman; + + CAddress addr1 = CAddress(ResolveService("250.1.2.1", 8333), NODE_NONE); + CAddress addr2 = CAddress(ResolveService("250.1.2.1", 9999), NODE_NONE); + + CNetAddr source1 = ResolveIP("250.1.2.1"); + + CAddrInfo info1 = CAddrInfo(addr1, source1); + + uint256 nKey1 = (uint256)(CHashWriter(SER_GETHASH, 0) << 1).GetHash(); + uint256 nKey2 = (uint256)(CHashWriter(SER_GETHASH, 0) << 2).GetHash(); + + std::vector asmap; // use /16 + + // Test: Make sure the buckets are what we expect + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, asmap), 786); + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, source1, asmap), 786); + + // Test: Make sure key actually randomizes bucket placement. A fail on + // this test could be a security issue. + BOOST_CHECK(info1.GetNewBucket(nKey1, asmap) != info1.GetNewBucket(nKey2, asmap)); + + // Test: Ports should not affect bucket placement in the addr + CAddrInfo info2 = CAddrInfo(addr2, source1); + BOOST_CHECK(info1.GetKey() != info2.GetKey()); + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, asmap), info2.GetNewBucket(nKey1, asmap)); + + std::set buckets; + for (int i = 0; i < 255; i++) { + CAddrInfo infoi = CAddrInfo( + CAddress(ResolveService("250.1.1." + std::to_string(i)), NODE_NONE), + ResolveIP("250.1.1." + std::to_string(i))); + int bucket = infoi.GetNewBucket(nKey1, asmap); + buckets.insert(bucket); + } + // Test: IP addresses in the same group (\16 prefix for IPv4) should + // always map to the same bucket. + BOOST_CHECK_EQUAL(buckets.size(), 1U); + + buckets.clear(); + for (int j = 0; j < 4 * 255; j++) { + CAddrInfo infoj = CAddrInfo(CAddress( + ResolveService( + std::to_string(250 + (j / 255)) + "." + std::to_string(j % 256) + ".1.1"), NODE_NONE), + ResolveIP("251.4.1.1")); + int bucket = infoj.GetNewBucket(nKey1, asmap); + buckets.insert(bucket); + } + // Test: IP addresses in the same source groups should map to NO MORE + // than 64 buckets. + BOOST_CHECK(buckets.size() <= 64); + + buckets.clear(); + for (int p = 0; p < 255; p++) { + CAddrInfo infoj = CAddrInfo( + CAddress(ResolveService("250.1.1.1"), NODE_NONE), + ResolveIP("250." + std::to_string(p) + ".1.1")); + int bucket = infoj.GetNewBucket(nKey1, asmap); + buckets.insert(bucket); + } + // Test: IP addresses in the different source groups should map to MORE + // than 64 buckets. + BOOST_CHECK(buckets.size() > 64); +} + +// The following three test cases use asmap.raw +// We use an artificial minimal mock mapping +// 250.0.0.0/8 AS1000 +// 101.1.0.0/16 AS1 +// 101.2.0.0/16 AS2 +// 101.3.0.0/16 AS3 +// 101.4.0.0/16 AS4 +// 101.5.0.0/16 AS5 +// 101.6.0.0/16 AS6 +// 101.7.0.0/16 AS7 +// 101.8.0.0/16 AS8 BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) { CAddrManTest addrman; @@ -430,43 +607,44 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket) uint256 nKey1 = (uint256)(CHashWriter(SER_GETHASH, 0) << 1).GetHash(); uint256 nKey2 = (uint256)(CHashWriter(SER_GETHASH, 0) << 2).GetHash(); + std::vector asmap = FromBytes(raw_tests::asmap, sizeof(raw_tests::asmap) * 8); - BOOST_CHECK_EQUAL(info1.GetTriedBucket(nKey1), 40); + BOOST_CHECK_EQUAL(info1.GetTriedBucket(nKey1, asmap), 236); // Test: Make sure key actually randomizes bucket placement. A fail on // this test could be a security issue. - BOOST_CHECK(info1.GetTriedBucket(nKey1) != info1.GetTriedBucket(nKey2)); + BOOST_CHECK(info1.GetTriedBucket(nKey1, asmap) != info1.GetTriedBucket(nKey2, asmap)); // Test: Two addresses with same IP but different ports can map to // different buckets because they have different keys. CAddrInfo info2 = CAddrInfo(addr2, source1); BOOST_CHECK(info1.GetKey() != info2.GetKey()); - BOOST_CHECK(info1.GetTriedBucket(nKey1) != info2.GetTriedBucket(nKey1)); + BOOST_CHECK(info1.GetTriedBucket(nKey1, asmap) != info2.GetTriedBucket(nKey1, asmap)); std::set buckets; - for (int i = 0; i < 255; i++) { - CAddrInfo infoi = CAddrInfo( - CAddress(ResolveService("250.1.1." + std::to_string(i)), NODE_NONE), - ResolveIP("250.1.1." + std::to_string(i))); - int bucket = infoi.GetTriedBucket(nKey1); + for (int j = 0; j < 255; j++) { + CAddrInfo infoj = CAddrInfo( + CAddress(ResolveService("101." + std::to_string(j) + ".1.1"), NODE_NONE), + ResolveIP("101." + std::to_string(j) + ".1.1")); + int bucket = infoj.GetTriedBucket(nKey1, asmap); buckets.insert(bucket); } - // Test: IP addresses in the same group (\16 prefix for IPv4) should - // never get more than 8 buckets - BOOST_CHECK_EQUAL(buckets.size(), 8U); + // Test: IP addresses in the different /16 prefix MAY map to more than + // 8 buckets. + BOOST_CHECK(buckets.size() > 8); buckets.clear(); for (int j = 0; j < 255; j++) { CAddrInfo infoj = CAddrInfo( CAddress(ResolveService("250." + std::to_string(j) + ".1.1"), NODE_NONE), ResolveIP("250." + std::to_string(j) + ".1.1")); - int bucket = infoj.GetTriedBucket(nKey1); + int bucket = infoj.GetTriedBucket(nKey1, asmap); buckets.insert(bucket); } - // Test: IP addresses in the different groups should map to more than - // 8 buckets. - BOOST_CHECK_EQUAL(buckets.size(), 160U); + // Test: IP addresses in the different /16 prefix MAY NOT map to more than + // 8 buckets. + BOOST_CHECK(buckets.size() == 8); } BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) @@ -483,29 +661,31 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) uint256 nKey1 = (uint256)(CHashWriter(SER_GETHASH, 0) << 1).GetHash(); uint256 nKey2 = (uint256)(CHashWriter(SER_GETHASH, 0) << 2).GetHash(); + std::vector asmap = FromBytes(raw_tests::asmap, sizeof(raw_tests::asmap) * 8); + // Test: Make sure the buckets are what we expect - BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1), 786); - BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, source1), 786); + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, asmap), 795); + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, source1, asmap), 795); // Test: Make sure key actually randomizes bucket placement. A fail on // this test could be a security issue. - BOOST_CHECK(info1.GetNewBucket(nKey1) != info1.GetNewBucket(nKey2)); + BOOST_CHECK(info1.GetNewBucket(nKey1, asmap) != info1.GetNewBucket(nKey2, asmap)); // Test: Ports should not affect bucket placement in the addr CAddrInfo info2 = CAddrInfo(addr2, source1); BOOST_CHECK(info1.GetKey() != info2.GetKey()); - BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1), info2.GetNewBucket(nKey1)); + BOOST_CHECK_EQUAL(info1.GetNewBucket(nKey1, asmap), info2.GetNewBucket(nKey1, asmap)); std::set buckets; for (int i = 0; i < 255; i++) { CAddrInfo infoi = CAddrInfo( CAddress(ResolveService("250.1.1." + std::to_string(i)), NODE_NONE), ResolveIP("250.1.1." + std::to_string(i))); - int bucket = infoi.GetNewBucket(nKey1); + int bucket = infoi.GetNewBucket(nKey1, asmap); buckets.insert(bucket); } - // Test: IP addresses in the same group (\16 prefix for IPv4) should - // always map to the same bucket. + // Test: IP addresses in the same /16 prefix + // usually map to the same bucket. BOOST_CHECK_EQUAL(buckets.size(), 1U); buckets.clear(); @@ -514,24 +694,103 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket) ResolveService( std::to_string(250 + (j / 255)) + "." + std::to_string(j % 256) + ".1.1"), NODE_NONE), ResolveIP("251.4.1.1")); - int bucket = infoj.GetNewBucket(nKey1); + int bucket = infoj.GetNewBucket(nKey1, asmap); buckets.insert(bucket); } - // Test: IP addresses in the same source groups should map to no more - // than 64 buckets. + // Test: IP addresses in the same source /16 prefix should not map to more + // than 64 buckets. BOOST_CHECK(buckets.size() <= 64); buckets.clear(); for (int p = 0; p < 255; p++) { CAddrInfo infoj = CAddrInfo( CAddress(ResolveService("250.1.1.1"), NODE_NONE), - ResolveIP("250." + std::to_string(p) + ".1.1")); - int bucket = infoj.GetNewBucket(nKey1); + ResolveIP("101." + std::to_string(p) + ".1.1")); + int bucket = infoj.GetNewBucket(nKey1, asmap); buckets.insert(bucket); } - // Test: IP addresses in the different source groups should map to more - // than 64 buckets. - BOOST_CHECK(buckets.size() > 64); + // Test: IP addresses in the different source /16 prefixes usually map to MORE + // than 1 bucket. + BOOST_CHECK(buckets.size() > 1); + + buckets.clear(); + for (int p = 0; p < 255; p++) { + CAddrInfo infoj = CAddrInfo( + CAddress(ResolveService("250.1.1.1"), NODE_NONE), + ResolveIP("250." + std::to_string(p) + ".1.1")); + int bucket = infoj.GetNewBucket(nKey1, asmap); + buckets.insert(bucket); + } + // Test: IP addresses in the different source /16 prefixes sometimes map to NO MORE + // than 1 bucket. + BOOST_CHECK(buckets.size() == 1); + +} + +BOOST_AUTO_TEST_CASE(addrman_serialization) +{ + std::vector asmap1 = FromBytes(raw_tests::asmap, sizeof(raw_tests::asmap) * 8); + + CAddrManTest addrman_asmap1(true, asmap1); + CAddrManTest addrman_asmap1_dup(true, asmap1); + CAddrManTest addrman_noasmap; + CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); + + CAddress addr = CAddress(ResolveService("250.1.1.1"), NODE_NONE); + CNetAddr default_source; + + + addrman_asmap1.Add(addr, default_source); + + stream << addrman_asmap1; + // serizalizing/deserializing addrman with the same asmap + stream >> addrman_asmap1_dup; + + std::pair bucketAndEntry_asmap1 = addrman_asmap1.GetBucketAndEntry(addr); + std::pair bucketAndEntry_asmap1_dup = addrman_asmap1_dup.GetBucketAndEntry(addr); + BOOST_CHECK(bucketAndEntry_asmap1.second != -1); + BOOST_CHECK(bucketAndEntry_asmap1_dup.second != -1); + + BOOST_CHECK(bucketAndEntry_asmap1.first == bucketAndEntry_asmap1_dup.first); + BOOST_CHECK(bucketAndEntry_asmap1.second == bucketAndEntry_asmap1_dup.second); + + // deserializing asmaped peers.dat to non-asmaped addrman + stream << addrman_asmap1; + stream >> addrman_noasmap; + std::pair bucketAndEntry_noasmap = addrman_noasmap.GetBucketAndEntry(addr); + BOOST_CHECK(bucketAndEntry_noasmap.second != -1); + BOOST_CHECK(bucketAndEntry_asmap1.first != bucketAndEntry_noasmap.first); + BOOST_CHECK(bucketAndEntry_asmap1.second != bucketAndEntry_noasmap.second); + + // deserializing non-asmaped peers.dat to asmaped addrman + addrman_asmap1.Clear(); + addrman_noasmap.Clear(); + addrman_noasmap.Add(addr, default_source); + stream << addrman_noasmap; + stream >> addrman_asmap1; + std::pair bucketAndEntry_asmap1_deser = addrman_asmap1.GetBucketAndEntry(addr); + BOOST_CHECK(bucketAndEntry_asmap1_deser.second != -1); + BOOST_CHECK(bucketAndEntry_asmap1_deser.first != bucketAndEntry_noasmap.first); + BOOST_CHECK(bucketAndEntry_asmap1_deser.first == bucketAndEntry_asmap1_dup.first); + BOOST_CHECK(bucketAndEntry_asmap1_deser.second == bucketAndEntry_asmap1_dup.second); + + // used to map to different buckets, now maps to the same bucket. + addrman_asmap1.Clear(); + addrman_noasmap.Clear(); + CAddress addr1 = CAddress(ResolveService("250.1.1.1"), NODE_NONE); + CAddress addr2 = CAddress(ResolveService("250.2.1.1"), NODE_NONE); + addrman_noasmap.Add(addr, default_source); + addrman_noasmap.Add(addr2, default_source); + std::pair bucketAndEntry_noasmap_addr1 = addrman_noasmap.GetBucketAndEntry(addr1); + std::pair bucketAndEntry_noasmap_addr2 = addrman_noasmap.GetBucketAndEntry(addr2); + BOOST_CHECK(bucketAndEntry_noasmap_addr1.first != bucketAndEntry_noasmap_addr2.first); + BOOST_CHECK(bucketAndEntry_noasmap_addr1.second != bucketAndEntry_noasmap_addr2.second); + stream << addrman_noasmap; + stream >> addrman_asmap1; + std::pair bucketAndEntry_asmap1_deser_addr1 = addrman_asmap1.GetBucketAndEntry(addr1); + std::pair bucketAndEntry_asmap1_deser_addr2 = addrman_asmap1.GetBucketAndEntry(addr2); + BOOST_CHECK(bucketAndEntry_asmap1_deser_addr1.first == bucketAndEntry_asmap1_deser_addr2.first); + BOOST_CHECK(bucketAndEntry_asmap1_deser_addr1.second != bucketAndEntry_asmap1_deser_addr2.second); } diff --git a/src/test/data/asmap.raw b/src/test/data/asmap.raw new file mode 100644 index 0000000000000000000000000000000000000000..3dcf1f3940890af14f613b08aa11387fd002c228 GIT binary patch literal 59 zcmey({Dyyn{egcBzfl?P2TK5m Cn;6Xi literal 0 HcmV?d00001 diff --git a/src/test/netbase_tests.cpp b/src/test/netbase_tests.cpp index 5505f4696d..f4de458ecd 100644 --- a/src/test/netbase_tests.cpp +++ b/src/test/netbase_tests.cpp @@ -297,23 +297,23 @@ BOOST_AUTO_TEST_CASE(subnet_test) BOOST_AUTO_TEST_CASE(netbase_getgroup) { - - BOOST_CHECK(ResolveIP("127.0.0.1").GetGroup() == std::vector({0})); // Local -> !Routable() - BOOST_CHECK(ResolveIP("257.0.0.1").GetGroup() == std::vector({0})); // !Valid -> !Routable() - BOOST_CHECK(ResolveIP("10.0.0.1").GetGroup() == std::vector({0})); // RFC1918 -> !Routable() - BOOST_CHECK(ResolveIP("169.254.1.1").GetGroup() == std::vector({0})); // RFC3927 -> !Routable() - BOOST_CHECK(ResolveIP("1.2.3.4").GetGroup() == std::vector({(unsigned char)NET_IPV4, 1, 2})); // IPv4 - BOOST_CHECK(ResolveIP("::FFFF:0:102:304").GetGroup() == std::vector({(unsigned char)NET_IPV4, 1, 2})); // RFC6145 - BOOST_CHECK(ResolveIP("64:FF9B::102:304").GetGroup() == std::vector({(unsigned char)NET_IPV4, 1, 2})); // RFC6052 - BOOST_CHECK(ResolveIP("2002:102:304:9999:9999:9999:9999:9999").GetGroup() == std::vector({(unsigned char)NET_IPV4, 1, 2})); // RFC3964 - BOOST_CHECK(ResolveIP("2001:0:9999:9999:9999:9999:FEFD:FCFB").GetGroup() == std::vector({(unsigned char)NET_IPV4, 1, 2})); // RFC4380 - BOOST_CHECK(ResolveIP("FD87:D87E:EB43:edb1:8e4:3588:e546:35ca").GetGroup() == std::vector({(unsigned char)NET_ONION, 239})); // Tor - BOOST_CHECK(ResolveIP("2001:470:abcd:9999:9999:9999:9999:9999").GetGroup() == std::vector({(unsigned char)NET_IPV6, 32, 1, 4, 112, 175})); //he.net - BOOST_CHECK(ResolveIP("2001:2001:9999:9999:9999:9999:9999:9999").GetGroup() == std::vector({(unsigned char)NET_IPV6, 32, 1, 32, 1})); //IPv6 + std::vector asmap; // use /16 + BOOST_CHECK(ResolveIP("127.0.0.1").GetGroup(asmap) == std::vector({0})); // Local -> !Routable() + BOOST_CHECK(ResolveIP("257.0.0.1").GetGroup(asmap) == std::vector({0})); // !Valid -> !Routable() + BOOST_CHECK(ResolveIP("10.0.0.1").GetGroup(asmap) == std::vector({0})); // RFC1918 -> !Routable() + BOOST_CHECK(ResolveIP("169.254.1.1").GetGroup(asmap) == std::vector({0})); // RFC3927 -> !Routable() + BOOST_CHECK(ResolveIP("1.2.3.4").GetGroup(asmap) == std::vector({(unsigned char)NET_IPV4, 1, 2})); // IPv4 + BOOST_CHECK(ResolveIP("::FFFF:0:102:304").GetGroup(asmap) == std::vector({(unsigned char)NET_IPV4, 1, 2})); // RFC6145 + BOOST_CHECK(ResolveIP("64:FF9B::102:304").GetGroup(asmap) == std::vector({(unsigned char)NET_IPV4, 1, 2})); // RFC6052 + BOOST_CHECK(ResolveIP("2002:102:304:9999:9999:9999:9999:9999").GetGroup(asmap) == std::vector({(unsigned char)NET_IPV4, 1, 2})); // RFC3964 + BOOST_CHECK(ResolveIP("2001:0:9999:9999:9999:9999:FEFD:FCFB").GetGroup(asmap) == std::vector({(unsigned char)NET_IPV4, 1, 2})); // RFC4380 + BOOST_CHECK(ResolveIP("FD87:D87E:EB43:edb1:8e4:3588:e546:35ca").GetGroup(asmap) == std::vector({(unsigned char)NET_ONION, 239})); // Tor + BOOST_CHECK(ResolveIP("2001:470:abcd:9999:9999:9999:9999:9999").GetGroup(asmap) == std::vector({(unsigned char)NET_IPV6, 32, 1, 4, 112, 175})); //he.net + BOOST_CHECK(ResolveIP("2001:2001:9999:9999:9999:9999:9999:9999").GetGroup(asmap) == std::vector({(unsigned char)NET_IPV6, 32, 1, 32, 1})); //IPv6 // baz.net sha256 hash: 12929400eb4607c4ac075f087167e75286b179c693eb059a01774b864e8fe505 std::vector internal_group = {NET_INTERNAL, 0x12, 0x92, 0x94, 0x00, 0xeb, 0x46, 0x07, 0xc4, 0xac, 0x07}; - BOOST_CHECK(CreateInternal("baz.net").GetGroup() == internal_group); + BOOST_CHECK(CreateInternal("baz.net").GetGroup(asmap) == internal_group); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/utilasmap.cpp b/src/utilasmap.cpp new file mode 100644 index 0000000000..ac230e9ee5 --- /dev/null +++ b/src/utilasmap.cpp @@ -0,0 +1,97 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include +#include +#include + +namespace { + +uint32_t DecodeBits(std::vector::const_iterator& bitpos, uint8_t minval, const std::vector &bit_sizes) +{ + uint32_t val = minval; + bool bit; + for (std::vector::const_iterator bit_sizes_it = bit_sizes.begin(); + bit_sizes_it != bit_sizes.end(); ++bit_sizes_it) { + if (bit_sizes_it + 1 != bit_sizes.end()) { + bit = *bitpos; + bitpos++; + } else { + bit = 0; + } + if (bit) { + val += (1 << *bit_sizes_it); + } else { + for (int b = 0; b < *bit_sizes_it; b++) { + bit = *bitpos; + bitpos++; + val += bit << (*bit_sizes_it - 1 - b); + } + return val; + } + } + return -1; +} + +const std::vector TYPE_BIT_SIZES{0, 0, 1}; +uint32_t DecodeType(std::vector::const_iterator& bitpos) +{ + return DecodeBits(bitpos, 0, TYPE_BIT_SIZES); +} + +const std::vector ASN_BIT_SIZES{15, 16, 17, 18, 19, 20, 21, 22, 23, 24}; +uint32_t DecodeASN(std::vector::const_iterator& bitpos) +{ + return DecodeBits(bitpos, 1, ASN_BIT_SIZES); +} + + +const std::vector MATCH_BIT_SIZES{1, 2, 3, 4, 5, 6, 7, 8}; +uint32_t DecodeMatch(std::vector::const_iterator& bitpos) +{ + return DecodeBits(bitpos, 2, MATCH_BIT_SIZES); +} + + +const std::vector JUMP_BIT_SIZES{5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30}; +uint32_t DecodeJump(std::vector::const_iterator& bitpos) +{ + return DecodeBits(bitpos, 17, JUMP_BIT_SIZES); +} + +} + +uint32_t Interpret(const std::vector &asmap, const std::vector &ip) +{ + std::vector::const_iterator pos = asmap.begin(); + uint8_t bits = ip.size(); + uint8_t default_asn = 0; + uint32_t opcode, jump, match, matchlen; + while (1) { + assert(pos != asmap.end()); + opcode = DecodeType(pos); + if (opcode == 0) { + return DecodeASN(pos); + } else if (opcode == 1) { + jump = DecodeJump(pos); + if (ip[ip.size() - bits]) { + pos += jump; + } + bits--; + } else if (opcode == 2) { + match = DecodeMatch(pos); + matchlen = CountBits(match) - 1; + for (uint32_t bit = 0; bit < matchlen; bit++) { + if ((ip[ip.size() - bits]) != ((match >> (matchlen - 1 - bit)) & 1)) { + return default_asn; + } + bits--; + } + } else if (opcode == 3) { + default_asn = DecodeASN(pos); + } else { + assert(0); + } + } +} diff --git a/src/utilasmap.h b/src/utilasmap.h new file mode 100644 index 0000000000..2840f28069 --- /dev/null +++ b/src/utilasmap.h @@ -0,0 +1,10 @@ +// Copyright (c) 2019 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTILASMAP_H +#define BITCOIN_UTILASMAP_H + +uint32_t Interpret(const std::vector &asmap, const std::vector &ip); + +#endif // BITCOIN_UTILASMAP_H From 401098283ebfca88aabff9052410a0b8c0bf6635 Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <6098974-kittywhiskers@users.noreply.gitlab.com> Date: Tue, 18 May 2021 22:00:48 +0530 Subject: [PATCH 3/4] Merge #17812: asmap feature refinements and functional tests --- src/addrman.cpp | 12 ++-- src/addrman.h | 11 ++-- src/init.cpp | 46 ++++++++------ src/netaddress.cpp | 10 ++- src/netaddress.h | 5 +- test/functional/feature_asmap.py | 106 +++++++++++++++++++++++++++++++ test/functional/test_runner.py | 1 + 7 files changed, 154 insertions(+), 37 deletions(-) create mode 100755 test/functional/feature_asmap.py diff --git a/src/addrman.cpp b/src/addrman.cpp index a04265c382..b1b53b3698 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -6,9 +6,9 @@ #include #include -#include -#include #include +#include +#include int CAddrInfo::GetTriedBucket(const uint256& nKey, const std::vector &asmap) const { @@ -16,7 +16,7 @@ int CAddrInfo::GetTriedBucket(const uint256& nKey, const std::vector &asma uint64_t hash2 = (CHashWriter(SER_GETHASH, 0) << nKey << GetGroup(asmap) << (hash1 % ADDRMAN_TRIED_BUCKETS_PER_GROUP)).GetHash().GetCheapHash(); int tried_bucket = hash2 % ADDRMAN_TRIED_BUCKET_COUNT; uint32_t mapped_as = GetMappedAS(asmap); - LogPrint(BCLog::NET, "IP %s mapped to AS%i belongs to tried bucket %i.\n", ToStringIP(), mapped_as, tried_bucket); + LogPrint(BCLog::NET, "IP %s mapped to AS%i belongs to tried bucket %i\n", ToStringIP(), mapped_as, tried_bucket); return tried_bucket; } @@ -27,7 +27,7 @@ int CAddrInfo::GetNewBucket(const uint256& nKey, const CNetAddr& src, const std: uint64_t hash2 = (CHashWriter(SER_GETHASH, 0) << nKey << vchSourceGroupKey << (hash1 % ADDRMAN_NEW_BUCKETS_PER_SOURCE_GROUP)).GetHash().GetCheapHash(); int new_bucket = hash2 % ADDRMAN_NEW_BUCKET_COUNT; uint32_t mapped_as = GetMappedAS(asmap); - LogPrint(BCLog::NET, "IP %s mapped to AS%i belongs to new bucket %i.\n", ToStringIP(), mapped_as, new_bucket); + LogPrint(BCLog::NET, "IP %s mapped to AS%i belongs to new bucket %i\n", ToStringIP(), mapped_as, new_bucket); return new_bucket; } @@ -658,12 +658,12 @@ std::vector CAddrMan::DecodeAsmap(fs::path path) FILE *filestr = fsbridge::fopen(path, "rb"); CAutoFile file(filestr, SER_DISK, CLIENT_VERSION); if (file.IsNull()) { - LogPrintf("Failed to open asmap file from disk.\n"); + LogPrintf("Failed to open asmap file from disk\n"); return bits; } fseek(filestr, 0, SEEK_END); int length = ftell(filestr); - LogPrintf("Opened asmap file %s (%d bytes) from disk.\n", path, length); + LogPrintf("Opened asmap file %s (%d bytes) from disk\n", path, length); fseek(filestr, 0, SEEK_SET); char cur_byte; for (int i = 0; i < length; ++i) { diff --git a/src/addrman.h b/src/addrman.h index 0698e1c555..9ea6441c06 100644 --- a/src/addrman.h +++ b/src/addrman.h @@ -6,23 +6,22 @@ #ifndef BITCOIN_ADDRMAN_H #define BITCOIN_ADDRMAN_H +#include #include #include #include #include #include #include -#include +#include +#include +#include #include #include #include -#include -#include #include -#include -#include - +#include /** * Extended statistics about a CAddress diff --git a/src/init.cpp b/src/init.cpp index da31b5e992..36fa28c42f 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -511,6 +511,7 @@ void SetupServerArgs() gArgs.AddArg("-timestampindex", strprintf("Maintain a timestamp index for block hashes, used to query blocks hashes by a range of timestamps (default: %u)", DEFAULT_TIMESTAMPINDEX), false, OptionsCategory::INDEXING); gArgs.AddArg("-txindex", strprintf("Maintain a full transaction index, used by the getrawtransaction rpc call (default: %u)", DEFAULT_TXINDEX), false, OptionsCategory::INDEXING); + gArgs.AddArg("-asmap=", strprintf("Specify asn mapping used for bucketing of the peers (default: %s). Relative paths will be prefixed by the net-specific datadir location.", DEFAULT_ASMAP_FILENAME), false, OptionsCategory::CONNECTION); gArgs.AddArg("-addnode=", "Add a node to connect to and attempt to keep the connection open (see the `addnode` RPC command help for more info). This option can be specified multiple times to add multiple nodes.", false, OptionsCategory::CONNECTION); gArgs.AddArg("-allowprivatenet", strprintf("Allow RFC1918 addresses to be relayed and connected to (default: %u)", DEFAULT_ALLOWPRIVATENET), false, OptionsCategory::CONNECTION); gArgs.AddArg("-banscore=", strprintf("Threshold for disconnecting misbehaving peers (default: %u)", DEFAULT_BANSCORE_THRESHOLD), false, OptionsCategory::CONNECTION); @@ -543,7 +544,6 @@ void SetupServerArgs() gArgs.AddArg("-timeout=", strprintf("Specify connection timeout in milliseconds (minimum: 1, default: %d)", DEFAULT_CONNECT_TIMEOUT), false, OptionsCategory::CONNECTION); gArgs.AddArg("-torcontrol=:", strprintf("Tor control port to use if onion listening enabled (default: %s)", DEFAULT_TOR_CONTROL), false, OptionsCategory::CONNECTION); gArgs.AddArg("-torpassword=", "Tor control port password (default: empty)", false, OptionsCategory::CONNECTION); - gArgs.AddArg("-asmap=", "Specify asn mapping used for bucketing of the peers. Path should be relative to the -datadir path.", false, OptionsCategory::CONNECTION); #ifdef USE_UPNP #if USE_UPNP gArgs.AddArg("-upnp", "Use UPnP to map the listening port (default: 1 when listening and no -proxy)", false, OptionsCategory::CONNECTION); @@ -1901,6 +1901,31 @@ bool AppInitMain() return InitError(ResolveErrMsg("externalip", strAddr)); } + // Read asmap file if configured + if (gArgs.IsArgSet("-asmap")) { + fs::path asmap_path = fs::path(gArgs.GetArg("-asmap", "")); + if (asmap_path.empty()) { + asmap_path = DEFAULT_ASMAP_FILENAME; + } + if (!asmap_path.is_absolute()) { + asmap_path = GetDataDir() / asmap_path; + } + if (!fs::exists(asmap_path)) { + InitError(strprintf(_("Could not find asmap file %s"), asmap_path)); + return false; + } + std::vector asmap = CAddrMan::DecodeAsmap(asmap_path); + if (asmap.size() == 0) { + InitError(strprintf(_("Could not parse asmap file %s"), asmap_path)); + return false; + } + const uint256 asmap_version = SerializeHash(asmap); + g_connman->SetAsmap(asmap); + LogPrintf("Using asmap version %s for IP bucketing\n", asmap_version.ToString()); + } else { + LogPrintf("Using /16 prefix for IP bucketing\n"); + } + #if ENABLE_ZMQ g_zmq_notification_interface = CZMQNotificationInterface::Create(); @@ -2458,25 +2483,6 @@ bool AppInitMain() return false; } - // Read asmap file if configured - if (gArgs.IsArgSet("-asmap")) { - std::string asmap_file = gArgs.GetArg("-asmap", ""); - if (asmap_file.empty()) { - asmap_file = DEFAULT_ASMAP_FILENAME; - } - const fs::path asmap_path = GetDataDir() / asmap_file; - std::vector asmap = CAddrMan::DecodeAsmap(asmap_path); - if (asmap.size() == 0) { - InitError(strprintf(_("Could not find or parse specified asmap: '%s'"), asmap_path)); - return false; - } - g_connman->SetAsmap(asmap); - const uint256 asmap_version = SerializeHash(asmap); - LogPrintf("Using asmap version %s for IP bucketing.\n", asmap_version.ToString()); - } else { - LogPrintf("Using /16 prefix for IP bucketing.\n"); - } - // ********************************************************* Step 13: finished SetRPCWarmupFinished(); diff --git a/src/netaddress.cpp b/src/netaddress.cpp index dabdd8f90c..0861cb2487 100644 --- a/src/netaddress.cpp +++ b/src/netaddress.cpp @@ -202,6 +202,11 @@ bool CNetAddr::IsRFC7343() const bool CNetAddr::IsTor() const { return m_net == NET_ONION; } +bool CNetAddr::IsHeNet() const +{ + return (GetByte(15) == 0x20 && GetByte(14) == 0x01 && GetByte(13) == 0x04 && GetByte(12) == 0x70); +} + bool CNetAddr::IsLocal() const { // IPv4 loopback @@ -439,9 +444,8 @@ std::vector CNetAddr::GetGroup(const std::vector &asmap) co { nStartByte = 6; nBits = 4; - } - // for he.net, use /36 groups - else if (GetByte(15) == 0x20 && GetByte(14) == 0x01 && GetByte(13) == 0x04 && GetByte(12) == 0x70) + } else if (IsHeNet()) { + // for he.net, use /36 groups nBits = 36; // for the rest of the IPv6 network, use /32 groups else diff --git a/src/netaddress.h b/src/netaddress.h index 1e992d44c1..f24c1768d0 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -109,8 +109,9 @@ class CNetAddr bool IsRFC4843() const; // IPv6 ORCHID (deprecated) (2001:10::/28) bool IsRFC7343() const; // IPv6 ORCHIDv2 (2001:20::/28) bool IsRFC4862() const; // IPv6 autoconfig (FE80::/64) - bool IsRFC6052() const; // IPv6 well-known prefix (64:FF9B::/96) - bool IsRFC6145() const; // IPv6 IPv4-translated address (::FFFF:0:0:0/96) + bool IsRFC6052() const; // IPv6 well-known prefix for IPv4-embedded address (64:FF9B::/96) + bool IsRFC6145() const; // IPv6 IPv4-translated address (::FFFF:0:0:0/96) (actually defined in RFC2765) + bool IsHeNet() const; // IPv6 Hurricane Electric - https://he.net (2001:0470::/36) bool IsTor() const; bool IsLocal() const; bool IsRoutable() const; diff --git a/test/functional/feature_asmap.py b/test/functional/feature_asmap.py new file mode 100755 index 0000000000..60c4dd1632 --- /dev/null +++ b/test/functional/feature_asmap.py @@ -0,0 +1,106 @@ +#!/usr/bin/env python3 +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +"""Test asmap config argument for ASN-based IP bucketing. + +Verify node behaviour and debug log when launching dashd in these cases: + +1. `dashd` with no -asmap arg, using /16 prefix for IP bucketing + +2. `dashd -asmap=`, using the unit test skeleton asmap + +3. `dashd -asmap=`, using the unit test skeleton asmap + +4. `dashd -asmap/-asmap=` with no file specified, using the default asmap + +5. `dashd -asmap` with no file specified and a missing default asmap file + +6. `dashd -asmap` with an empty (unparsable) default asmap file + +The tests are order-independent. + +""" +import os +import shutil + +from test_framework.test_framework import BitcoinTestFramework + +DEFAULT_ASMAP_FILENAME = 'ip_asn.map' # defined in src/init.cpp +ASMAP = '../../src/test/data/asmap.raw' # path to unit test skeleton asmap +VERSION = 'fec61fa21a9f46f3b17bdcd660d7f4cd90b966aad3aec593c99b35f0aca15853' + +def expected_messages(filename): + return ['Opened asmap file "{}" (59 bytes) from disk'.format(filename), + 'Using asmap version {} for IP bucketing'.format(VERSION)] + +class AsmapTest(BitcoinTestFramework): + def set_test_params(self): + self.setup_clean_chain = False + self.num_nodes = 1 + + def test_without_asmap_arg(self): + self.log.info('Test dashd with no -asmap arg passed') + self.stop_node(0) + with self.node.assert_debug_log(['Using /16 prefix for IP bucketing']): + self.start_node(0) + + def test_asmap_with_absolute_path(self): + self.log.info('Test dashd -asmap=') + self.stop_node(0) + filename = os.path.join(self.datadir, 'my-map-file.map') + shutil.copyfile(self.asmap_raw, filename) + with self.node.assert_debug_log(expected_messages(filename)): + self.start_node(0, ['-asmap={}'.format(filename)]) + os.remove(filename) + + def test_asmap_with_relative_path(self): + self.log.info('Test dashd -asmap=') + self.stop_node(0) + name = 'ASN_map' + filename = os.path.join(self.datadir, name) + shutil.copyfile(self.asmap_raw, filename) + with self.node.assert_debug_log(expected_messages(filename)): + self.start_node(0, ['-asmap={}'.format(name)]) + os.remove(filename) + + def test_default_asmap(self): + shutil.copyfile(self.asmap_raw, self.default_asmap) + for arg in ['-asmap', '-asmap=']: + self.log.info('Test dashd {} (using default map file)'.format(arg)) + self.stop_node(0) + with self.node.assert_debug_log(expected_messages(self.default_asmap)): + self.start_node(0, [arg]) + os.remove(self.default_asmap) + + def test_default_asmap_with_missing_file(self): + self.log.info('Test dashd -asmap with missing default map file') + self.stop_node(0) + msg = "Error: Could not find asmap file \"{}\"".format(self.default_asmap) + self.node.assert_start_raises_init_error(extra_args=['-asmap'], expected_msg=msg) + + def test_empty_asmap(self): + self.log.info('Test dashd -asmap with empty map file') + self.stop_node(0) + with open(self.default_asmap, "w", encoding="utf-8") as f: + f.write("") + msg = "Error: Could not parse asmap file \"{}\"".format(self.default_asmap) + self.node.assert_start_raises_init_error(extra_args=['-asmap'], expected_msg=msg) + os.remove(self.default_asmap) + + def run_test(self): + self.node = self.nodes[0] + self.datadir = os.path.join(self.node.datadir, self.chain) + self.default_asmap = os.path.join(self.datadir, DEFAULT_ASMAP_FILENAME) + self.asmap_raw = os.path.join(os.path.dirname(os.path.realpath(__file__)), ASMAP) + + self.test_without_asmap_arg() + self.test_asmap_with_absolute_path() + self.test_asmap_with_relative_path() + self.test_default_asmap() + self.test_default_asmap_with_missing_file() + self.test_empty_asmap() + + +if __name__ == '__main__': + AsmapTest().main() diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 7106bbbee8..e5c1c0b022 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -168,6 +168,7 @@ BASE_SCRIPTS= [ 'feature_dip0020_activation.py', 'feature_uacomment.py', 'p2p_unrequested_blocks.py', + 'feature_asmap.py', 'feature_logging.py', 'p2p_node_network_limited.py', 'feature_blocksdir.py', From f3819c4eefa79cb0e98cb37d5d1394c674d02033 Mon Sep 17 00:00:00 2001 From: Kittywhiskers Van Gogh <6098974-kittywhiskers@users.noreply.gitlab.com> Date: Wed, 29 Jan 2020 13:55:40 -0800 Subject: [PATCH 4/4] Merge #18023: Fix some asmap issues --- src/init.cpp | 2 +- src/net.cpp | 2 +- src/net.h | 4 +- src/netaddress.cpp | 92 +++++++++++++++++++++++++++------------------- src/netaddress.h | 5 +++ src/utilasmap.cpp | 42 ++++++++++++--------- 6 files changed, 87 insertions(+), 60 deletions(-) diff --git a/src/init.cpp b/src/init.cpp index 36fa28c42f..f2bc170e99 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -1920,7 +1920,7 @@ bool AppInitMain() return false; } const uint256 asmap_version = SerializeHash(asmap); - g_connman->SetAsmap(asmap); + g_connman->SetAsmap(std::move(asmap)); LogPrintf("Using asmap version %s for IP bucketing\n", asmap_version.ToString()); } else { LogPrintf("Using /16 prefix for IP bucketing\n"); diff --git a/src/net.cpp b/src/net.cpp index 99d4dc3adc..037cbec4f2 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -762,7 +762,7 @@ std::string CNode::GetLogString() const #undef X #define X(name) stats.name = name -void CNode::copyStats(CNodeStats &stats, std::vector &m_asmap) +void CNode::copyStats(CNodeStats &stats, const std::vector &m_asmap) { stats.nodeid = this->GetId(); X(nServices); diff --git a/src/net.h b/src/net.h index 9eda4d67a7..1569ba5737 100644 --- a/src/net.h +++ b/src/net.h @@ -477,7 +477,7 @@ public: */ int64_t PoissonNextSendInbound(int64_t now, int average_interval_seconds); - void SetAsmap(std::vector asmap) { addrman.m_asmap = asmap; } + void SetAsmap(std::vector asmap) { addrman.m_asmap = std::move(asmap); } private: struct ListenSocket { @@ -1120,7 +1120,7 @@ public: void CloseSocketDisconnect(CConnman* connman); - void copyStats(CNodeStats &stats, std::vector &m_asmap); + void copyStats(CNodeStats &stats, const std::vector &m_asmap); ServiceFlags GetLocalServices() const { diff --git a/src/netaddress.cpp b/src/netaddress.cpp index 0861cb2487..75ae678eb0 100644 --- a/src/netaddress.cpp +++ b/src/netaddress.cpp @@ -344,6 +344,26 @@ bool CNetAddr::GetIn6Addr(struct in6_addr* pipv6Addr) const return true; } +bool CNetAddr::HasLinkedIPv4() const +{ + return IsRoutable() && (IsIPv4() || IsRFC6145() || IsRFC6052() || IsRFC3964() || IsRFC4380()); +} + +uint32_t CNetAddr::GetLinkedIPv4() const +{ + if (IsIPv4() || IsRFC6145() || IsRFC6052()) { + // IPv4, mapped IPv4, SIIT translated IPv4: the IPv4 address is the last 4 bytes of the address + return ReadBE32(ip + 12); + } else if (IsRFC3964()) { + // 6to4 tunneled IPv4: the IPv4 address is in bytes 2-6 + return ReadBE32(ip + 2); + } else if (IsRFC4380()) { + // Teredo tunneled IPv4: the IPv4 address is in the last 4 bytes of the address, but bitflipped + return ~ReadBE32(ip + 12); + } + assert(false); +} + uint32_t CNetAddr::GetNetClass() const { uint32_t net_class = NET_IPV6; if (IsLocal()) { @@ -353,7 +373,7 @@ uint32_t CNetAddr::GetNetClass() const { net_class = NET_INTERNAL; } else if (!IsRoutable()) { net_class = NET_UNROUTABLE; - } else if (IsIPv4() || IsRFC6145() || IsRFC6052() || IsRFC3964() || IsRFC4380()) { + } else if (HasLinkedIPv4()) { net_class = NET_IPV4; } else if (IsTor()) { net_class = NET_ONION; @@ -367,10 +387,24 @@ uint32_t CNetAddr::GetMappedAS(const std::vector &asmap) const { return 0; // Indicates not found, safe because AS0 is reserved per RFC7607. } std::vector ip_bits(128); - for (int8_t byte_i = 0; byte_i < 16; ++byte_i) { - uint8_t cur_byte = GetByte(15 - byte_i); - for (uint8_t bit_i = 0; bit_i < 8; ++bit_i) { - ip_bits[byte_i * 8 + bit_i] = (cur_byte >> (7 - bit_i)) & 1; + if (HasLinkedIPv4()) { + // For lookup, treat as if it was just an IPv4 address (pchIPv4 prefix + IPv4 bits) + for (int8_t byte_i = 0; byte_i < 12; ++byte_i) { + for (uint8_t bit_i = 0; bit_i < 8; ++bit_i) { + ip_bits[byte_i * 8 + bit_i] = (pchIPv4[byte_i] >> (7 - bit_i)) & 1; + } + } + uint32_t ipv4 = GetLinkedIPv4(); + for (int i = 0; i < 32; ++i) { + ip_bits[96 + i] = (ipv4 >> (31 - i)) & 1; + } + } else { + // Use all 128 bits of the IPv6 address otherwise + for (int8_t byte_i = 0; byte_i < 16; ++byte_i) { + uint8_t cur_byte = GetByte(15 - byte_i); + for (uint8_t bit_i = 0; bit_i < 8; ++bit_i) { + ip_bits[byte_i * 8 + bit_i] = (cur_byte >> (7 - bit_i)) & 1; + } } } uint32_t mapped_as = Interpret(asmap, ip_bits); @@ -406,50 +440,32 @@ std::vector CNetAddr::GetGroup(const std::vector &asmap) co int nStartByte = 0; int nBits = 16; - // all local addresses belong to the same group - if (IsLocal()) - { + if (IsLocal()) { + // all local addresses belong to the same group nBits = 0; - } - // all internal-usage addresses get their own group - if (IsInternal()) - { + } else if (IsInternal()) { + // all internal-usage addresses get their own group nStartByte = sizeof(g_internal_prefix); nBits = (sizeof(ip) - sizeof(g_internal_prefix)) * 8; - } - // all other unroutable addresses belong to the same group - else if (!IsRoutable()) - { + } else if (!IsRoutable()) { + // all other unroutable addresses belong to the same group nBits = 0; - } - // for IPv4 addresses, '1' + the 16 higher-order bits of the IP - // includes mapped IPv4, SIIT translated IPv4, and the well-known prefix - else if (IsIPv4() || IsRFC6145() || IsRFC6052()) - { - nStartByte = 12; - } - // for 6to4 tunnelled addresses, use the encapsulated IPv4 address - else if (IsRFC3964()) - { - nStartByte = 2; - } - // for Teredo-tunnelled IPv6 addresses, use the encapsulated IPv4 address - else if (IsRFC4380()) - { - vchRet.push_back(GetByte(3) ^ 0xFF); - vchRet.push_back(GetByte(2) ^ 0xFF); + } else if (HasLinkedIPv4()) { + // IPv4 addresses (and mapped IPv4 addresses) use /16 groups + uint32_t ipv4 = GetLinkedIPv4(); + vchRet.push_back((ipv4 >> 24) & 0xFF); + vchRet.push_back((ipv4 >> 16) & 0xFF); return vchRet; - } - else if (IsTor()) - { + } else if (IsTor()) { nStartByte = 6; nBits = 4; } else if (IsHeNet()) { // for he.net, use /36 groups nBits = 36; - // for the rest of the IPv6 network, use /32 groups - else + } else { + // for the rest of the IPv6 network, use /32 groups nBits = 32; + } while (nBits >= 8) { diff --git a/src/netaddress.h b/src/netaddress.h index f24c1768d0..8964531fd6 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -125,6 +125,11 @@ class CNetAddr bool GetInAddr(struct in_addr* pipv4Addr) const; uint32_t GetNetClass() const; + //! For IPv4, mapped IPv4, SIIT translated IPv4, Teredo, 6to4 tunneled addresses, return the relevant IPv4 address as a uint32. + uint32_t GetLinkedIPv4() const; + //! Whether this address has a linked IPv4 address (see GetLinkedIPv4()). + bool HasLinkedIPv4() const; + // The AS on the BGP path to the node we use to diversify // peers in AddrMan bucketing based on the AS infrastructure. // The ip->AS mapping depends on how asmap is constructed. diff --git a/src/utilasmap.cpp b/src/utilasmap.cpp index ac230e9ee5..60bd27bf90 100644 --- a/src/utilasmap.cpp +++ b/src/utilasmap.cpp @@ -8,13 +8,14 @@ namespace { -uint32_t DecodeBits(std::vector::const_iterator& bitpos, uint8_t minval, const std::vector &bit_sizes) +uint32_t DecodeBits(std::vector::const_iterator& bitpos, const std::vector::const_iterator& endpos, uint8_t minval, const std::vector &bit_sizes) { uint32_t val = minval; bool bit; for (std::vector::const_iterator bit_sizes_it = bit_sizes.begin(); bit_sizes_it != bit_sizes.end(); ++bit_sizes_it) { if (bit_sizes_it + 1 != bit_sizes.end()) { + if (bitpos == endpos) break; bit = *bitpos; bitpos++; } else { @@ -24,6 +25,7 @@ uint32_t DecodeBits(std::vector::const_iterator& bitpos, uint8_t minval, c val += (1 << *bit_sizes_it); } else { for (int b = 0; b < *bit_sizes_it; b++) { + if (bitpos == endpos) break; bit = *bitpos; bitpos++; val += bit << (*bit_sizes_it - 1 - b); @@ -35,29 +37,29 @@ uint32_t DecodeBits(std::vector::const_iterator& bitpos, uint8_t minval, c } const std::vector TYPE_BIT_SIZES{0, 0, 1}; -uint32_t DecodeType(std::vector::const_iterator& bitpos) +uint32_t DecodeType(std::vector::const_iterator& bitpos, const std::vector::const_iterator& endpos) { - return DecodeBits(bitpos, 0, TYPE_BIT_SIZES); + return DecodeBits(bitpos, endpos, 0, TYPE_BIT_SIZES); } const std::vector ASN_BIT_SIZES{15, 16, 17, 18, 19, 20, 21, 22, 23, 24}; -uint32_t DecodeASN(std::vector::const_iterator& bitpos) +uint32_t DecodeASN(std::vector::const_iterator& bitpos, const std::vector::const_iterator& endpos) { - return DecodeBits(bitpos, 1, ASN_BIT_SIZES); + return DecodeBits(bitpos, endpos, 1, ASN_BIT_SIZES); } const std::vector MATCH_BIT_SIZES{1, 2, 3, 4, 5, 6, 7, 8}; -uint32_t DecodeMatch(std::vector::const_iterator& bitpos) +uint32_t DecodeMatch(std::vector::const_iterator& bitpos, const std::vector::const_iterator& endpos) { - return DecodeBits(bitpos, 2, MATCH_BIT_SIZES); + return DecodeBits(bitpos, endpos, 2, MATCH_BIT_SIZES); } const std::vector JUMP_BIT_SIZES{5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30}; -uint32_t DecodeJump(std::vector::const_iterator& bitpos) +uint32_t DecodeJump(std::vector::const_iterator& bitpos, const std::vector::const_iterator& endpos) { - return DecodeBits(bitpos, 17, JUMP_BIT_SIZES); + return DecodeBits(bitpos, endpos, 17, JUMP_BIT_SIZES); } } @@ -65,33 +67,37 @@ uint32_t DecodeJump(std::vector::const_iterator& bitpos) uint32_t Interpret(const std::vector &asmap, const std::vector &ip) { std::vector::const_iterator pos = asmap.begin(); + const std::vector::const_iterator endpos = asmap.end(); uint8_t bits = ip.size(); - uint8_t default_asn = 0; + uint32_t default_asn = 0; uint32_t opcode, jump, match, matchlen; - while (1) { - assert(pos != asmap.end()); - opcode = DecodeType(pos); + while (pos != endpos) { + opcode = DecodeType(pos, endpos); if (opcode == 0) { - return DecodeASN(pos); + return DecodeASN(pos, endpos); } else if (opcode == 1) { - jump = DecodeJump(pos); + jump = DecodeJump(pos, endpos); + if (bits == 0) break; if (ip[ip.size() - bits]) { + if (jump >= endpos - pos) break; pos += jump; } bits--; } else if (opcode == 2) { - match = DecodeMatch(pos); + match = DecodeMatch(pos, endpos); matchlen = CountBits(match) - 1; for (uint32_t bit = 0; bit < matchlen; bit++) { + if (bits == 0) break; if ((ip[ip.size() - bits]) != ((match >> (matchlen - 1 - bit)) & 1)) { return default_asn; } bits--; } } else if (opcode == 3) { - default_asn = DecodeASN(pos); + default_asn = DecodeASN(pos, endpos); } else { - assert(0); + break; } } + return 0; // 0 is not a valid ASN }