Modify makesseeds.py to work with "protx list valid 1" instead of "masternode list (#3235)

* Modify makesseeds.py to work with "protx list valid 1" instead of "masternode list"

This allows better filtering for MN owners with multiple MNs. This commit
also removes some unsupported fields, e.g. "protocol", "lastseen", ...

* Update contrib/seeds/README.md with new instructions

contrib/seeds/README.md

@@ -3,14 +3,16 @@
 Utility to generate the seeds.txt list that is compiled into the client
 (see [src/chainparamsseeds.h](/src/chainparamsseeds.h) and other utilities in [contrib/seeds](/contrib/seeds)).
 
-Be sure to update `MIN_PROTOCOL_VERSION` in `makeseeds.py` to include the current version.
+The seeds compiled into the release are created from the current protx list, like this:
 
-The seeds compiled into the release are created from the current masternode list, like this:
+    dash-cli protx list valid 1 1185193 > protx_list.json
-
+    python3 makeseeds.py < protx_list.json > nodes_main.txt
-    dash-cli masternodelist full > mnlist.json
-    python3 makeseeds.py < mnlist.json > nodes_main.txt
     python3 generate-seeds.py . > ../../src/chainparamsseeds.h
 
+Make sure to use a recent block height in the "protx list" call. After updating, create a PR and
+specify which block height you used so that reviewers can re-run the same commands and verify
+that the list is as expected.
+
 ## Dependencies
 
 Ubuntu:

contrib/seeds/makeseeds.py

@@ -3,17 +3,13 @@
 # Distributed under the MIT software license, see the accompanying
 # file COPYING or http://www.opensource.org/licenses/mit-license.php.
 #
-# Generate seeds.txt from masternode list
+# Generate seeds.txt from "protx list valid 1"
 #
 
 NSEEDS=512
 
 MAX_SEEDS_PER_ASN=4
 
-MIN_PROTOCOL_VERSION = 70213
-MAX_LAST_SEEN_DIFF = 60 * 60 * 24 * 1 # 1 day
-MAX_LAST_PAID_DIFF = 60 * 60 * 24 * 30 # 1 month
-
 # These are hosts that have been observed to be behaving strangely (e.g.
 # aggressively connecting to every node).
 SUSPICIOUS_HOSTS = {
@@ -31,17 +27,14 @@ PATTERN_IPV4 = re.compile(r"^((\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})):(\d+)$
 PATTERN_IPV6 = re.compile(r"^\[([0-9a-z:]+)\]:(\d+)$")
 PATTERN_ONION = re.compile(r"^([abcdefghijklmnopqrstuvwxyz234567]{16}\.onion):(\d+)$")
 
-def parseline(line):
+def parseip(ip):
-    # line format: status protocol payee lastseen activeseconds lastpaidtime lastpaidblock IP
+    m = PATTERN_IPV4.match(ip)
-    sline = line.split()
-
-    m = PATTERN_IPV4.match(sline[7])
     sortkey = None
     ip = None
     if m is None:
-        m = PATTERN_IPV6.match(sline[7])
+        m = PATTERN_IPV6.match(ip)
         if m is None:
-            m = PATTERN_ONION.match(sline[7])
+            m = PATTERN_ONION.match(ip)
             if m is None:
                 return None
             else:
@@ -70,13 +63,6 @@ def parseline(line):
         port = int(m.group(6))
 
     return {
-        "status": sline[0],
-        "protocol": int(sline[1]),
-        "payee": sline[2],
-        "lastseen": int(sline[3]),
-        "activeseconds": int(sline[4]),
-        "lastpaidtime": int(sline[5]),
-        "lastpaidblock": int(sline[6]),
         "net": net,
         "ip": ipstr,
         "port": port,
@@ -84,12 +70,26 @@ def parseline(line):
         "sortkey": sortkey
     }
 
-def filtermultiport(ips):
+def filtermulticollateralhash(mns):
-    '''Filter out hosts with more nodes per IP'''
+    '''Filter out MNs sharing the same collateral hash'''
     hist = collections.defaultdict(list)
-    for ip in ips:
+    for mn in mns:
-        hist[ip['sortkey']].append(ip)
+        hist[mn['collateralHash']].append(mn)
-    return [value[0] for (key,value) in list(hist.items()) if len(value)==1]
+    return [mn for mn in mns if len(hist[mn['collateralHash']]) == 1]
+
+def filtermulticollateraladdress(mns):
+    '''Filter out MNs sharing the same collateral address'''
+    hist = collections.defaultdict(list)
+    for mn in mns:
+        hist[mn['collateralAddress']].append(mn)
+    return [mn for mn in mns if len(hist[mn['collateralAddress']]) == 1]
+
+def filtermultipayoutaddress(mns):
+    '''Filter out MNs sharing the same payout address'''
+    hist = collections.defaultdict(list)
+    for mn in mns:
+        hist[mn['state']['payoutAddress']].append(mn)
+    return [mn for mn in mns if len(hist[mn['state']['payoutAddress']]) == 1]
 
 def resolveasn(resolver, ip):
     asn = int([x.to_text() for x in resolver.query('.'.join(reversed(ip.split('.'))) + '.origin.asn.cymru.com', 'TXT').response.answer][0].split('\"')[1].split(' ')[0])
@@ -138,29 +138,23 @@ def filterbyasn(ips, max_per_asn, max_total):
     return result
 
 def main():
+    # This expects a json as outputted by "protx list valid 1"
     if len(sys.argv) > 1:
         with open(sys.argv[1], 'r') as f:
-            js = json.load(f)
+            mns = json.load(f)
     else:
-        js = json.load(sys.stdin)
+        mns = json.load(sys.stdin)
-    ips = [parseline(line) for collateral, line in js.items()]
 
-    cur_time = int(time.time())
+    # Skip PoSe banned MNs
-
+    mns = [mn for mn in mns if mn['state']['PoSeBanHeight'] == -1]
-    # Skip entries with valid address.
+    # Skip MNs with < 10000 confirmations
-    ips = [ip for ip in ips if ip is not None]
+    mns = [mn for mn in mns if mn['confirmations'] >= 10000]
-    # Enforce ENABLED state
+    # Filter out MNs which are definitely from the same person/operator
-    ips = [ip for ip in ips if ip['status'] == "ENABLED"]
+    mns = filtermulticollateralhash(mns)
-    # Enforce minimum protocol version
+    mns = filtermulticollateraladdress(mns)
-    ips = [ip for ip in ips if ip['protocol'] >= MIN_PROTOCOL_VERSION]
+    mns = filtermultipayoutaddress(mns)
-    # Require at least 2 week uptime
+    # Extract IPs
-    ips = [ip for ip in ips if cur_time - ip['lastseen'] < MAX_LAST_SEEN_DIFF]
+    ips = [parseip(mn['state']['service']) for mn in mns]
-    # Require to be paid recently
-    ips = [ip for ip in ips if cur_time - ip['lastpaidtime'] < MAX_LAST_PAID_DIFF]
-    # Sort by availability (and use lastpaidtime as tie breaker)
-    ips.sort(key=lambda x: (x['activeseconds'], x['lastpaidtime'], x['ip']), reverse=True)
-    # Filter out hosts with multiple ports, these are likely abusive
-    ips = filtermultiport(ips)
     # Look up ASNs and limit results, both per ASN and globally.
     ips = filterbyasn(ips, MAX_SEEDS_PER_ASN, NSEEDS)
     # Sort the results by IP address (for deterministic output).