Merge #15257: Scripts and tools: Bump flake8 to 3.7.8

3d0a82cff8cbb809876e82dbe62d14d2adc07d94 devtools: Accomodate block-style copyright blocks (Ben Woosley)
0ef0e51fe4bb592e67255776b5a0ba04679fb8c4 lint: Bump flake8 to 3.7.8 (Ben Woosley)
838920704ad90a71cf288b700052503db8abb17e lint: Disable flake8 W504 warning (Ben Woosley)
b21680baf5391a602b295b9d7d0ef66553661cb9 test/contrib: Fix invalid escapes in regex strings (Ben Woosley)

Pull request description:

  This is a second go at #15221, fixing new lints in:
  W504 line break after binary operator
  W605 invalid escape sequence
  F841 local variable 'e' is assigned to but never used

  This time around:
  * One commit per rule, for easier review
  * I went with the PEP-8 style of breaking before binary operators
  * I looked into the raw regex newline issue, and found that raw strings with newlines embedded do work appropriately. E.g. run `re.match(r" \n ", " \n ")` to check this for yourself. `re.MULTILINE` exists to modify `^` and `$` in multiline scenarios, but  all of these searches are per-line.

ACKs for top commit:
  practicalswift:
    ACK 3d0a82cff8cbb809876e82dbe62d14d2adc07d94 -- diff looks correct

Tree-SHA512: bea0c144cadd72e4adf2e9a4b4ee0535dd91a8e694206924cf8a389dc9253f364a717edfe9abda88108fbb67fda19b9e823f46822d7303c0aaa72e48909a6105

.travis/lint_04_install.sh

@@ -7,7 +7,7 @@
 export LC_ALL=C
 
 travis_retry pip3 install codespell==1.15.0
-travis_retry pip3 install flake8==3.5.0
+travis_retry pip3 install flake8==3.7.8
 travis_retry pip3 install vulture==2.3
 travis_retry pip3 install yq
 

ci/Dockerfile.builder

@@ -15,7 +15,7 @@ RUN apt-get update && apt-get install $APT_ARGS python3-pip python3-setuptools &
 # Python stuff
 RUN pip3 install pyzmq # really needed?
 RUN pip3 install jinja2
-RUN pip3 install flake8==3.5.0
+RUN pip3 install flake8==3.7.8
 RUN pip3 install codespell==1.15.0
 RUN pip3 install vulture==2.3
 RUN pip3 install yq

contrib/devtools/clang-format-diff.py

@@ -106,7 +106,7 @@ def main():
   filename = None
   lines_by_file = {}
   for line in sys.stdin:
-    match = re.search('^\+\+\+\ (.*?/){%s}(\S*)' % args.p, line)
+    match = re.search(r'^\+\+\+\ (.*?/){%s}(\S*)' % args.p, line)
     if match:
       filename = match.group(2)
     if filename == None:
@@ -119,7 +119,7 @@ def main():
       if not re.match('^%s$' % args.iregex, filename, re.IGNORECASE):
         continue
 
-    match = re.search('^@@.*\+(\d+)(,(\d+))?', line)
+    match = re.search(r'^@@.*\+(\d+)(,(\d+))?', line)
     if match:
       start_line = int(match.group(1))
       line_count = 1

contrib/devtools/copyright_header.py

@@ -74,7 +74,7 @@ def get_filenames_to_examine(base_directory):
 ################################################################################
 
 
-COPYRIGHT_WITH_C = 'Copyright \(c\)'
+COPYRIGHT_WITH_C = r'Copyright \(c\)'
 COPYRIGHT_WITHOUT_C = 'Copyright'
 ANY_COPYRIGHT_STYLE = '(%s|%s)' % (COPYRIGHT_WITH_C, COPYRIGHT_WITHOUT_C)
 
@@ -88,22 +88,22 @@ ANY_COPYRIGHT_STYLE_OR_YEAR_STYLE = ("%s %s" % (ANY_COPYRIGHT_STYLE,
 ANY_COPYRIGHT_COMPILED = re.compile(ANY_COPYRIGHT_STYLE_OR_YEAR_STYLE)
 
 def compile_copyright_regex(copyright_style, year_style, name):
-    return re.compile('%s %s,? %s' % (copyright_style, year_style, name))
+    return re.compile(r'%s %s,? %s( +\*)?\n' % (copyright_style, year_style, name))
 
 EXPECTED_HOLDER_NAMES = [
-    "Satoshi Nakamoto\n",
+    r"Satoshi Nakamoto",
-    "The Bitcoin Core developers\n",
+    r"The Bitcoin Core developers",
-    "BitPay Inc\.\n",
+    r"The Dash Core developers",
-    "University of Illinois at Urbana-Champaign\.\n",
+    r"BitPay Inc\.",
-    "Pieter Wuille\n",
+    r"University of Illinois at Urbana-Champaign\.",
-    "Wladimir J. van der Laan\n",
+    r"Pieter Wuille",
-    "Jeff Garzik\n",
+    r"Wladimir J\. van der Laan",
-    "Jan-Klaas Kollhof\n",
+    r"Jeff Garzik",
-    "ArtForz -- public domain half-a-node\n",
+    r"Jan-Klaas Kollhof",
-    "The Dash Core developers\n",
+    r"ArtForz -- public domain half-a-node",
-    "Intel Corporation",
+    r"Intel Corporation ?",
-    "The Zcash developers",
+    r"The Zcash developers",
-    "Jeremy Rubin",
+    r"Jeremy Rubin",
 ]
 
 DOMINANT_STYLE_COMPILED = {}
@@ -333,7 +333,7 @@ def write_file_lines(filename, file_lines):
 # update header years execution
 ################################################################################
 
-COPYRIGHT = 'Copyright \(c\)'
+COPYRIGHT = r'Copyright \(c\)'
 YEAR = "20[0-9][0-9]"
 YEAR_RANGE = '(%s)(-%s)?' % (YEAR, YEAR)
 HOLDER = 'The Dash Core developers'

contrib/devtools/symbol-check.py

@@ -141,7 +141,7 @@ def read_libraries(filename):
     for line in stdout.splitlines():
         tokens = line.split()
         if len(tokens)>2 and tokens[1] == '(NEEDED)':
-            match = re.match('^Shared library: \[(.*)\]$', ' '.join(tokens[2:]))
+            match = re.match(r'^Shared library: \[(.*)\]$', ' '.join(tokens[2:]))
             if match:
                 libraries.append(match.group(1))
             else:
@@ -171,5 +171,3 @@ if __name__ == '__main__':
                 retval = 1
 
     sys.exit(retval)
-
-

contrib/linearize/linearize-data.py

@@ -295,12 +295,12 @@ if __name__ == '__main__':
     f = open(sys.argv[1], encoding="utf8")
     for line in f:
         # skip comment lines
-        m = re.search('^\s*#', line)
+        m = re.search(r'^\s*#', line)
         if m:
             continue
 
         # parse key=value lines
-        m = re.search('^(\w+)\s*=\s*(\S.*)$', line)
+        m = re.search(r'^(\w+)\s*=\s*(\S.*)$', line)
         if m is None:
             continue
         settings[m.group(1)] = m.group(2)

contrib/linearize/linearize-hashes.py

@@ -106,12 +106,12 @@ if __name__ == '__main__':
     f = open(sys.argv[1], encoding="utf8")
     for line in f:
         # skip comment lines
-        m = re.search('^\s*#', line)
+        m = re.search(r'^\s*#', line)
         if m:
             continue
 
         # parse key=value lines
-        m = re.search('^(\w+)\s*=\s*(\S.*)$', line)
+        m = re.search(r'^(\w+)\s*=\s*(\S.*)$', line)
         if m is None:
             continue
         settings[m.group(1)] = m.group(2)

contrib/macdeploy/macdeployqtplus

@@ -766,7 +766,7 @@ if config.dmg is not None:
         except subprocess.CalledProcessError as e:
             sys.exit(e.returncode)
 
-        m = re.search("/Volumes/(.+$)", output)
+        m = re.search(r"/Volumes/(.+$)", output)
         disk_root = m.group(0)
         disk_name = m.group(1)
 

contrib/seeds/generate-seeds.py

@@ -74,7 +74,7 @@ def name_to_ipv6(addr):
         raise ValueError('Could not parse address %s' % addr)
 
 def parse_spec(s, defaultport):
-    match = re.match('\[([0-9a-fA-F:]+)\](?::([0-9]+))?$', s)
+    match = re.match(r'\[([0-9a-fA-F:]+)\](?::([0-9]+))?$', s)
     if match: # ipv6
         host = match.group(1)
         port = match.group(2)
@@ -136,4 +136,3 @@ def main():
 
 if __name__ == '__main__':
     main()
-

test/functional/combine_logs.py

@@ -49,7 +49,7 @@ def read_logs(tmp_dir):
     chain = glob.glob("{}/node0/*/debug.log".format(tmp_dir))
     if chain:
         chain = chain[0]  # pick the first one if more than one chain was found (should never happen)
-        chain = re.search('node0/(.+?)/debug\.log$', chain).group(1)  # extract the chain name
+        chain = re.search(r'node0/(.+?)/debug\.log$', chain).group(1)  # extract the chain name
     else:
         chain = 'regtest'  # fallback to regtest (should only happen when none exists)
 

test/functional/feature_logging.py

@@ -36,7 +36,7 @@ class LoggingTest(BitcoinTestFramework):
         invdir = self.relative_log_path("foo")
         invalidname = os.path.join("foo", "foo.log")
         self.stop_node(0)
-        exp_stderr = "Error: Could not open debug log file \S+$"
+        exp_stderr = r"Error: Could not open debug log file \S+$"
         self.nodes[0].assert_start_raises_init_error(["-debuglogfile=%s" % (invalidname)], exp_stderr, match=ErrorMatch.FULL_REGEX)
         assert not os.path.isfile(os.path.join(invdir, "foo.log"))
 

test/functional/feature_uacomment.py

@@ -27,12 +27,12 @@ class UacommentTest(BitcoinTestFramework):
 
         self.log.info("test -uacomment max length")
         self.stop_node(0)
-        expected = "Error: Total length of network version string \([0-9]+\) exceeds maximum length \(256\). Reduce the number or size of uacomments."
+        expected = r"Error: Total length of network version string \([0-9]+\) exceeds maximum length \(256\). Reduce the number or size of uacomments."
         self.nodes[0].assert_start_raises_init_error(["-uacomment=" + 'a' * 256], expected, match=ErrorMatch.FULL_REGEX)
 
         self.log.info("test -uacomment unsafe characters")
         for unsafe_char in ['/', ':', '(', ')', '₿', '🏃']:
-            expected = "Error: User Agent comment \(" + re.escape(unsafe_char) + "\) contains unsafe characters."
+            expected = r"Error: User Agent comment \(" + re.escape(unsafe_char) + r"\) contains unsafe characters."
             self.nodes[0].assert_start_raises_init_error(["-uacomment=" + unsafe_char], expected, match=ErrorMatch.FULL_REGEX)
 
 

test/functional/wallet_multiwallet.py

@@ -96,13 +96,13 @@ class MultiWalletTest(BitcoinTestFramework):
 
         # should not initialize if one wallet is a copy of another
         shutil.copyfile(wallet_dir('w8'), wallet_dir('w8_copy'))
-        exp_stderr = "Can't open database w8_copy \(duplicates fileid \w+ from w8\)"
+        exp_stderr = r"Can't open database w8_copy \(duplicates fileid \w+ from w8\)"
         self.nodes[0].assert_start_raises_init_error(['-wallet=w8', '-wallet=w8_copy'], exp_stderr, match=ErrorMatch.PARTIAL_REGEX)
 
         # should not initialize if wallet file is a symlink
         if os.name != 'nt':
             os.symlink('w8', wallet_dir('w8_symlink'))
-            self.nodes[0].assert_start_raises_init_error(['-wallet=w8_symlink'], 'Error: Invalid -wallet path \'w8_symlink\'\. .*', match=ErrorMatch.FULL_REGEX)
+            self.nodes[0].assert_start_raises_init_error(['-wallet=w8_symlink'], r'Error: Invalid -wallet path \'w8_symlink\'\. .*', match=ErrorMatch.FULL_REGEX)
 
         # should not initialize if the specified walletdir does not exist
         self.nodes[0].assert_start_raises_init_error(['-walletdir=bad'], 'Error: Specified -walletdir "bad" does not exist')
@@ -143,7 +143,7 @@ class MultiWalletTest(BitcoinTestFramework):
         competing_wallet_dir = os.path.join(self.options.tmpdir, 'competing_walletdir')
         os.mkdir(competing_wallet_dir)
         self.restart_node(0, ['-walletdir=' + competing_wallet_dir])
-        exp_stderr = "Error: Error initializing wallet database environment \"\S+competing_walletdir\"!"
+        exp_stderr = r"Error: Error initializing wallet database environment \"\S+competing_walletdir\"!"
         self.nodes[1].assert_start_raises_init_error(['-walletdir=' + competing_wallet_dir], exp_stderr, match=ErrorMatch.PARTIAL_REGEX)
 
         self.restart_node(0, extra_args)

test/lint/check-doc.py

@@ -15,8 +15,8 @@ import re
 
 FOLDER_GREP = 'src'
 FOLDER_TEST = 'src/test/'
-REGEX_ARG = '(?:ForceSet|SoftSet|Get|Is)(?:Bool)?Args?(?:Set)?\("(-[^"]+)"'
+REGEX_ARG = r'(?:ForceSet|SoftSet|Get|Is)(?:Bool)?Args?(?:Set)?\("(-[^"]+)"'
-REGEX_DOC = 'AddArg\("(-[^"=]+?)(?:=|")'
+REGEX_DOC = r'AddArg\("(-[^"=]+?)(?:=|")'
 CMD_ROOT_DIR = '$(git rev-parse --show-toplevel)/{}'.format(FOLDER_GREP)
 CMD_GREP_ARGS = r"git grep --perl-regexp '{}' -- {} ':(exclude){}'".format(REGEX_ARG, CMD_ROOT_DIR, FOLDER_TEST)
 CMD_GREP_WALLET_ARGS = r"git grep --function-context 'void WalletInit::AddWalletOptions' -- {} | grep AddArg".format(CMD_ROOT_DIR)

test/lint/check-rpc-mappings.py

@@ -53,13 +53,13 @@ def process_commands(fname):
         for line in f:
             line = line.rstrip()
             if not in_rpcs:
-                if re.match("static const CRPCCommand .*\[\] =", line):
+                if re.match(r"static const CRPCCommand .*\[\] =", line):
                     in_rpcs = True
             else:
                 if line.startswith('};'):
                     in_rpcs = False
                 elif '{' in line and '"' in line:
-                    m = re.search('{ *("[^"]*"), *("[^"]*"), *&([^,]*), *{([^}]*)} *},', line)
+                    m = re.search(r'{ *("[^"]*"), *("[^"]*"), *&([^,]*), *{([^}]*)} *},', line)
                     assert m, 'No match to table expression: %s' % line
                     name = parse_string(m.group(2))
                     args_str = m.group(4).strip()
@@ -85,7 +85,7 @@ def process_mapping(fname):
                 if line.startswith('};'):
                     in_rpcs = False
                 elif '{' in line and '"' in line:
-                    m = re.search('{ *("[^"]*"), *([0-9]+) *, *("[^"]*") *},', line)
+                    m = re.search(r'{ *("[^"]*"), *([0-9]+) *, *("[^"]*") *},', line)
                     assert m, 'No match to table expression: %s' % line
                     name = parse_string(m.group(1))
                     idx = int(m.group(2))

test/lint/lint-format-strings.py

@@ -63,7 +63,7 @@ def normalize(s):
     assert type(s) is str
     s = s.replace("\n", " ")
     s = s.replace("\t", " ")
-    s = re.sub("/\*.*?\*/", " ", s)
+    s = re.sub(r"/\*.*?\*/", " ", s)
     s = re.sub(" {2,}", " ", s)
     return s.strip()
 

test/lint/lint-python.sh

@@ -73,7 +73,6 @@ enabled=(
     W291 # trailing whitespace
     W292 # no newline at end of file
     W293 # blank line contains whitespace
-    # W504 # line break after binary operator
     W601 # .has_key() is deprecated, use "in"
     W602 # deprecated form of raising exception
     W603 # "<>" is deprecated, use "!="