Merge #15196: [test]: Update all subprocess.check_output functions to be Python 3.4 compatible

fdf82ba18 Update all subprocess.check_output functions in CI scripts to be Python 3.4 compatible (Graham Krizek)

Pull request description:

  CI is failing the `lint` stage on every Cron run (regular PR/Push runs still pass). The failure was introduced in 74ce326 and has been broken since. The Python version running in CI was downgraded to 3.4 from 3.6. There were a couple files that were using the `encoding` argument in the `subprocess.check_output` function. This was introduced in Python 3.6 and therefore broke the scripts that were using it. The `universal_newlines` argument was used as well, but in order to use it we must be able to set encoding because of issues on some BSD systems.

  To get CI to pass, I removed all `universal_newline` and `encoding` args to the `check_ouput` function. Then I decoded all `check_output` return values. This should keep the same behavior but be Python 3.4 compatible.

Tree-SHA512: f5e5885e98cf4777be9cc254446a873eedb03bdccbd8e06772a964db95e9fcf46736aa9cdcab1d8f123ea9f4947ed6020679898d8b2f47ffb1d94c21a4b08209

contrib/verify-commits/verify-commits.py

@@ -91,7 +91,7 @@ def main():
     no_sha1 = True
     prev_commit = ""
     initial_commit = current_commit
-    branch = subprocess.check_output([GIT, 'show', '-s', '--format=%H', initial_commit], universal_newlines=True, encoding='utf8').splitlines()[0]
+    branch = subprocess.check_output([GIT, 'show', '-s', '--format=%H', initial_commit]).decode('utf8').splitlines()[0]
 
     # Iterate through commits
     while True:
@@ -112,7 +112,7 @@ def main():
             if prev_commit != "":
                 print("No parent of {} was signed with a trusted key!".format(prev_commit), file=sys.stderr)
                 print("Parents are:", file=sys.stderr)
-                parents = subprocess.check_output([GIT, 'show', '-s', '--format=format:%P', prev_commit], universal_newlines=True, encoding='utf8').splitlines()[0].split(' ')
+                parents = subprocess.check_output([GIT, 'show', '-s', '--format=format:%P', prev_commit]).decode('utf8').splitlines()[0].split(' ')
                 for parent in parents:
                     subprocess.call([GIT, 'show', '-s', parent], stdout=sys.stderr)
             else:
@@ -122,25 +122,25 @@ def main():
         # Check the Tree-SHA512
         if (verify_tree or prev_commit == "") and current_commit not in incorrect_sha512_allowed:
             tree_hash = tree_sha512sum(current_commit)
-            if ("Tree-SHA512: {}".format(tree_hash)) not in subprocess.check_output([GIT, 'show', '-s', '--format=format:%B', current_commit], universal_newlines=True, encoding='utf8').splitlines():
+            if ("Tree-SHA512: {}".format(tree_hash)) not in subprocess.check_output([GIT, 'show', '-s', '--format=format:%B', current_commit]).decode('utf8').splitlines():
                 print("Tree-SHA512 did not match for commit " + current_commit, file=sys.stderr)
                 sys.exit(1)
 
         # Merge commits should only have two parents
-        parents = subprocess.check_output([GIT, 'show', '-s', '--format=format:%P', current_commit], universal_newlines=True, encoding='utf8').splitlines()[0].split(' ')
+        parents = subprocess.check_output([GIT, 'show', '-s', '--format=format:%P', current_commit]).decode('utf8').splitlines()[0].split(' ')
         if len(parents) > 2:
             print("Commit {} is an octopus merge".format(current_commit), file=sys.stderr)
             sys.exit(1)
 
         # Check that the merge commit is clean
-        commit_time = int(subprocess.check_output([GIT, 'show', '-s', '--format=format:%ct', current_commit], universal_newlines=True, encoding='utf8').splitlines()[0])
+        commit_time = int(subprocess.check_output([GIT, 'show', '-s', '--format=format:%ct', current_commit]).decode('utf8').splitlines()[0])
         check_merge = commit_time > time.time() - args.clean_merge * 24 * 60 * 60  # Only check commits in clean_merge days
         allow_unclean = current_commit in unclean_merge_allowed
         if len(parents) == 2 and check_merge and not allow_unclean:
-            current_tree = subprocess.check_output([GIT, 'show', '--format=%T', current_commit], universal_newlines=True, encoding='utf8').splitlines()[0]
+            current_tree = subprocess.check_output([GIT, 'show', '--format=%T', current_commit]).decode('utf8').splitlines()[0]
             subprocess.call([GIT, 'checkout', '--force', '--quiet', parents[0]])
             subprocess.call([GIT, 'merge', '--no-ff', '--quiet', '--no-gpg-sign', parents[1]], stdout=subprocess.DEVNULL)
-            recreated_tree = subprocess.check_output([GIT, 'show', '--format=format:%T', 'HEAD'], universal_newlines=True, encoding='utf8').splitlines()[0]
+            recreated_tree = subprocess.check_output([GIT, 'show', '--format=format:%T', 'HEAD']).decode('utf8').splitlines()[0]
             if current_tree != recreated_tree:
                 print("Merge commit {} is not clean".format(current_commit), file=sys.stderr)
                 subprocess.call([GIT, 'diff', current_commit])

test/lint/check-doc.py

@@ -30,8 +30,8 @@ def main():
         used = check_output(CMD_GREP_ARGS, shell=True, universal_newlines=True, encoding='utf8')
         docd = check_output(CMD_GREP_DOCS, shell=True, universal_newlines=True, encoding='utf8')
     else:
-        used = check_output(CMD_GREP_ARGS, shell=True, universal_newlines=True) # encoding='utf8'
+        used = check_output(CMD_GREP_ARGS, shell=True).decode('utf8').strip()
-        docd = check_output(CMD_GREP_DOCS, shell=True, universal_newlines=True) # encoding='utf8'
+        docd = check_output(CMD_GREP_DOCS, shell=True).decode('utf8').strip()
 
     args_used = set(re.findall(re.compile(REGEX_ARG), used))
     args_docd = set(re.findall(re.compile(REGEX_DOC), docd)).union(SET_DOC_OPTIONAL)