diff --git a/contrib/auto_gdb/log_size.py b/contrib/auto_gdb/log_size.py index 0ae8ee157b..e079724d5f 100644 --- a/contrib/auto_gdb/log_size.py +++ b/contrib/auto_gdb/log_size.py @@ -23,7 +23,7 @@ class LogSizeCommand (gdb.Command): try: args = gdb.string_to_argv(arg) obj = gdb.parse_and_eval(args[0]) - logfile = open(args[1], 'a') + logfile = open(args[1], 'a', encoding="utf8") size = common_helpers.get_instance_size(obj) logfile.write("%s %s: %d\n" % (str(datetime.datetime.now()), args[0], size)) logfile.close() diff --git a/contrib/devtools/check-rpc-mappings.py b/contrib/devtools/check-rpc-mappings.py index b7c9638ed3..9958826853 100755 --- a/contrib/devtools/check-rpc-mappings.py +++ b/contrib/devtools/check-rpc-mappings.py @@ -49,7 +49,7 @@ def process_commands(fname): """Find and parse dispatch table in implementation file `fname`.""" cmds = [] in_rpcs = False - with open(fname, "r") as f: + with open(fname, "r", encoding="utf8") as f: for line in f: line = line.rstrip() if not in_rpcs: @@ -75,7 +75,7 @@ def process_mapping(fname): """Find and parse conversion table in implementation file `fname`.""" cmds = [] in_rpcs = False - with open(fname, "r") as f: + with open(fname, "r", encoding="utf8") as f: for line in f: line = line.rstrip() if not in_rpcs: diff --git a/contrib/devtools/clang-format-diff.py b/contrib/devtools/clang-format-diff.py index 5402870fba..77e845a9b4 100755 --- a/contrib/devtools/clang-format-diff.py +++ b/contrib/devtools/clang-format-diff.py @@ -152,7 +152,7 @@ def main(): sys.exit(p.returncode) if not args.i: - with open(filename) as f: + with open(filename, encoding="utf8") as f: code = f.readlines() formatted_code = io.StringIO(stdout).readlines() diff = difflib.unified_diff(code, formatted_code, diff --git a/contrib/devtools/copyright_header.py b/contrib/devtools/copyright_header.py index 82ed871239..6df5bb9498 100755 --- a/contrib/devtools/copyright_header.py +++ b/contrib/devtools/copyright_header.py @@ -148,7 +148,7 @@ def file_has_without_c_style_copyright_for_holder(contents, holder_name): ################################################################################ def read_file(filename): - return open(os.path.abspath(filename), 'r').read() + return open(os.path.abspath(filename), 'r', encoding="utf8").read() def gather_file_info(filename): info = {} @@ -327,13 +327,13 @@ def get_most_recent_git_change_year(filename): ################################################################################ def read_file_lines(filename): - f = open(os.path.abspath(filename), 'r') + f = open(os.path.abspath(filename), 'r', encoding="utf8") file_lines = f.readlines() f.close() return file_lines def write_file_lines(filename, file_lines): - f = open(os.path.abspath(filename), 'w') + f = open(os.path.abspath(filename), 'w', encoding="utf8") f.write(''.join(file_lines)) f.close() diff --git a/contrib/devtools/github-merge.py b/contrib/devtools/github-merge.py index 2941d2cb6d..0e51dce322 100755 --- a/contrib/devtools/github-merge.py +++ b/contrib/devtools/github-merge.py @@ -190,7 +190,7 @@ def main(): merge_branch = 'pull/'+pull+'/merge' local_merge_branch = 'pull/'+pull+'/local-merge' - devnull = open(os.devnull,'w') + devnull = open(os.devnull, 'w', encoding="utf8") try: subprocess.check_call([GIT,'checkout','-q',branch]) except subprocess.CalledProcessError as e: diff --git a/contrib/devtools/test-security-check.py b/contrib/devtools/test-security-check.py index 8bea43e18f..bf53256db9 100755 --- a/contrib/devtools/test-security-check.py +++ b/contrib/devtools/test-security-check.py @@ -9,7 +9,7 @@ import subprocess import unittest def write_testcode(filename): - with open(filename, 'w') as f: + with open(filename, 'w', encoding="utf8") as f: f.write(''' #include int main() diff --git a/contrib/filter-lcov.py b/contrib/filter-lcov.py index 299377d691..df1db76e92 100755 --- a/contrib/filter-lcov.py +++ b/contrib/filter-lcov.py @@ -13,8 +13,8 @@ pattern = args.pattern outfile = args.outfile in_remove = False -with open(tracefile, 'r') as f: - with open(outfile, 'w') as wf: +with open(tracefile, 'r', encoding="utf8") as f: + with open(outfile, 'w', encoding="utf8") as wf: for line in f: for p in pattern: if line.startswith("SF:") and p in line: diff --git a/contrib/linearize/linearize-data.py b/contrib/linearize/linearize-data.py index ae13720c3a..f1fd8f92ce 100755 --- a/contrib/linearize/linearize-data.py +++ b/contrib/linearize/linearize-data.py @@ -78,7 +78,7 @@ def get_blk_dt(blk_hdr): # When getting the list of block hashes, undo any byte reversals. def get_block_hashes(settings): blkindex = [] - f = open(settings['hashlist'], "r") + f = open(settings['hashlist'], "r", encoding="utf8") for line in f: line = line.rstrip() if settings['rev_hash_bytes'] == 'true': @@ -264,7 +264,7 @@ if __name__ == '__main__': print("Usage: linearize-data.py CONFIG-FILE") sys.exit(1) - f = open(sys.argv[1]) + f = open(sys.argv[1], encoding="utf8") for line in f: # skip comment lines m = re.search('^\s*#', line) diff --git a/contrib/linearize/linearize-hashes.py b/contrib/linearize/linearize-hashes.py index 999ff689bd..890cdd3506 100755 --- a/contrib/linearize/linearize-hashes.py +++ b/contrib/linearize/linearize-hashes.py @@ -97,7 +97,7 @@ def get_block_hashes(settings, max_blocks_per_call=10000): def get_rpc_cookie(): # Open the cookie file - with open(os.path.join(os.path.expanduser(settings['datadir']), '.cookie'), 'r') as f: + with open(os.path.join(os.path.expanduser(settings['datadir']), '.cookie'), 'r', encoding="ascii") as f: combined = f.readline() combined_split = combined.split(":") settings['rpcuser'] = combined_split[0] @@ -108,7 +108,7 @@ if __name__ == '__main__': print("Usage: linearize-hashes.py CONFIG-FILE") sys.exit(1) - f = open(sys.argv[1]) + f = open(sys.argv[1], encoding="utf8") for line in f: # skip comment lines m = re.search('^\s*#', line) diff --git a/contrib/seeds/generate-seeds.py b/contrib/seeds/generate-seeds.py index 515f1ccca4..c4ab4c9839 100755 --- a/contrib/seeds/generate-seeds.py +++ b/contrib/seeds/generate-seeds.py @@ -11,7 +11,7 @@ argument: nodes_main.txt nodes_test.txt -These files must consist of lines in the format +These files must consist of lines in the format : @@ -126,13 +126,13 @@ def main(): g.write(' * Each line contains a 16-byte IPv6 address and a port.\n') g.write(' * IPv4 as well as onion addresses are wrapped inside an IPv6 address accordingly.\n') g.write(' */\n') - with open(os.path.join(indir,'nodes_main.txt'),'r') as f: + with open(os.path.join(indir,'nodes_main.txt'), 'r', encoding="utf8") as f: process_nodes(g, f, 'pnSeed6_main', 9999) g.write('\n') - with open(os.path.join(indir,'nodes_test.txt'),'r') as f: + with open(os.path.join(indir,'nodes_test.txt'), 'r', encoding="utf8") as f: process_nodes(g, f, 'pnSeed6_test', 19999) g.write('#endif // DASH_CHAINPARAMSSEEDS_H\n') - + if __name__ == '__main__': main() diff --git a/contrib/seeds/makeseeds.py b/contrib/seeds/makeseeds.py index 6fe3e38632..a2e73bf52e 100755 --- a/contrib/seeds/makeseeds.py +++ b/contrib/seeds/makeseeds.py @@ -139,7 +139,7 @@ def filterbyasn(ips, max_per_asn, max_total): def main(): # This expects a json as outputted by "protx list valid 1" if len(sys.argv) > 1: - with open(sys.argv[1], 'r') as f: + with open(sys.argv[1], 'r', encoding="utf8") as f: mns = json.load(f) else: mns = json.load(sys.stdin) diff --git a/share/qt/extract_strings_qt.py b/share/qt/extract_strings_qt.py index a11456c852..45efbaaedd 100755 --- a/share/qt/extract_strings_qt.py +++ b/share/qt/extract_strings_qt.py @@ -63,7 +63,7 @@ child = Popen([XGETTEXT,'--output=-','-n','--keyword=_'] + files, stdout=PIPE) messages = parse_po(out.decode('utf-8')) -f = open(OUT_CPP, 'w') +f = open(OUT_CPP, 'w', encoding="utf8") f.write(""" #include diff --git a/test/functional/multiwallet.py b/test/functional/multiwallet.py index 9d79e02491..9d52276e38 100755 --- a/test/functional/multiwallet.py +++ b/test/functional/multiwallet.py @@ -81,7 +81,7 @@ class MultiWalletTest(BitcoinTestFramework): self.assert_start_raises_init_error(0, ['-walletdir=bad'], 'Error: Specified -walletdir "bad" does not exist') # should not initialize if the specified walletdir is not a directory not_a_dir = wallet_dir('notadir') - open(not_a_dir, 'a').close() + open(not_a_dir, 'a', encoding="utf8").close() self.assert_start_raises_init_error(0, ['-walletdir=' + not_a_dir], 'Error: Specified -walletdir "' + not_a_dir + '" is not a directory') # if wallets/ doesn't exist, datadir should be the default wallet dir diff --git a/test/functional/notifications.py b/test/functional/notifications.py index 85eaaabd58..4ce716ef54 100755 --- a/test/functional/notifications.py +++ b/test/functional/notifications.py @@ -36,7 +36,7 @@ class NotificationsTest(BitcoinTestFramework): wait_until(lambda: os.path.isfile(self.block_filename) and os.stat(self.block_filename).st_size >= (block_count * 65), timeout=10) # file content should equal the generated blocks hashes - with open(self.block_filename, 'r') as f: + with open(self.block_filename, 'r', encoding="utf-8") as f: assert_equal(sorted(blocks), sorted(f.read().splitlines())) self.log.info("test -walletnotify") @@ -45,7 +45,7 @@ class NotificationsTest(BitcoinTestFramework): # file content should equal the generated transaction hashes txids_rpc = list(map(lambda t: t['txid'], self.nodes[1].listtransactions("*", block_count))) - with open(self.tx_filename, 'r') as f: + with open(self.tx_filename, 'r', encoding="ascii") as f: assert_equal(sorted(txids_rpc), sorted(f.read().splitlines())) os.remove(self.tx_filename) @@ -58,7 +58,7 @@ class NotificationsTest(BitcoinTestFramework): # file content should equal the generated transaction hashes txids_rpc = list(map(lambda t: t['txid'], self.nodes[1].listtransactions("*", block_count))) - with open(self.tx_filename, 'r') as f: + with open(self.tx_filename, 'r', encoding="ascii") as f: assert_equal(sorted(txids_rpc), sorted(f.read().splitlines())) # Mine another 41 up-version blocks. -alertnotify should trigger on the 51st. diff --git a/test/functional/rpc_getblockstats.py b/test/functional/rpc_getblockstats.py index fcb9fc944b..d650a77bd7 100755 --- a/test/functional/rpc_getblockstats.py +++ b/test/functional/rpc_getblockstats.py @@ -80,11 +80,11 @@ class GetblockstatsTest(BitcoinTestFramework): 'mocktime': int(self.mocktime), 'stats': self.expected_stats, } - with open(filename, 'w') as f: + with open(filename, 'w', encoding="utf8") as f: json.dump(to_dump, f, sort_keys=True, indent=2) def load_test_data(self, filename): - with open(filename, 'r') as f: + with open(filename, 'r', encoding="utf8") as f: d = json.load(f) blocks = d['blocks'] self.mocktime = d['mocktime'] diff --git a/test/functional/test_framework/util.py b/test/functional/test_framework/util.py index 1de2351e04..e8727fcb6d 100644 --- a/test/functional/test_framework/util.py +++ b/test/functional/test_framework/util.py @@ -325,7 +325,7 @@ def get_auth_cookie(datadir): assert password is None # Ensure that there is only one rpcpassword line password = line.split("=")[1].strip("\n") if os.path.isfile(os.path.join(datadir, "regtest", ".cookie")): - with open(os.path.join(datadir, "regtest", ".cookie"), 'r') as f: + with open(os.path.join(datadir, "regtest", ".cookie"), 'r', encoding="ascii") as f: userpass = f.read() split_userpass = userpass.split(':') user = split_userpass[0] diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index 6191a8d178..f69ef06375 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -222,7 +222,7 @@ def main(): # Read config generated by configure. config = configparser.ConfigParser() configfile = os.path.abspath(os.path.dirname(__file__)) + "/../config.ini" - config.read_file(open(configfile)) + config.read_file(open(configfile, encoding="utf8")) passon_args.append("--configfile=%s" % configfile) @@ -596,7 +596,7 @@ class RPCCoverage(): if not os.path.isfile(coverage_ref_filename): raise RuntimeError("No coverage reference found") - with open(coverage_ref_filename, 'r') as file: + with open(coverage_ref_filename, 'r', encoding="utf8") as file: all_cmds.update([line.strip() for line in file.readlines()]) for root, dirs, files in os.walk(self.dir): @@ -605,7 +605,7 @@ class RPCCoverage(): coverage_filenames.add(os.path.join(root, filename)) for filename in coverage_filenames: - with open(filename, 'r') as file: + with open(filename, 'r', encoding="utf8") as file: covered_cmds.update([line.strip() for line in file.readlines()]) return all_cmds - covered_cmds diff --git a/test/lint/lint-python-utf8-encoding.sh b/test/lint/lint-python-utf8-encoding.sh new file mode 100755 index 0000000000..ce973e710a --- /dev/null +++ b/test/lint/lint-python-utf8-encoding.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# +# Copyright (c) 2018 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Make sure we explicitly open all text files using UTF-8 (or ASCII) encoding to +# avoid potential issues on the BSDs where the locale is not always set. + +EXIT_CODE=0 +OUTPUT=$(git grep " open(" -- "*.py" | grep -vE "encoding=.(ascii|utf8|utf-8)." | grep -vE "open\([^,]*, ['\"][^'\"]*b[^'\"]*['\"]") +if [[ ${OUTPUT} != "" ]]; then + echo "Python's open(...) seems to be used to open text files without explicitly" + echo "specifying encoding=\"utf8\":" + echo + echo "${OUTPUT}" + EXIT_CODE=1 +fi +exit ${EXIT_CODE} diff --git a/test/util/bitcoin-util-test.py b/test/util/bitcoin-util-test.py index 30bd13d0dc..16371a6234 100755 --- a/test/util/bitcoin-util-test.py +++ b/test/util/bitcoin-util-test.py @@ -28,7 +28,7 @@ import sys def main(): config = configparser.ConfigParser() config.optionxform = str - config.readfp(open(os.path.join(os.path.dirname(__file__), "../config.ini"))) + config.readfp(open(os.path.join(os.path.dirname(__file__), "../config.ini"), encoding="utf8")) env_conf = dict(config.items('environment')) parser = argparse.ArgumentParser(description=__doc__) @@ -49,7 +49,7 @@ def main(): def bctester(testDir, input_basename, buildenv): """ Loads and parses the input file, runs all tests and reports results""" input_filename = os.path.join(testDir, input_basename) - raw_data = open(input_filename).read() + raw_data = open(input_filename, encoding="utf8").read() input_data = json.loads(raw_data) failed_testcases = [] @@ -86,7 +86,7 @@ def bctest(testDir, testObj, buildenv): inputData = None if "input" in testObj: filename = os.path.join(testDir, testObj["input"]) - inputData = open(filename).read() + inputData = open(filename, encoding="utf8").read() stdinCfg = subprocess.PIPE # Read the expected output data (if there is any) @@ -97,7 +97,7 @@ def bctest(testDir, testObj, buildenv): outputFn = testObj['output_cmp'] outputType = os.path.splitext(outputFn)[1][1:] # output type from file extension (determines how to compare) try: - outputData = open(os.path.join(testDir, outputFn)).read() + outputData = open(os.path.join(testDir, outputFn), encoding="utf8").read() except: logging.error("Output file " + outputFn + " can not be opened") raise