diff --git a/ci/dash/build_src.sh b/ci/dash/build_src.sh index e940aac194..4a7be858ae 100755 --- a/ci/dash/build_src.sh +++ b/ci/dash/build_src.sh @@ -25,7 +25,7 @@ if [ "$CHECK_DOC" = 1 ]; then # TODO: Check docs (re-enable after all Bitcoin PRs have been merged and docs fully fixed) #test/lint/check-doc.py # Run all linters - test/lint/lint-all.sh + test/lint/all-lint.py fi ccache --zero-stats --max-size=$CCACHE_SIZE diff --git a/ci/lint/06_script.sh b/ci/lint/06_script.sh index edfedac64e..f4a3f3f985 100755 --- a/ci/lint/06_script.sh +++ b/ci/lint/06_script.sh @@ -21,7 +21,7 @@ test/lint/git-subtree-check.sh src/minisketch test/lint/git-subtree-check.sh src/univalue test/lint/git-subtree-check.sh src/leveldb test/lint/check-doc.py -test/lint/lint-all.sh +test/lint/all-lint.py if [ "$CIRRUS_REPO_FULL_NAME" = "dashpay/dash" ] && [ -n "$CIRRUS_CRON" ]; then git log --merges --before="2 days ago" -1 --format='%H' > ./contrib/verify-commits/trusted-sha512-root-commit diff --git a/contrib/guix/libexec/prelude.bash b/contrib/guix/libexec/prelude.bash index 859095b04d..e0d653e46c 100644 --- a/contrib/guix/libexec/prelude.bash +++ b/contrib/guix/libexec/prelude.bash @@ -2,10 +2,10 @@ export LC_ALL=C set -e -o pipefail -# shellcheck source=../../shell/realpath.bash +# shellcheck source=contrib/shell/realpath.bash source contrib/shell/realpath.bash -# shellcheck source=../../shell/git-utils.bash +# shellcheck source=contrib/shell/git-utils.bash source contrib/shell/git-utils.bash ################ diff --git a/src/banman.h b/src/banman.h index 1c534f653f..e7eef39f85 100644 --- a/src/banman.h +++ b/src/banman.h @@ -97,4 +97,4 @@ private: CRollingBloomFilter m_discouraged GUARDED_BY(m_cs_banned) {50000, 0.000001}; }; -#endif +#endif // BITCOIN_BANMAN_H diff --git a/src/batchedlogger.h b/src/batchedlogger.h index 6f2b13bf9b..d653ccf3fd 100644 --- a/src/batchedlogger.h +++ b/src/batchedlogger.h @@ -35,4 +35,4 @@ public: void Flush(); }; -#endif//BITCOIN_BATCHEDLOGGER_H +#endif // BITCOIN_BATCHEDLOGGER_H diff --git a/src/coinjoin/common.h b/src/coinjoin/common.h index def3ffc83a..7b7c939f11 100644 --- a/src/coinjoin/common.h +++ b/src/coinjoin/common.h @@ -132,4 +132,4 @@ constexpr int CalculateAmountPriority(CAmount nInputAmount) } // namespace CoinJoin -#endif +#endif // BITCOIN_COINJOIN_COMMON_H diff --git a/src/consensus/amount.h b/src/consensus/amount.h index 96566ea13f..59b8e3417a 100644 --- a/src/consensus/amount.h +++ b/src/consensus/amount.h @@ -26,4 +26,4 @@ static constexpr CAmount COIN = 100000000; static constexpr CAmount MAX_MONEY = 21000000 * COIN; inline bool MoneyRange(const CAmount& nValue) { return (nValue >= 0 && nValue <= MAX_MONEY); } -#endif // BITCOIN_CONSENSUS_AMOUNT_H +#endif // BITCOIN_CONSENSUS_AMOUNT_H diff --git a/src/context.h b/src/context.h index d792df2944..39da4ec9a8 100644 --- a/src/context.h +++ b/src/context.h @@ -33,4 +33,4 @@ T* GetContext(const CoreContext& context) noexcept : nullptr; } -#endif // BITCOIN_CONTEXT_VARIANT_H +#endif // BITCOIN_CONTEXT_H diff --git a/src/evo/creditpool.h b/src/evo/creditpool.h index 4d4169974d..de6f13595b 100644 --- a/src/evo/creditpool.h +++ b/src/evo/creditpool.h @@ -141,4 +141,4 @@ std::optional GetCreditPoolDiffForBlock(CCreditPoolManager& cpo const CBlock& block, const CBlockIndex* pindexPrev, const Consensus::Params& consensusParams, const CAmount blockSubsidy, BlockValidationState& state); -#endif +#endif // BITCOIN_EVO_CREDITPOOL_H diff --git a/src/evo/dmnstate.h b/src/evo/dmnstate.h index acee5c2b49..a9eef0b555 100644 --- a/src/evo/dmnstate.h +++ b/src/evo/dmnstate.h @@ -377,4 +377,4 @@ public: }; -#endif //BITCOIN_EVO_DMNSTATE_H +#endif // BITCOIN_EVO_DMNSTATE_H diff --git a/src/governance/common.h b/src/governance/common.h index ae053d8504..48606f39b7 100644 --- a/src/governance/common.h +++ b/src/governance/common.h @@ -81,6 +81,6 @@ public: } } }; - } // namespace Governance -#endif + +#endif // BITCOIN_GOVERNANCE_COMMON_H diff --git a/src/httprpc.h b/src/httprpc.h index 18f8551816..d85a46f025 100644 --- a/src/httprpc.h +++ b/src/httprpc.h @@ -31,4 +31,4 @@ void InterruptREST(); */ void StopREST(); -#endif +#endif // BITCOIN_HTTPRPC_H diff --git a/src/i2p.h b/src/i2p.h index 6183885435..f9619de9c5 100644 --- a/src/i2p.h +++ b/src/i2p.h @@ -286,4 +286,4 @@ private: } // namespace sam } // namespace i2p -#endif /* BITCOIN_I2P_H */ +#endif // BITCOIN_I2P_H diff --git a/src/llmq/snapshot.h b/src/llmq/snapshot.h index f02698704a..e1e32751c5 100644 --- a/src/llmq/snapshot.h +++ b/src/llmq/snapshot.h @@ -233,4 +233,4 @@ extern std::unique_ptr quorumSnapshotManager; } // namespace llmq -#endif //BITCOIN_LLMQ_SNAPSHOT_H +#endif // BITCOIN_LLMQ_SNAPSHOT_H diff --git a/src/node/psbt.h b/src/node/psbt.h index fd2f063351..8f41ec2291 100644 --- a/src/node/psbt.h +++ b/src/node/psbt.h @@ -50,4 +50,4 @@ struct PSBTAnalysis { */ PSBTAnalysis AnalyzePSBT(PartiallySignedTransaction psbtx); -#endif // BITCOIN_PSBT_H +#endif // BITCOIN_NODE_PSBT_H diff --git a/src/policy/feerate.h b/src/policy/feerate.h index df700dfed5..f0d82a7402 100644 --- a/src/policy/feerate.h +++ b/src/policy/feerate.h @@ -66,4 +66,4 @@ public: SERIALIZE_METHODS(CFeeRate, obj) { READWRITE(obj.nSatoshisPerK); } }; -#endif // BITCOIN_POLICY_FEERATE_H +#endif // BITCOIN_POLICY_FEERATE_H diff --git a/src/randomenv.h b/src/randomenv.h index 46cea6f6f2..746516b79b 100644 --- a/src/randomenv.h +++ b/src/randomenv.h @@ -14,4 +14,4 @@ void RandAddDynamicEnv(CSHA512& hasher); /** Gather non-cryptographic environment data that does not change over time. */ void RandAddStaticEnv(CSHA512& hasher); -#endif +#endif // BITCOIN_RANDOMENV_H diff --git a/src/rpc/blockchain.h b/src/rpc/blockchain.h index e383aba436..15a1d0e723 100644 --- a/src/rpc/blockchain.h +++ b/src/rpc/blockchain.h @@ -64,4 +64,4 @@ void TxToUniv(const CTransaction& tx, const uint256& hashBlock, UniValue& entry, */ UniValue CreateUTXOSnapshot(NodeContext& node, CChainState& chainstate, CAutoFile& afile); -#endif +#endif // BITCOIN_RPC_BLOCKCHAIN_H diff --git a/src/rpc/index_util.h b/src/rpc/index_util.h index 6bd3b73124..a2a632c8d1 100644 --- a/src/rpc/index_util.h +++ b/src/rpc/index_util.h @@ -42,4 +42,4 @@ bool GetTimestampIndex(CBlockTreeDB& block_tree_db, const uint32_t high, const u std::vector& hashes) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); -#endif // BITCOIN_RPC_CLIENT_H +#endif // BITCOIN_RPC_INDEX_UTIL_H diff --git a/src/scheduler.h b/src/scheduler.h index 438fb8c126..727270e195 100644 --- a/src/scheduler.h +++ b/src/scheduler.h @@ -152,4 +152,4 @@ public: size_t CallbacksPending() EXCLUSIVE_LOCKS_REQUIRED(!m_callbacks_mutex); }; -#endif +#endif // BITCOIN_SCHEDULER_H diff --git a/src/shutdown.h b/src/shutdown.h index 31a0b441a4..3e22bfd0f0 100644 --- a/src/shutdown.h +++ b/src/shutdown.h @@ -36,4 +36,4 @@ bool RestartRequested(); */ void WaitForShutdown(); -#endif +#endif // BITCOIN_SHUTDOWN_H diff --git a/src/span.h b/src/span.h index 84a3f4b140..a7ebd48cc7 100644 --- a/src/span.h +++ b/src/span.h @@ -295,4 +295,4 @@ template return span.end(); } -#endif +#endif // BITCOIN_SPAN_H diff --git a/src/stacktraces.h b/src/stacktraces.h index 01ae135ee8..e6f577a818 100644 --- a/src/stacktraces.h +++ b/src/stacktraces.h @@ -37,4 +37,4 @@ inline std::string GetExceptionWhat(const T& e) void RegisterPrettyTerminateHander(); void RegisterPrettySignalHandlers(); -#endif//BITCOIN_STACKTRACES_H +#endif // BITCOIN_STACKTRACES_H diff --git a/src/test/scriptnum10.h b/src/test/scriptnum10.h index 352797f18d..3937366f01 100644 --- a/src/test/scriptnum10.h +++ b/src/test/scriptnum10.h @@ -179,4 +179,4 @@ private: }; -#endif // BITCOIN_TEST_BIGNUM_H +#endif // BITCOIN_TEST_SCRIPTNUM10_H diff --git a/src/test/util/setup_common.h b/src/test/util/setup_common.h index f21205cff2..639c96ffe7 100644 --- a/src/test/util/setup_common.h +++ b/src/test/util/setup_common.h @@ -250,4 +250,4 @@ private: // define an implicit conversion here so that uint256 may be used directly in BOOST_CHECK_* std::ostream& operator<<(std::ostream& os, const uint256& num); -#endif +#endif // BITCOIN_TEST_UTIL_SETUP_COMMON_H diff --git a/src/threadinterrupt.h b/src/threadinterrupt.h index c053c01499..bf7945a0a1 100644 --- a/src/threadinterrupt.h +++ b/src/threadinterrupt.h @@ -32,4 +32,4 @@ private: std::atomic flag; }; -#endif //BITCOIN_THREADINTERRUPT_H +#endif // BITCOIN_THREADINTERRUPT_H diff --git a/src/torcontrol.h b/src/torcontrol.h index 75464b148e..97dadd88fd 100644 --- a/src/torcontrol.h +++ b/src/torcontrol.h @@ -159,4 +159,4 @@ public: static void reconnect_cb(evutil_socket_t fd, short what, void *arg); }; -#endif /* BITCOIN_TORCONTROL_H */ +#endif // BITCOIN_TORCONTROL_H diff --git a/src/util/edge.h b/src/util/edge.h index 1137ce624a..8e3ec357a1 100644 --- a/src/util/edge.h +++ b/src/util/edge.h @@ -59,4 +59,4 @@ private: int m_fd{-1}; }; -#endif /* BITCOIN_UTIL_EDGE_H */ +#endif // BITCOIN_UTIL_EDGE_H diff --git a/src/util/readwritefile.h b/src/util/readwritefile.h index 1dab874b38..a59d0be131 100644 --- a/src/util/readwritefile.h +++ b/src/util/readwritefile.h @@ -25,4 +25,4 @@ std::pair ReadBinaryFile(const fs::path &filename, size_t maxs */ bool WriteBinaryFile(const fs::path &filename, const std::string &data); -#endif /* BITCOIN_UTIL_READWRITEFILE_H */ +#endif // BITCOIN_UTIL_READWRITEFILE_H diff --git a/src/util/trace.h b/src/util/trace.h index ec468330d7..d0fd841edf 100644 --- a/src/util/trace.h +++ b/src/util/trace.h @@ -55,4 +55,4 @@ #endif -#endif /* BITCOIN_UTIL_TRACE_H */ +#endif // BITCOIN_UTIL_TRACE_H diff --git a/src/util/underlying.h b/src/util/underlying.h index aa036b18fb..55821bb742 100644 --- a/src/util/underlying.h +++ b/src/util/underlying.h @@ -11,4 +11,4 @@ template return static_cast::type>(e); } -#endif //BITCOIN_UTIL_UNDERLYING_H +#endif // BITCOIN_UTIL_UNDERLYING_H diff --git a/src/util/wpipe.h b/src/util/wpipe.h index c4fff558ec..30912149b4 100644 --- a/src/util/wpipe.h +++ b/src/util/wpipe.h @@ -56,4 +56,4 @@ private: EdgeTriggeredEvents* m_edge_trig_events{nullptr}; }; -#endif /* BITCOIN_UTIL_WPIPE_H */ +#endif // BITCOIN_UTIL_WPIPE_H diff --git a/src/wallet/rpcwallet.h b/src/wallet/rpcwallet.h index 5c8c643a43..ce3d5a659f 100644 --- a/src/wallet/rpcwallet.h +++ b/src/wallet/rpcwallet.h @@ -41,4 +41,4 @@ RPCHelpMan getaddressinfo(); RPCHelpMan getrawchangeaddress(); RPCHelpMan addmultisigaddress(); RPCHelpMan signrawtransactionwithwallet(); -#endif //BITCOIN_WALLET_RPCWALLET_H +#endif // BITCOIN_WALLET_RPCWALLET_H diff --git a/src/warnings.h b/src/warnings.h index c38edb4570..7ab0a93e3f 100644 --- a/src/warnings.h +++ b/src/warnings.h @@ -20,4 +20,4 @@ void SetfLargeWorkInvalidChainFound(bool flag); */ bilingual_str GetWarnings(bool verbose); -#endif // BITCOIN_WARNINGS_H +#endif // BITCOIN_WARNINGS_H diff --git a/src/zmq/zmqrpc.h b/src/zmq/zmqrpc.h index 5a810a16fb..8538adf9d3 100644 --- a/src/zmq/zmqrpc.h +++ b/src/zmq/zmqrpc.h @@ -9,4 +9,4 @@ class CRPCTable; void RegisterZMQRPCCommands(CRPCTable& t); -#endif // BITCOIN_ZMQ_ZMRRPC_H +#endif // BITCOIN_ZMQ_ZMQRPC_H diff --git a/test/README.md b/test/README.md index 3e69765074..4802ed7007 100644 --- a/test/README.md +++ b/test/README.md @@ -310,11 +310,11 @@ Use the `-v` option for verbose output. | Lint test | Dependency | |-----------|:----------:| -| [`lint-python.sh`](lint/lint-python.sh) | [flake8](https://gitlab.com/pycqa/flake8) -| [`lint-python.sh`](lint/lint-python.sh) | [mypy](https://github.com/python/mypy) -| [`lint-python.sh`](lint/lint-python.sh) | [pyzmq](https://github.com/zeromq/pyzmq) +| [`lint-python.py`](lint/lint-python.py) | [flake8](https://gitlab.com/pycqa/flake8) +| [`lint-python.py`](lint/lint-python.py) | [mypy](https://github.com/python/mypy) +| [`lint-python.py`](lint/lint-python.py) | [pyzmq](https://github.com/zeromq/pyzmq) | [`lint-python-dead-code.py`](lint/lint-python-dead-code.py) | [vulture](https://github.com/jendrikseipp/vulture) -| [`lint-shell.sh`](lint/lint-shell.sh) | [ShellCheck](https://github.com/koalaman/shellcheck) +| [`lint-shell.py`](lint/lint-shell.py) | [ShellCheck](https://github.com/koalaman/shellcheck) | [`lint-spelling.py`](lint/lint-spelling.py) | [codespell](https://github.com/codespell-project/codespell) In use versions and install instructions are available in the [CI setup](../ci/lint/04_install.sh). @@ -332,7 +332,7 @@ test/lint/lint-files.py You can run all the shell-based lint tests by running: ``` -test/lint/lint-all.sh +test/lint/all-lint.py ``` # Writing functional tests diff --git a/test/lint/README.md b/test/lint/README.md index 177356abc4..748a845e59 100644 --- a/test/lint/README.md +++ b/test/lint/README.md @@ -39,6 +39,6 @@ To do so, add the upstream repository as remote: git remote add --fetch secp256k1 https://github.com/bitcoin-core/secp256k1.git ``` -lint-all.sh +all-lint.py =========== Calls other scripts with the `lint-` prefix. diff --git a/test/lint/all-lint.py b/test/lint/all-lint.py new file mode 100755 index 0000000000..40274fcc41 --- /dev/null +++ b/test/lint/all-lint.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2017-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# This script runs all test/lint/lint-* files, and fails if any exit +# with a non-zero status code. + +from glob import glob +from os import path as os_path, remove +from pathlib import Path +from shutil import which +from subprocess import run + +exit_code = 0 +mod_path = Path(__file__).parent +lints = glob(f"{mod_path}/lint-*") +if which("parallel") and which("column"): + logfile = "parallel_out.log" + command = ["parallel", "--jobs", "100%", "--will-cite", "--joblog", logfile, ":::"] + lints + result = run(command) + if result.returncode != 0: + print(f"^---- failure generated") + exit_code = result.returncode + result = run(["column", "-t", logfile]) + if os_path.isfile(logfile): + remove(logfile) +else: + for lint in lints: + result = run([lint]) + if result.returncode != 0: + print(f"^---- failure generated from {lint.split('/')[-1]}") + exit_code |= result.returncode + +exit(exit_code) diff --git a/test/lint/lint-all.sh b/test/lint/lint-all.sh deleted file mode 100755 index 3472a09908..0000000000 --- a/test/lint/lint-all.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2017-2019 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# This script runs all contrib/devtools/lint-* files, and fails if any exit -# with a non-zero status code. - -# This script is intentionally locale dependent by not setting "export LC_ALL=C" -# in order to allow for the executed lint scripts to opt in or opt out of locale -# dependence themselves. - -set -u - -SCRIPTDIR=$(dirname "${BASH_SOURCE[0]}") -LINTALL=$(basename "${BASH_SOURCE[0]}") - -EXIT_CODE=0 - -if ! command -v parallel > /dev/null; then - for f in "${SCRIPTDIR}"/lint-*; do - if [ "$(basename "$f")" != "$LINTALL" ]; then - if ! "$f"; then - echo "^---- failure generated from $f" - EXIT_CODE=1 - fi - fi - done -else - SCRIPTS=() - - for f in "${SCRIPTDIR}"/lint-*; do - if [ "$(basename "$f")" != "$LINTALL" ]; then - SCRIPTS+=("$f") - fi - done - - if ! parallel --jobs 100% --will-cite --joblog parallel_out.log ::: "${SCRIPTS[@]}"; then - echo "^---- failure generated" - EXIT_CODE=1 - fi - column -t parallel_out.log && rm parallel_out.log -fi - -exit ${EXIT_CODE} diff --git a/test/lint/lint-circular-dependencies.py b/test/lint/lint-circular-dependencies.py new file mode 100755 index 0000000000..74782e401e --- /dev/null +++ b/test/lint/lint-circular-dependencies.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2020-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Check for circular dependencies + +import glob +import os +import re +import subprocess +import sys + +EXPECTED_CIRCULAR_DEPENDENCIES = ( + "chainparamsbase -> util/system -> chainparamsbase", + "node/blockstorage -> validation -> node/blockstorage", + "index/coinstatsindex -> node/coinstats -> index/coinstatsindex", + "policy/fees -> txmempool -> policy/fees", + "qt/addresstablemodel -> qt/walletmodel -> qt/addresstablemodel", + "qt/recentrequeststablemodel -> qt/walletmodel -> qt/recentrequeststablemodel", + "qt/transactiontablemodel -> qt/walletmodel -> qt/transactiontablemodel", + "wallet/fees -> wallet/wallet -> wallet/fees", + "wallet/wallet -> wallet/walletdb -> wallet/wallet", + "node/coinstats -> validation -> node/coinstats", + # Dash + "banman -> common/bloom -> evo/assetlocktx -> llmq/quorums -> net -> banman", + "banman -> common/bloom -> evo/assetlocktx -> llmq/signing -> net_processing -> banman", + "coinjoin/client -> net_processing -> coinjoin/client", + "coinjoin/client -> net_processing -> coinjoin/context -> coinjoin/client", + "coinjoin/coinjoin -> llmq/chainlocks -> net -> coinjoin/coinjoin", + "coinjoin/context -> coinjoin/server -> net_processing -> coinjoin/context", + "coinjoin/server -> net_processing -> coinjoin/server", + "common/bloom -> evo/assetlocktx -> llmq/quorums -> net -> common/bloom", + "common/bloom -> evo/assetlocktx -> llmq/signing -> net_processing -> merkleblock -> common/bloom", + "consensus/tx_verify -> evo/assetlocktx -> validation -> consensus/tx_verify", + "consensus/tx_verify -> evo/assetlocktx -> validation -> txmempool -> consensus/tx_verify", + "core_io -> evo/cbtx -> evo/simplifiedmns -> core_io", + "dsnotificationinterface -> llmq/chainlocks -> node/blockstorage -> dsnotificationinterface", + "evo/assetlocktx -> validation -> txmempool -> evo/assetlocktx", + "evo/cbtx -> evo/simplifiedmns -> evo/cbtx", + "evo/chainhelper -> evo/specialtxman -> validation -> evo/chainhelper", + "evo/deterministicmns -> llmq/commitment -> evo/deterministicmns", + "evo/deterministicmns -> llmq/utils -> evo/deterministicmns", + "evo/deterministicmns -> llmq/utils -> llmq/snapshot -> evo/simplifiedmns -> evo/deterministicmns", + "evo/deterministicmns -> llmq/utils -> net -> evo/deterministicmns", + "evo/deterministicmns -> validation -> evo/deterministicmns", + "evo/deterministicmns -> validation -> txmempool -> evo/deterministicmns", + "evo/deterministicmns -> validationinterface -> evo/deterministicmns", + "evo/deterministicmns -> validationinterface -> governance/vote -> evo/deterministicmns", + "evo/mnhftx -> validation -> evo/mnhftx", + "evo/simplifiedmns -> llmq/blockprocessor -> llmq/utils -> llmq/snapshot -> evo/simplifiedmns", + "evo/specialtxman -> validation -> evo/specialtxman", + "governance/governance -> governance/object -> governance/governance", + "governance/governance -> masternode/sync -> governance/governance", + "governance/governance -> net_processing -> governance/governance", + "governance/governance -> validation -> governance/governance", + "governance/vote -> masternode/node -> validationinterface -> governance/vote", + "llmq/blockprocessor -> llmq/utils -> llmq/snapshot -> llmq/blockprocessor", + "llmq/chainlocks -> llmq/instantsend -> llmq/chainlocks", + "llmq/chainlocks -> llmq/instantsend -> net_processing -> llmq/chainlocks", + "llmq/chainlocks -> validation -> llmq/chainlocks", + "llmq/commitment -> llmq/utils -> llmq/snapshot -> llmq/commitment", + "llmq/context -> llmq/instantsend -> net_processing -> llmq/context", + "llmq/dkgsession -> llmq/dkgsessionmgr -> llmq/dkgsessionhandler -> llmq/dkgsession", + "llmq/dkgsessionhandler -> net_processing -> llmq/dkgsessionmgr -> llmq/dkgsessionhandler", + "llmq/instantsend -> net_processing -> llmq/instantsend", + "llmq/instantsend -> txmempool -> llmq/instantsend", + "llmq/instantsend -> validation -> llmq/instantsend", + "llmq/signing -> llmq/signing_shares -> llmq/signing", + "llmq/signing -> masternode/node -> validationinterface -> llmq/signing", + "llmq/signing -> net_processing -> llmq/signing", + "llmq/signing_shares -> net_processing -> llmq/signing_shares", + "logging -> util/system -> logging", + "logging -> util/system -> stacktraces -> logging", + "logging -> util/system -> sync -> logging", + "logging -> util/system -> sync -> logging/timer -> logging", + "logging -> util/system -> util/getuniquepath -> random -> logging", + "masternode/payments -> validation -> masternode/payments", + "masternode/sync -> validation -> masternode/sync", + "net -> netmessagemaker -> net", + "net_processing -> spork -> net_processing", + "netaddress -> netbase -> netaddress", + "policy/policy -> policy/settings -> policy/policy", + "qt/appearancewidget -> qt/guiutil -> qt/appearancewidget", + "qt/appearancewidget -> qt/guiutil -> qt/optionsdialog -> qt/appearancewidget", + "qt/bitcoinaddressvalidator -> qt/guiutil -> qt/bitcoinaddressvalidator", + "qt/bitcoingui -> qt/guiutil -> qt/bitcoingui", + "qt/guiutil -> qt/optionsdialog -> qt/guiutil", + "qt/guiutil -> qt/optionsdialog -> qt/optionsmodel -> qt/guiutil", + "qt/guiutil -> qt/qvalidatedlineedit -> qt/guiutil", + "rpc/blockchain -> rpc/server -> rpc/blockchain" +) + +CODE_DIR = "src" + + +def main(): + circular_dependencies = [] + exit_code = 0 + os.chdir( + CODE_DIR + ) # We change dir before globbing since glob.glob's root_dir option is only available in Python 3.10 + + # Using glob.glob since subprocess.run's globbing won't work without shell=True + files = [] + for path in ["*", "*/*", "*/*/*"]: + for extension in ["h", "cpp"]: + files.extend(glob.glob(f"{path}.{extension}")) + + command = ["python3", "../contrib/devtools/circular-dependencies.py", *files] + dependencies_output = subprocess.run( + command, + stdout=subprocess.PIPE, + universal_newlines=True, + ) + + for dependency_str in dependencies_output.stdout.rstrip().split("\n"): + circular_dependencies.append( + re.sub("^Circular dependency: ", "", dependency_str) + ) + + # Check for an unexpected dependencies + for dependency in circular_dependencies: + if dependency not in EXPECTED_CIRCULAR_DEPENDENCIES: + exit_code = 1 + print( + f'A new circular dependency in the form of "{dependency}" appears to have been introduced.\n', + file=sys.stderr, + ) + + # Check for missing expected dependencies + for expected_dependency in EXPECTED_CIRCULAR_DEPENDENCIES: + if expected_dependency not in circular_dependencies: + exit_code = 1 + print( + f'Good job! The circular dependency "{expected_dependency}" is no longer present.', + ) + print( + f"Please remove it from EXPECTED_CIRCULAR_DEPENDENCIES in {__file__}", + ) + print( + "to make sure this circular dependency is not accidentally reintroduced.\n", + ) + + sys.exit(exit_code) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-circular-dependencies.sh b/test/lint/lint-circular-dependencies.sh deleted file mode 100755 index 3516033753..0000000000 --- a/test/lint/lint-circular-dependencies.sh +++ /dev/null @@ -1,133 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Check for circular dependencies - -export LC_ALL=C - -EXPECTED_CIRCULAR_DEPENDENCIES=( - "chainparamsbase -> util/system -> chainparamsbase" - "node/blockstorage -> validation -> node/blockstorage" - "index/coinstatsindex -> node/coinstats -> index/coinstatsindex" - "policy/fees -> txmempool -> policy/fees" - "qt/addresstablemodel -> qt/walletmodel -> qt/addresstablemodel" - "qt/recentrequeststablemodel -> qt/walletmodel -> qt/recentrequeststablemodel" - "qt/transactiontablemodel -> qt/walletmodel -> qt/transactiontablemodel" - "wallet/fees -> wallet/wallet -> wallet/fees" - "wallet/wallet -> wallet/walletdb -> wallet/wallet" - "node/coinstats -> validation -> node/coinstats" - # Dash - "dsnotificationinterface -> llmq/chainlocks -> node/blockstorage -> dsnotificationinterface" - "evo/cbtx -> evo/simplifiedmns -> evo/cbtx" - "evo/deterministicmns -> llmq/commitment -> evo/deterministicmns" - "evo/deterministicmns -> llmq/utils -> evo/deterministicmns" - "governance/governance -> governance/object -> governance/governance" - "governance/governance -> masternode/sync -> governance/governance" - "llmq/chainlocks -> llmq/instantsend -> llmq/chainlocks" - "llmq/dkgsessionhandler -> net_processing -> llmq/dkgsessionmgr -> llmq/dkgsessionhandler" - "llmq/instantsend -> net_processing -> llmq/instantsend" - "llmq/instantsend -> txmempool -> llmq/instantsend" - "llmq/instantsend -> validation -> llmq/instantsend" - "llmq/signing -> llmq/signing_shares -> llmq/signing" - "llmq/signing -> net_processing -> llmq/signing" - "llmq/signing_shares -> net_processing -> llmq/signing_shares" - "logging -> util/system -> logging" - "masternode/payments -> validation -> masternode/payments" - "masternode/sync -> validation -> masternode/sync" - "net -> netmessagemaker -> net" - "netaddress -> netbase -> netaddress" - "qt/appearancewidget -> qt/guiutil -> qt/appearancewidget" - "qt/bitcoinaddressvalidator -> qt/guiutil -> qt/bitcoinaddressvalidator" - "qt/bitcoingui -> qt/guiutil -> qt/bitcoingui" - "qt/guiutil -> qt/optionsdialog -> qt/guiutil" - "qt/guiutil -> qt/qvalidatedlineedit -> qt/guiutil" - "core_io -> evo/cbtx -> evo/simplifiedmns -> core_io" - "llmq/dkgsession -> llmq/dkgsessionmgr -> llmq/dkgsessionhandler -> llmq/dkgsession" - "logging -> util/system -> sync -> logging" - "logging -> util/system -> stacktraces -> logging" - "logging -> util/system -> util/getuniquepath -> random -> logging" - "qt/appearancewidget -> qt/guiutil -> qt/optionsdialog -> qt/appearancewidget" - "qt/guiutil -> qt/optionsdialog -> qt/optionsmodel -> qt/guiutil" - - "common/bloom -> evo/assetlocktx -> llmq/quorums -> net -> common/bloom" - "common/bloom -> evo/assetlocktx -> llmq/signing -> net_processing -> merkleblock -> common/bloom" - "banman -> common/bloom -> evo/assetlocktx -> llmq/quorums -> net -> banman" - "banman -> common/bloom -> evo/assetlocktx -> llmq/signing -> net_processing -> banman" - - "llmq/chainlocks -> validation -> llmq/chainlocks" - "coinjoin/coinjoin -> llmq/chainlocks -> net -> coinjoin/coinjoin" - "evo/assetlocktx -> validation -> txmempool -> evo/assetlocktx" - "evo/deterministicmns -> llmq/utils -> llmq/snapshot -> evo/simplifiedmns -> evo/deterministicmns" - "evo/deterministicmns -> llmq/utils -> net -> evo/deterministicmns" - "evo/deterministicmns -> validation -> txmempool -> evo/deterministicmns" - "policy/policy -> policy/settings -> policy/policy" - "consensus/tx_verify -> evo/assetlocktx -> validation -> consensus/tx_verify" - "consensus/tx_verify -> evo/assetlocktx -> validation -> txmempool -> consensus/tx_verify" - - "evo/simplifiedmns -> llmq/blockprocessor -> llmq/utils -> llmq/snapshot -> evo/simplifiedmns" - "llmq/blockprocessor -> llmq/utils -> llmq/snapshot -> llmq/blockprocessor" - "llmq/commitment -> llmq/utils -> llmq/snapshot -> llmq/commitment" - "governance/governance -> validation -> governance/governance" - "evo/deterministicmns -> validationinterface -> governance/vote -> evo/deterministicmns" - "governance/vote -> masternode/node -> validationinterface -> governance/vote" - "llmq/signing -> masternode/node -> validationinterface -> llmq/signing" - "evo/mnhftx -> validation -> evo/mnhftx" - "evo/deterministicmns -> validation -> evo/deterministicmns" - "evo/specialtxman -> validation -> evo/specialtxman" - "evo/chainhelper -> evo/specialtxman -> validation -> evo/chainhelper" - "evo/deterministicmns -> validationinterface -> evo/deterministicmns" - "logging -> util/system -> sync -> logging/timer -> logging" - - "coinjoin/client -> net_processing -> coinjoin/client" - "coinjoin/client -> net_processing -> coinjoin/context -> coinjoin/client" - "coinjoin/context -> coinjoin/server -> net_processing -> coinjoin/context" - "coinjoin/server -> net_processing -> coinjoin/server" - "llmq/context -> llmq/instantsend -> net_processing -> llmq/context" - "llmq/chainlocks -> llmq/instantsend -> net_processing -> llmq/chainlocks" - "net_processing -> spork -> net_processing" - "governance/governance -> net_processing -> governance/governance" - "rpc/blockchain -> rpc/server -> rpc/blockchain" -) - -EXIT_CODE=0 - -CIRCULAR_DEPENDENCIES=() - -IFS=$'\n' -for CIRC in $(cd src && ../contrib/devtools/circular-dependencies.py {*,*/*,*/*/*}.{h,cpp} | sed -e 's/^Circular dependency: //'); do - CIRCULAR_DEPENDENCIES+=( "$CIRC" ) - IS_EXPECTED_CIRC=0 - for EXPECTED_CIRC in "${EXPECTED_CIRCULAR_DEPENDENCIES[@]}"; do - if [[ "${CIRC}" == "${EXPECTED_CIRC}" ]]; then - IS_EXPECTED_CIRC=1 - break - fi - done - if [[ ${IS_EXPECTED_CIRC} == 0 ]]; then - echo "A new circular dependency in the form of \"${CIRC}\" appears to have been introduced." - echo - EXIT_CODE=1 - fi -done - -for EXPECTED_CIRC in "${EXPECTED_CIRCULAR_DEPENDENCIES[@]}"; do - IS_PRESENT_EXPECTED_CIRC=0 - for CIRC in "${CIRCULAR_DEPENDENCIES[@]}"; do - if [[ "${CIRC}" == "${EXPECTED_CIRC}" ]]; then - IS_PRESENT_EXPECTED_CIRC=1 - break - fi - done - if [[ ${IS_PRESENT_EXPECTED_CIRC} == 0 ]]; then - echo "Good job! The circular dependency \"${EXPECTED_CIRC}\" is no longer present." - echo "Please remove it from EXPECTED_CIRCULAR_DEPENDENCIES in $0" - echo "to make sure this circular dependency is not accidentally reintroduced." - echo - EXIT_CODE=1 - fi -done - -exit ${EXIT_CODE} diff --git a/test/lint/lint-include-guards.py b/test/lint/lint-include-guards.py new file mode 100755 index 0000000000..8fd4c7d636 --- /dev/null +++ b/test/lint/lint-include-guards.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" +Check include guards. +""" + +import re +import sys +from subprocess import check_output +from typing import List + + +HEADER_ID_PREFIX = 'BITCOIN_' +HEADER_ID_SUFFIX = '_H' + +EXCLUDE_FILES_WITH_PREFIX = ['src/crypto/ctaes', + 'src/leveldb', + 'src/crc32c', + 'src/secp256k1', + 'src/minisketch', + 'src/univalue', + 'src/tinyformat.h', + 'src/bench/nanobench.h', + 'src/test/fuzz/FuzzedDataProvider.h', + 'src/bls', + 'src/crypto/x11/sph', + 'src/ctpl_stl.h', + 'src/dashbls', + 'src/gsl', + 'src/immer', + 'src/util/expected.h'] + + +def _get_header_file_lst() -> List[str]: + """ Helper function to get a list of header filepaths to be + checked for include guards. + """ + git_cmd_lst = ['git', 'ls-files', '--', '*.h'] + header_file_lst = check_output( + git_cmd_lst).decode('utf-8').splitlines() + + header_file_lst = [hf for hf in header_file_lst + if not any(ef in hf for ef + in EXCLUDE_FILES_WITH_PREFIX)] + + return header_file_lst + + +def _get_header_id(header_file: str) -> str: + """ Helper function to get the header id from a header file + string. + + eg: 'src/wallet/walletdb.h' -> 'BITCOIN_WALLET_WALLETDB_H' + + Args: + header_file: Filepath to header file. + + Returns: + The header id. + """ + header_id_base = header_file.split('/')[1:] + header_id_base = '_'.join(header_id_base) + header_id_base = header_id_base.replace('.h', '').replace('-', '_') + header_id_base = header_id_base.upper() + + header_id = f'{HEADER_ID_PREFIX}{header_id_base}{HEADER_ID_SUFFIX}' + + return header_id + + +def main(): + exit_code = 0 + + header_file_lst = _get_header_file_lst() + for header_file in header_file_lst: + header_id = _get_header_id(header_file) + + regex_pattern = f'^#(ifndef|define|endif //) {header_id}' + + with open(header_file, 'r', encoding='utf-8') as f: + header_file_contents = f.readlines() + + count = 0 + for header_file_contents_string in header_file_contents: + include_guard_lst = re.findall( + regex_pattern, header_file_contents_string) + + count += len(include_guard_lst) + + if count != 3: + print(f'{header_file} seems to be missing the expected ' + 'include guard:') + print(f' #ifndef {header_id}') + print(f' #define {header_id}') + print(' ...') + print(f' #endif // {header_id}\n') + exit_code = 1 + + sys.exit(exit_code) + + +if __name__ == '__main__': + main() diff --git a/test/lint/lint-include-guards.sh b/test/lint/lint-include-guards.sh deleted file mode 100755 index e88680aec0..0000000000 --- a/test/lint/lint-include-guards.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Check include guards. - -export LC_ALL=C -HEADER_ID_PREFIX="BITCOIN_" -HEADER_ID_SUFFIX="_H" - -REGEXP_EXCLUDE_FILES_WITH_PREFIX="src/(crypto/ctaes/|dashbls/|immer/|leveldb/|crc32c/|secp256k1/|minisketch/|test/fuzz/FuzzedDataProvider.h|tinyformat.h|bench/nanobench.h|univalue/|ctpl_stl.h|bls/|crypto/x11/sph|gsl|util/expected.h)" - -EXIT_CODE=0 -for HEADER_FILE in $(git ls-files -- "*.h" | grep -vE "^${REGEXP_EXCLUDE_FILES_WITH_PREFIX}") -do - HEADER_ID_BASE=$(cut -f2- -d/ <<< "${HEADER_FILE}" | sed "s/\.h$//g" | tr / _ | tr - _ | tr "[:lower:]" "[:upper:]") - HEADER_ID="${HEADER_ID_PREFIX}${HEADER_ID_BASE}${HEADER_ID_SUFFIX}" - if [[ $(grep -cE "^#(ifndef|define) ${HEADER_ID}" "${HEADER_FILE}") != 2 ]]; then - echo "${HEADER_FILE} seems to be missing the expected include guard:" - echo " #ifndef ${HEADER_ID}" - echo " #define ${HEADER_ID}" - echo " ..." - echo " #endif // ${HEADER_ID}" - echo - EXIT_CODE=1 - fi -done -exit ${EXIT_CODE} diff --git a/test/lint/lint-python-utf8-encoding.py b/test/lint/lint-python-utf8-encoding.py new file mode 100755 index 0000000000..4f8b307c04 --- /dev/null +++ b/test/lint/lint-python-utf8-encoding.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Make sure we explicitly open all text files using UTF-8 (or ASCII) encoding to +# avoid potential issues on the BSDs where the locale is not always set. + +import sys +import re + +from subprocess import check_output, CalledProcessError + +EXCLUDED_DIRS = ["src/crc32c/", + "src/secp256k1/"] + + +def get_exclude_args(): + return [":(exclude)" + dir for dir in EXCLUDED_DIRS] + + +def check_fileopens(): + fileopens = list() + + try: + fileopens = check_output(["git", "grep", r" open(", "--", "*.py"] + get_exclude_args(), universal_newlines=True, encoding="utf8").splitlines() + except CalledProcessError as e: + if e.returncode > 1: + raise e + + filtered_fileopens = [fileopen for fileopen in fileopens if not re.search(r"encoding=.(ascii|utf8|utf-8).|open\([^,]*, ['\"][^'\"]*b[^'\"]*['\"]", fileopen)] + + return filtered_fileopens + + +def check_checked_outputs(): + checked_outputs = list() + + try: + checked_outputs = check_output(["git", "grep", "check_output(", "--", "*.py"] + get_exclude_args(), universal_newlines=True, encoding="utf8").splitlines() + except CalledProcessError as e: + if e.returncode > 1: + raise e + + filtered_checked_outputs = [checked_output for checked_output in checked_outputs if re.search(r"universal_newlines=True", checked_output) and not re.search(r"encoding=.(ascii|utf8|utf-8).", checked_output)] + + return filtered_checked_outputs + + +def main(): + exit_code = 0 + + nonexplicit_utf8_fileopens = check_fileopens() + if nonexplicit_utf8_fileopens: + print("Python's open(...) seems to be used to open text files without explicitly specifying encoding='utf8':\n") + for fileopen in nonexplicit_utf8_fileopens: + print(fileopen) + exit_code = 1 + + nonexplicit_utf8_checked_outputs = check_checked_outputs() + if nonexplicit_utf8_checked_outputs: + if nonexplicit_utf8_fileopens: + print("\n") + print("Python's check_output(...) seems to be used to get program outputs without explicitly specifying encoding='utf8':\n") + for checked_output in nonexplicit_utf8_checked_outputs: + print(checked_output) + exit_code = 1 + + sys.exit(exit_code) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-python-utf8-encoding.sh b/test/lint/lint-python-utf8-encoding.sh deleted file mode 100755 index 83c34bef7f..0000000000 --- a/test/lint/lint-python-utf8-encoding.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Make sure we explicitly open all text files using UTF-8 (or ASCII) encoding to -# avoid potential issues on the BSDs where the locale is not always set. - -export LC_ALL=C -EXIT_CODE=0 -OUTPUT=$(git grep " open(" -- "*.py" ":(exclude)src/crc32c/" ":(exclude)src/secp256k1/" | grep -vE "encoding=.(ascii|utf8|utf-8)." | grep -vE "open\([^,]*, ['\"][^'\"]*b[^'\"]*['\"]") -if [[ ${OUTPUT} != "" ]]; then - echo "Python's open(...) seems to be used to open text files without explicitly" - echo "specifying encoding=\"utf8\":" - echo - echo "${OUTPUT}" - EXIT_CODE=1 -fi -OUTPUT=$(git grep "check_output(" -- "*.py" ":(exclude)src/crc32c/" ":(exclude)src/secp256k1/" | grep "universal_newlines=True" | grep -vE "encoding=.(ascii|utf8|utf-8).") -if [[ ${OUTPUT} != "" ]]; then - echo "Python's check_output(...) seems to be used to get program outputs without explicitly" - echo "specifying encoding=\"utf8\":" - echo - echo "${OUTPUT}" - EXIT_CODE=1 -fi -exit ${EXIT_CODE} diff --git a/test/lint/lint-python.py b/test/lint/lint-python.py new file mode 100755 index 0000000000..61d2c49a46 --- /dev/null +++ b/test/lint/lint-python.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" +Check for specified flake8 and mypy warnings in python files. +""" + +import os +import pkg_resources +import subprocess +import sys + +DEPS = ['flake8', 'mypy', 'pyzmq'] +MYPY_CACHE_DIR = f"{os.getenv('BASE_ROOT_DIR', '')}/test/.mypy_cache" +FILES_ARGS = ['git', 'ls-files', '--','test/functional/*.py', 'contrib/devtools/*.py', ':(exclude)contrib/devtools/github-merge.py'] +EXCLUDE_DIRS = ['src/dashbls/', + 'src/immer/'] + +ENABLED = ( + 'E101,' # indentation contains mixed spaces and tabs + 'E112,' # expected an indented block + 'E113,' # unexpected indentation + 'E115,' # expected an indented block (comment) + 'E116,' # unexpected indentation (comment) + 'E125,' # continuation line with same indent as next logical line + 'E129,' # visually indented line with same indent as next logical line + 'E131,' # continuation line unaligned for hanging indent + 'E133,' # closing bracket is missing indentation + 'E223,' # tab before operator + 'E224,' # tab after operator + 'E242,' # tab after ',' + 'E266,' # too many leading '#' for block comment + 'E271,' # multiple spaces after keyword + 'E272,' # multiple spaces before keyword + 'E273,' # tab after keyword + 'E274,' # tab before keyword + # TODO: enable it after bitcoin/bitcoin#26257 - too many warnings with newer flake + # 'E275,' # missing whitespace after keyword + 'E304,' # blank lines found after function decorator + 'E306,' # expected 1 blank line before a nested definition + 'E401,' # multiple imports on one line + 'E402,' # module level import not at top of file + 'E502,' # the backslash is redundant between brackets + 'E701,' # multiple statements on one line (colon) + 'E702,' # multiple statements on one line (semicolon) + 'E703,' # statement ends with a semicolon + 'E711,' # comparison to None should be 'if cond is None:' + 'E714,' # test for object identity should be "is not" + 'E721,' # do not compare types, use "isinstance()" + 'E742,' # do not define classes named "l", "O", or "I" + 'E743,' # do not define functions named "l", "O", or "I" + 'E901,' # SyntaxError: invalid syntax + 'E902,' # TokenError: EOF in multi-line string + 'F401,' # module imported but unused + 'F402,' # import module from line N shadowed by loop variable + 'F403,' # 'from foo_module import *' used; unable to detect undefined names + 'F404,' # future import(s) name after other statements + 'F405,' # foo_function may be undefined, or defined from star imports: bar_module + 'F406,' # "from module import *" only allowed at module level + 'F407,' # an undefined __future__ feature name was imported + 'F601,' # dictionary key name repeated with different values + 'F602,' # dictionary key variable name repeated with different values + 'F621,' # too many expressions in an assignment with star-unpacking + 'F622,' # two or more starred expressions in an assignment (a, *b, *c = d) + 'F631,' # assertion test is a tuple, which are always True + 'F632,' # use ==/!= to compare str, bytes, and int literals + 'F701,' # a break statement outside of a while or for loop + 'F702,' # a continue statement outside of a while or for loop + 'F703,' # a continue statement in a finally block in a loop + 'F704,' # a yield or yield from statement outside of a function + 'F705,' # a return statement with arguments inside a generator + 'F706,' # a return statement outside of a function/method + 'F707,' # an except: block as not the last exception handler + 'F811,' # redefinition of unused name from line N + 'F812,' # list comprehension redefines 'foo' from line N + 'F821,' # undefined name 'Foo' + 'F822,' # undefined name name in __all__ + 'F823,' # local variable name … referenced before assignment + 'F831,' # duplicate argument name in function definition + 'F841,' # local variable 'foo' is assigned to but never used + 'W191,' # indentation contains tabs + 'W291,' # trailing whitespace + 'W292,' # no newline at end of file + 'W293,' # blank line contains whitespace + 'W601,' # .has_key() is deprecated, use "in" + 'W602,' # deprecated form of raising exception + 'W603,' # "<>" is deprecated, use "!=" + 'W604,' # backticks are deprecated, use "repr()" + # 'W605,' # invalid escape sequence "x" + 'W606,' # 'async' and 'await' are reserved keywords starting with Python 3.7 +) + + +def check_dependencies(): + working_set = {pkg.key for pkg in pkg_resources.working_set} + + for dep in DEPS: + if dep not in working_set: + print(f"Skipping Python linting since {dep} is not installed.") + exit(0) + + +def main(): + check_dependencies() + + if len(sys.argv) > 1: + flake8_files = sys.argv[1:] + else: + files_args = ['git', 'ls-files', '--', '*.py'] + for dir in EXCLUDE_DIRS: + files_args += [f':(exclude){dir}'] + flake8_files = subprocess.check_output(files_args).decode("utf-8").splitlines() + + flake8_args = ['flake8', '--ignore=B,C,E,F,I,N,W', f'--select={ENABLED}'] + flake8_files + flake8_env = os.environ.copy() + flake8_env["PYTHONWARNINGS"] = "ignore" + + try: + subprocess.check_call(flake8_args, env=flake8_env) + except subprocess.CalledProcessError: + exit(1) + + mypy_files = subprocess.check_output(FILES_ARGS).decode("utf-8").splitlines() + mypy_args = ['mypy', '--ignore-missing-imports', '--show-error-codes'] + mypy_files + + try: + subprocess.check_call(mypy_args) + except subprocess.CalledProcessError: + exit(1) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-python.sh b/test/lint/lint-python.sh deleted file mode 100755 index c7ecb11ea0..0000000000 --- a/test/lint/lint-python.sh +++ /dev/null @@ -1,120 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2017-2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Check for specified flake8 warnings in python files. - -export LC_ALL=C -export MYPY_CACHE_DIR="${BASE_ROOT_DIR}/test/.mypy_cache" - -enabled=( - E101 # indentation contains mixed spaces and tabs - E112 # expected an indented block - E113 # unexpected indentation - E115 # expected an indented block (comment) - E116 # unexpected indentation (comment) - E125 # continuation line with same indent as next logical line - E129 # visually indented line with same indent as next logical line - E131 # continuation line unaligned for hanging indent - E133 # closing bracket is missing indentation - E223 # tab before operator - E224 # tab after operator - E242 # tab after ',' - E266 # too many leading '#' for block comment - E271 # multiple spaces after keyword - E272 # multiple spaces before keyword - E273 # tab after keyword - E274 # tab before keyword - # TODO: enable it after bitcoin/bitcoin#26257 - too many warnings with newer flake - #E275 # missing whitespace after keyword - E304 # blank lines found after function decorator - E306 # expected 1 blank line before a nested definition - E401 # multiple imports on one line - E402 # module level import not at top of file - E502 # the backslash is redundant between brackets - E701 # multiple statements on one line (colon) - E702 # multiple statements on one line (semicolon) - E703 # statement ends with a semicolon - E711 # comparison to None should be 'if cond is None:' - E714 # test for object identity should be "is not" - E721 # do not compare types, use "isinstance()" - E742 # do not define classes named "l", "O", or "I" - E743 # do not define functions named "l", "O", or "I" - E901 # SyntaxError: invalid syntax - E902 # TokenError: EOF in multi-line string - F401 # module imported but unused - F402 # import module from line N shadowed by loop variable - F403 # 'from foo_module import *' used; unable to detect undefined names - F404 # future import(s) name after other statements - F405 # foo_function may be undefined, or defined from star imports: bar_module - F406 # "from module import *" only allowed at module level - F407 # an undefined __future__ feature name was imported - F601 # dictionary key name repeated with different values - F602 # dictionary key variable name repeated with different values - F621 # too many expressions in an assignment with star-unpacking - F622 # two or more starred expressions in an assignment (a, *b, *c = d) - F631 # assertion test is a tuple, which are always True - F632 # use ==/!= to compare str, bytes, and int literals - F701 # a break statement outside of a while or for loop - F702 # a continue statement outside of a while or for loop - F703 # a continue statement in a finally block in a loop - F704 # a yield or yield from statement outside of a function - F705 # a return statement with arguments inside a generator - F706 # a return statement outside of a function/method - F707 # an except: block as not the last exception handler - F811 # redefinition of unused name from line N - F812 # list comprehension redefines 'foo' from line N - F821 # undefined name 'Foo' - F822 # undefined name name in __all__ - F823 # local variable name … referenced before assignment - F831 # duplicate argument name in function definition - F841 # local variable 'foo' is assigned to but never used - W191 # indentation contains tabs - W291 # trailing whitespace - W292 # no newline at end of file - W293 # blank line contains whitespace - W601 # .has_key() is deprecated, use "in" - W602 # deprecated form of raising exception - W603 # "<>" is deprecated, use "!=" - W604 # backticks are deprecated, use "repr()" - # W605 # invalid escape sequence "x" - W606 # 'async' and 'await' are reserved keywords starting with Python 3.7 -) - -if ! command -v flake8 > /dev/null; then - echo "Skipping Python linting since flake8 is not installed." - exit 0 -elif PYTHONWARNINGS="ignore" flake8 --version | grep -q "Python 2"; then - echo "Skipping Python linting since flake8 is running under Python 2. Install the Python 3 version of flake8." - exit 0 -fi - -FLAKECMD=flake8 - -if command -v flake8-cached > /dev/null; then - FLAKECMD=flake8-cached -else - echo "Consider install flake8-cached for cached flake8 results." -fi - -EXIT_CODE=0 - -# shellcheck disable=SC2046 -if ! PYTHONWARNINGS="ignore" $FLAKECMD --ignore=B,C,E,F,I,N,W --select=$(IFS=","; echo "${enabled[*]}") $( - if [[ $# == 0 ]]; then - git ls-files "*.py" | grep -vE "src/(immer)/" - else - echo "$@" - fi -); then - EXIT_CODE=1 -fi - -mapfile -t FILES < <(git ls-files "test/functional/*.py" "contrib/devtools/*.py" | grep -v contrib/devtools/github-merge.py) -if ! mypy --ignore-missing-imports --show-error-codes "${FILES[@]}"; then - EXIT_CODE=1 -fi - -exit $EXIT_CODE diff --git a/test/lint/lint-shell-locale.py b/test/lint/lint-shell-locale.py new file mode 100755 index 0000000000..52928041cb --- /dev/null +++ b/test/lint/lint-shell-locale.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" +Make sure all shell scripts are: +a.) explicitly opt out of locale dependence using + "export LC_ALL=C" or "export LC_ALL=C.UTF-8", or +b.) explicitly opt in to locale dependence using the annotation below. +""" + +import subprocess +import sys +import re + +OPT_IN_LINE = '# This script is intentionally locale dependent by not setting \"export LC_ALL=C\"' + +OPT_OUT_LINES = [ + 'export LC_ALL=C', + 'export LC_ALL=C.UTF-8', +] + +def get_shell_files_list(): + command = [ + 'git', + 'ls-files', + '--', + '*.sh', + ] + try: + return subprocess.check_output(command, stderr = subprocess.STDOUT).decode('utf-8').splitlines() + except subprocess.CalledProcessError as e: + if e.returncode > 1: # return code is 1 when match is empty + print(e.output.decode('utf-8'), end='') + sys.exit(1) + return [] + +def main(): + exit_code = 0 + shell_files = get_shell_files_list() + for file_path in shell_files: + if re.search('src/(dashbls|secp256k1|minisketch|univalue)/', file_path): + continue + + with open(file_path, 'r', encoding='utf-8') as file_obj: + contents = file_obj.read() + + if OPT_IN_LINE in contents: + continue + + non_comment_pattern = re.compile(r'^\s*((?!#).+)$', re.MULTILINE) + non_comment_lines = re.findall(non_comment_pattern, contents) + if not non_comment_lines: + continue + + first_non_comment_line = non_comment_lines[0] + if first_non_comment_line not in OPT_OUT_LINES: + print(f'Missing "export LC_ALL=C" (to avoid locale dependence) as first non-comment non-empty line in {file_path}') + exit_code = 1 + + return sys.exit(exit_code) + +if __name__ == '__main__': + main() + diff --git a/test/lint/lint-shell-locale.sh b/test/lint/lint-shell-locale.sh deleted file mode 100755 index b21b5d9b8d..0000000000 --- a/test/lint/lint-shell-locale.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Make sure all shell scripts: -# a.) explicitly opt out of locale dependence using -# "export LC_ALL=C" or "export LC_ALL=C.UTF-8", or -# b.) explicitly opt in to locale dependence using the annotation below. - -export LC_ALL=C - -EXIT_CODE=0 -for SHELL_SCRIPT in $(git ls-files -- "*.sh" | grep -vE "src/(dashbls|secp256k1|minisketch|univalue)/"); do - if grep -q "# This script is intentionally locale dependent by not setting \"export LC_ALL=C\"" "${SHELL_SCRIPT}"; then - continue - fi - FIRST_NON_COMMENT_LINE=$(grep -vE '^(#.*)?$' "${SHELL_SCRIPT}" | head -1) - if [[ ${FIRST_NON_COMMENT_LINE} != "export LC_ALL=C" && ${FIRST_NON_COMMENT_LINE} != "export LC_ALL=C.UTF-8" ]]; then - echo "Missing \"export LC_ALL=C\" (to avoid locale dependence) as first non-comment non-empty line in ${SHELL_SCRIPT}" - EXIT_CODE=1 - fi -done -exit ${EXIT_CODE} diff --git a/test/lint/lint-shell.py b/test/lint/lint-shell.py new file mode 100755 index 0000000000..2b13bbb06e --- /dev/null +++ b/test/lint/lint-shell.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" +Check for shellcheck warnings in shell scripts. +""" + +import subprocess +import re +import sys + +# Disabled warnings: +DISABLED = [ + 'SC2046', # Quote this to prevent word splitting. + 'SC2086', # Double quote to prevent globbing and word splitting. + 'SC2162', # read without -r will mangle backslashes. +] + +def check_shellcheck_install(): + try: + subprocess.run(['shellcheck', '--version'], stdout=subprocess.DEVNULL, check=True) + except FileNotFoundError: + print('Skipping shell linting since shellcheck is not installed.') + sys.exit(0) + +def get_files(command): + output = subprocess.run(command, stdout=subprocess.PIPE, universal_newlines=True) + files = output.stdout.split('\n') + + # remove whitespace element + files = list(filter(None, files)) + return files + +def main(): + check_shellcheck_install() + + # build the `exclude` flag + exclude = '--exclude=' + ','.join(DISABLED) + + # build the `sourced files` list + sourced_files_cmd = [ + 'git', + 'grep', + '-El', + r'^# shellcheck shell=', + ] + sourced_files = get_files(sourced_files_cmd) + + # build the `guix files` list + guix_files_cmd = [ + 'git', + 'grep', + '-El', + r'^#!\/usr\/bin\/env bash', + '--', + 'contrib/guix', + 'contrib/shell', + ] + guix_files = get_files(guix_files_cmd) + + # build the other script files list + files_cmd = [ + 'git', + 'ls-files', + '--', + '*.sh', + ] + files = get_files(files_cmd) + # remove everything that doesn't match this regex + reg = re.compile(r'src/[dashbls,immer,leveldb,secp256k1,minisketch,univalue]') + files[:] = [file for file in files if not reg.match(file)] + + # build the `shellcheck` command + shellcheck_cmd = [ + 'shellcheck', + '--external-sources', + '--check-sourced', + '--source-path=SCRIPTDIR', + ] + shellcheck_cmd.append(exclude) + shellcheck_cmd.extend(sourced_files) + shellcheck_cmd.extend(guix_files) + shellcheck_cmd.extend(files) + + # run the `shellcheck` command + try: + subprocess.check_call(shellcheck_cmd) + except subprocess.CalledProcessError: + sys.exit(1) + +if __name__ == '__main__': + main() diff --git a/test/lint/lint-shell.sh b/test/lint/lint-shell.sh deleted file mode 100755 index 7741345453..0000000000 --- a/test/lint/lint-shell.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Check for shellcheck warnings in shell scripts. - -export LC_ALL=C - -# Disabled warnings: -disabled=( - SC2046 # Quote this to prevent word splitting. - SC2086 # Double quote to prevent globbing and word splitting. - SC2162 # read without -r will mangle backslashes. -) - -EXIT_CODE=0 - -if ! command -v shellcheck > /dev/null; then - echo "Skipping shell linting since shellcheck is not installed." - exit $EXIT_CODE -fi - -if ! command -v gawk > /dev/null; then - echo "Skipping shell linting since gawk is not installed." - exit $EXIT_CODE -fi - -SHELLCHECK_CMD=(shellcheck --external-sources --check-sourced) -EXCLUDE="--exclude=$(IFS=','; echo "${disabled[*]}")" -# Check shellcheck directive used for sourced files -mapfile -t SOURCED_FILES < <(git ls-files | xargs gawk '/^# shellcheck shell=/ {print FILENAME} {nextfile}') -mapfile -t FILES < <(git ls-files -- '*.sh' | grep -vE 'src/(dashbls|immer|leveldb|secp256k1|minisketch|univalue)/') -if ! "${SHELLCHECK_CMD[@]}" "$EXCLUDE" "${SOURCED_FILES[@]}" "${FILES[@]}"; then - EXIT_CODE=1 -fi - -exit $EXIT_CODE