diff --git a/ci/dash/build_src.sh b/ci/dash/build_src.sh index e940aac194..4a7be858ae 100755 --- a/ci/dash/build_src.sh +++ b/ci/dash/build_src.sh @@ -25,7 +25,7 @@ if [ "$CHECK_DOC" = 1 ]; then # TODO: Check docs (re-enable after all Bitcoin PRs have been merged and docs fully fixed) #test/lint/check-doc.py # Run all linters - test/lint/lint-all.sh + test/lint/all-lint.py fi ccache --zero-stats --max-size=$CCACHE_SIZE diff --git a/ci/lint/06_script.sh b/ci/lint/06_script.sh index edfedac64e..f4a3f3f985 100755 --- a/ci/lint/06_script.sh +++ b/ci/lint/06_script.sh @@ -21,7 +21,7 @@ test/lint/git-subtree-check.sh src/minisketch test/lint/git-subtree-check.sh src/univalue test/lint/git-subtree-check.sh src/leveldb test/lint/check-doc.py -test/lint/lint-all.sh +test/lint/all-lint.py if [ "$CIRRUS_REPO_FULL_NAME" = "dashpay/dash" ] && [ -n "$CIRRUS_CRON" ]; then git log --merges --before="2 days ago" -1 --format='%H' > ./contrib/verify-commits/trusted-sha512-root-commit diff --git a/contrib/guix/libexec/prelude.bash b/contrib/guix/libexec/prelude.bash index 859095b04d..e0d653e46c 100644 --- a/contrib/guix/libexec/prelude.bash +++ b/contrib/guix/libexec/prelude.bash @@ -2,10 +2,10 @@ export LC_ALL=C set -e -o pipefail -# shellcheck source=../../shell/realpath.bash +# shellcheck source=contrib/shell/realpath.bash source contrib/shell/realpath.bash -# shellcheck source=../../shell/git-utils.bash +# shellcheck source=contrib/shell/git-utils.bash source contrib/shell/git-utils.bash ################ diff --git a/src/banman.h b/src/banman.h index 1c534f653f..e7eef39f85 100644 --- a/src/banman.h +++ b/src/banman.h @@ -97,4 +97,4 @@ class BanMan CRollingBloomFilter m_discouraged GUARDED_BY(m_cs_banned) {50000, 0.000001}; }; -#endif +#endif // BITCOIN_BANMAN_H diff --git a/src/batchedlogger.h b/src/batchedlogger.h index 6f2b13bf9b..d653ccf3fd 100644 --- a/src/batchedlogger.h +++ b/src/batchedlogger.h @@ -35,4 +35,4 @@ class CBatchedLogger void Flush(); }; -#endif//BITCOIN_BATCHEDLOGGER_H +#endif // BITCOIN_BATCHEDLOGGER_H diff --git a/src/coinjoin/common.h b/src/coinjoin/common.h index def3ffc83a..7b7c939f11 100644 --- a/src/coinjoin/common.h +++ b/src/coinjoin/common.h @@ -132,4 +132,4 @@ constexpr int CalculateAmountPriority(CAmount nInputAmount) } // namespace CoinJoin -#endif +#endif // BITCOIN_COINJOIN_COMMON_H diff --git a/src/consensus/amount.h b/src/consensus/amount.h index 96566ea13f..59b8e3417a 100644 --- a/src/consensus/amount.h +++ b/src/consensus/amount.h @@ -26,4 +26,4 @@ static constexpr CAmount COIN = 100000000; static constexpr CAmount MAX_MONEY = 21000000 * COIN; inline bool MoneyRange(const CAmount& nValue) { return (nValue >= 0 && nValue <= MAX_MONEY); } -#endif // BITCOIN_CONSENSUS_AMOUNT_H +#endif // BITCOIN_CONSENSUS_AMOUNT_H diff --git a/src/context.h b/src/context.h index d792df2944..39da4ec9a8 100644 --- a/src/context.h +++ b/src/context.h @@ -33,4 +33,4 @@ T* GetContext(const CoreContext& context) noexcept : nullptr; } -#endif // BITCOIN_CONTEXT_VARIANT_H +#endif // BITCOIN_CONTEXT_H diff --git a/src/dashbls/.github/workflows/build-binds.yml b/src/dashbls/.github/workflows/build-binds.yml index f57acbd7e2..6fc729109d 100644 --- a/src/dashbls/.github/workflows/build-binds.yml +++ b/src/dashbls/.github/workflows/build-binds.yml @@ -23,8 +23,8 @@ jobs: fail-fast: false matrix: os: [macos-latest, ubuntu-latest] - golang: [ '1.17' ] - python: ['3.7', '3.8', '3.9', '3.10', '3.11'] + golang: [ '1.22' ] + python: ['3.9', '3.10', '3.11', '3.12'] steps: - name: Checkout code diff --git a/src/dashbls/configure.ac b/src/dashbls/configure.ac index e60fca8fc1..df411ced0c 100644 --- a/src/dashbls/configure.ac +++ b/src/dashbls/configure.ac @@ -1,5 +1,5 @@ AC_PREREQ([2.60]) -AC_INIT([libdashbls],[1.3.3]) +AC_INIT([libdashbls],[1.3.4]) AC_CONFIG_AUX_DIR([build-aux]) AC_CONFIG_MACRO_DIR([build-aux/m4]) @@ -100,7 +100,7 @@ dnl warning about something unrelated, for example about some path issue. If tha dnl -Werror cannot be used because all of those warnings would be turned into errors. AX_CHECK_COMPILE_FLAG([-Werror], [FLAG_WERROR="-Werror"], [FLAG_WERROR=""]) -if [[[ "$use_debug" == "yes" || "$use_optimizations" == "no" ]]]; then +if test x"$use_debug" = x"yes" -o x"$use_optimizations" = x"no"; then dnl Clear default -g -O2 flags if test x"$CFLAGS_overridden" = x"no"; then CFLAGS="" @@ -112,7 +112,7 @@ if [[[ "$use_debug" == "yes" || "$use_optimizations" == "no" ]]]; then dnl Disable optimizations AX_CHECK_COMPILE_FLAG([-O0], [[DEBUG_FLAGS="$DEBUG_FLAGS -O0"]], [], [[$FLAG_WERROR]]) - if [[[ "$use_debug" == "yes" ]]]; then + if test x"$use_debug" = x"yes"; then dnl Prefer -g3, fall back to -g if that is unavailable. AX_CHECK_COMPILE_FLAG( [-g3], @@ -220,7 +220,11 @@ case $host in GMP_LDFLAGS="-L$gmp_prefix/lib" fi fi - ;; + ;; + *freebsd*) + GMP_CPPFLAGS="-I/usr/local/include" + GMP_LDFLAGS="-L/usr/local/lib" + ;; esac if test x"$want_backend" = x"auto"; then @@ -432,44 +436,74 @@ fi use_pkgconfig=yes -if [[[ "$host_cpu" == x86_64 && "$use_optimizations" == "yes" ]]]; then - dnl Support for AMD64 (also known as x86_64 on some platforms) processors - CPU_ARCH="x64" - AC_DEFINE([ARCH], [X64], [Architecture.]) - AC_DEFINE([WSIZE], [64], [Size of word in this architecture.]) -elif [[[ "$host_cpu" == aarch* && "$use_optimizations" == "yes" ]]]; then - dnl Support for 64-bit ARM processors - dnl Relic doesn't support aarch64 yet, set CPU_ARCH to none and ARCH to RELIC_NONE. - CPU_ARCH="none" - AC_DEFINE([ARCH], [RELIC_NONE], [Architecture.]) - AC_DEFINE([WSIZE], [64], [Size of word in this architecture.]) -elif [[[ "$host_cpu" == i?86 && "$use_optimizations" == "yes" ]]]; then - dnl Support for Intel x86 processors - CPU_ARCH="x86" - AC_DEFINE([ARCH], [X86], [Architecture.]) - AC_DEFINE([WSIZE], [32], [Size of word in this architecture.]) -elif [[[ "$host_cpu" == arm* && "$use_optimizations" == "yes" ]]]; then - dnl Support for 32-bit native ARM processors - CPU_ARCH="arm" - AC_DEFINE([ARCH], [ARM], [Architecture.]) - AC_DEFINE([WSIZE], [32], [Size of word in this architecture.]) -elif [[[ "$host_cpu" == *64* ]]]; then - dnl Support for an undefined 64-bit architecture - CPU_ARCH="none" - AC_DEFINE([ARCH], [RELIC_NONE], [Architecture.]) - AC_DEFINE([WSIZE], [64], [Size of word in this architecture.]) -elif [[[ "$host_cpu" == *32* || "$host_cpu" == arm* || "$host_cpu" == i?86 ]]]; then - dnl Support for an undefined 32-bit architecture - CPU_ARCH="none" - AC_DEFINE([ARCH], [RELIC_NONE], [Architecture.]) - AC_DEFINE([WSIZE], [32], [Size of word in this architecture.]) +if test x"$use_optimizations" = x"yes"; then + case $host_cpu in + amd64 | x86_64) + dnl Support for AMD64 (also known as x86_64 on some platforms) processors + CPU_ARCH="x64" + AC_DEFINE([ARCH], [X64], [Architecture.]) + AC_DEFINE([WSIZE], [64], [Size of word in this architecture.]) + ;; + aarch*) + dnl Support for 64-bit ARM processors + dnl Relic does not support aarch64 yet, set CPU_ARCH to none and ARCH to RELIC_NONE. + CPU_ARCH="none" + AC_DEFINE([ARCH], [RELIC_NONE], [Architecture.]) + AC_DEFINE([WSIZE], [64], [Size of word in this architecture.]) + ;; + i?86) + dnl Support for Intel x86 processors + CPU_ARCH="x86" + AC_DEFINE([ARCH], [X86], [Architecture.]) + AC_DEFINE([WSIZE], [32], [Size of word in this architecture.]) + ;; + arm*) + dnl Support for 32-bit native ARM processors + CPU_ARCH="arm" + AC_DEFINE([ARCH], [ARM], [Architecture.]) + AC_DEFINE([WSIZE], [32], [Size of word in this architecture.]) + ;; + *64*) + dnl Support for an undefined 64-bit architecture + CPU_ARCH="none" + AC_DEFINE([ARCH], [RELIC_NONE], [Architecture.]) + AC_DEFINE([WSIZE], [64], [Size of word in this architecture.]) + ;; + *32*) + dnl Support for an undefined 32-bit architecture + CPU_ARCH="none" + AC_DEFINE([ARCH], [RELIC_NONE], [Architecture.]) + AC_DEFINE([WSIZE], [32], [Size of word in this architecture.]) + ;; + *) + AC_MSG_ERROR([Unable to determine host architecture, may not be supported!]) + ;; + esac else - AC_MSG_ERROR([Unable to determine host architecture, may not be supported!]) + case $host_cpu in + *64*) + dnl Support for an undefined 64-bit architecture + CPU_ARCH="none" + AC_DEFINE([ARCH], [RELIC_NONE], [Architecture.]) + AC_DEFINE([WSIZE], [64], [Size of word in this architecture.]) + ;; + *32* | arm* | i?86) + dnl Support for an undefined 32-bit architecture + CPU_ARCH="none" + AC_DEFINE([ARCH], [RELIC_NONE], [Architecture.]) + AC_DEFINE([WSIZE], [32], [Size of word in this architecture.]) + ;; + *) + AC_MSG_ERROR([Unable to determine host architecture, may not be supported!]) + ;; + esac fi case $host in *darwin*) AC_DEFINE([OPSYS], [MACOSX], [Detected operation system.]) + TARGET_OS=darwin + AC_PATH_PROG([BREW],brew,) if test x$BREW = x; then AC_PATH_PROG([PORT],port,) @@ -482,8 +516,10 @@ case $host in fi ;; *mingw*) - use_pkgconfig=no AC_DEFINE([OPSYS], [WINDOWS], [Detected operation system.]) + TARGET_OS=windows + + use_pkgconfig=no LIBTOOL_APP_LDFLAGS="$LIBTOOL_APP_LDFLAGS -all-static" dnl libtool insists upon adding -nostdlib and a list of objects/libs to link against. @@ -496,16 +532,21 @@ case $host in ;; *linux*) AC_DEFINE([OPSYS], [LINUX], [Detected operation system.]) + TARGET_OS=linux + RELIC_CPPFLAGS="-D_GNU_SOURCE" ;; *android*) AC_DEFINE([OPSYS], [DROID], [Detected operation system.]) + TARGET_OS=android ;; *freebsd*) AC_DEFINE([OPSYS], [FREEBSD], [Detected operation system.]) + TARGET_OS=freebsd ;; *netbsd*) AC_DEFINE([OPSYS], [NETBSD], [Detected operation system.]) + TARGET_OS=netbsd ;; *) AC_DEFINE([OPSYS], [RELIC_NONE], [Detected operation system.]) @@ -555,7 +596,7 @@ AC_COMPILE_IFELSE([AC_LANG_SOURCE([[char foo;]])], CFLAGS="$saved_CFLAGS" ]) -if [[[ "$CFLAGS_overridden" == "no" && "$CXXFLAGS_overridden" == "no" ]]]; then +if test x"$CFLAGS_overridden" = x"no" -a x"$CXXFLAGS_overridden" = x"no"; then dnl Enable warnings AX_CHECK_COMPILE_FLAG([-Wall],[WARN_FLAGS="$WARN_FLAGS -Wall"], [], [[$FLAG_WERROR]]) AX_CHECK_COMPILE_FLAG([-Wcast-align],[WARN_FLAGS="$WARN_FLAGS -Wcast-align"], [], [[$FLAG_WERROR]]) @@ -743,7 +784,6 @@ CORE_CPPFLAGS="$CORE_CPPFLAGS -DHAVE_BUILD_INFO" case $host in *mingw*) - TARGET_OS=windows AC_CHECK_LIB([user32], [main], [], [AC_MSG_ERROR([libuser32 missing])]) AC_CHECK_LIB([shell32], [SHGetSpecialFolderPathW], [], [AC_MSG_ERROR([libshell32 missing])]) AC_CHECK_LIB([advapi32], [CryptAcquireContextW], [], [AC_MSG_ERROR([libadvapi32 missing])]) @@ -771,15 +811,10 @@ case $host in AX_CHECK_LINK_FLAG([-Wl,--major-subsystem-version -Wl,6 -Wl,--minor-subsystem-version -Wl,1], [CORE_LDFLAGS="$CORE_LDFLAGS -Wl,--major-subsystem-version -Wl,6 -Wl,--minor-subsystem-version -Wl,1"], [], []) ;; *darwin*) - TARGET_OS=darwin - AX_CHECK_LINK_FLAG([-Wl,-headerpad_max_install_names], [CORE_LDFLAGS="$CORE_LDFLAGS -Wl,-headerpad_max_install_names"], [], []) CORE_CPPFLAGS="$CORE_CPPFLAGS -DMAC_OSX -DOBJC_OLD_DISPATCH_PROTOTYPES=0" OBJCXXFLAGS="$CXXFLAGS" ;; - *linux*) - TARGET_OS=linux - ;; esac dnl These flags are specific to ld64, and may cause issues with other linkers. @@ -799,19 +834,15 @@ AC_LANG_POP([C]) AC_MSG_CHECKING([whether to build runtest]) if test x$use_tests = xyes; then AC_MSG_RESULT([yes]) - BUILD_TEST="yes" else AC_MSG_RESULT([no]) - BUILD_TEST="" fi AC_MSG_CHECKING([whether to build runbench]) if test x$use_bench = xyes; then AC_MSG_RESULT([yes]) - BUILD_BENCH="yes" else AC_MSG_RESULT([no]) - BUILD_BENCH="" fi AM_CONDITIONAL([TARGET_DARWIN], [test "$TARGET_OS" = "darwin"]) @@ -836,8 +867,8 @@ AM_CONDITIONAL(WITH_MPC, test 1 -eq 1) AM_CONDITIONAL(WITH_DV, test 1 -eq 1) AM_CONDITIONAL(WITH_FBX, test 1 -eq 1) -AM_CONDITIONAL([USE_TESTS], [test x$BUILD_TEST = xyes]) -AM_CONDITIONAL([USE_BENCH], [test x$BUILD_BENCH = xyes]) +AM_CONDITIONAL([USE_TESTS], [test x"$use_tests" = x"yes"]) +AM_CONDITIONAL([USE_BENCH], [test x"$use_bench" = x"yes"]) AM_CONDITIONAL([HARDEN], [test "$use_hardening" = "yes"]) AM_CONDITIONAL([OPTIMIZE], [test "$use_optimizations" = "yes"]) @@ -874,8 +905,8 @@ echo echo "Options used to compile and link:" echo " target os = $TARGET_OS" echo " backend = $want_backend" -echo " build bench = $BUILD_BENCH" -echo " build test = $BUILD_TEST" +echo " build bench = $use_tests" +echo " build test = $use_bench" echo " use debug = $use_debug" echo " use hardening = $use_hardening" echo " use optimizations = $use_optimizations" diff --git a/src/evo/creditpool.h b/src/evo/creditpool.h index 4d4169974d..de6f13595b 100644 --- a/src/evo/creditpool.h +++ b/src/evo/creditpool.h @@ -141,4 +141,4 @@ std::optional GetCreditPoolDiffForBlock(CCreditPoolManager& cpo const CBlock& block, const CBlockIndex* pindexPrev, const Consensus::Params& consensusParams, const CAmount blockSubsidy, BlockValidationState& state); -#endif +#endif // BITCOIN_EVO_CREDITPOOL_H diff --git a/src/evo/dmnstate.h b/src/evo/dmnstate.h index acee5c2b49..a9eef0b555 100644 --- a/src/evo/dmnstate.h +++ b/src/evo/dmnstate.h @@ -377,4 +377,4 @@ class CDeterministicMNStateDiff }; -#endif //BITCOIN_EVO_DMNSTATE_H +#endif // BITCOIN_EVO_DMNSTATE_H diff --git a/src/governance/common.h b/src/governance/common.h index ae053d8504..48606f39b7 100644 --- a/src/governance/common.h +++ b/src/governance/common.h @@ -81,6 +81,6 @@ class Object } } }; - } // namespace Governance -#endif + +#endif // BITCOIN_GOVERNANCE_COMMON_H diff --git a/src/httprpc.h b/src/httprpc.h index 18f8551816..d85a46f025 100644 --- a/src/httprpc.h +++ b/src/httprpc.h @@ -31,4 +31,4 @@ void InterruptREST(); */ void StopREST(); -#endif +#endif // BITCOIN_HTTPRPC_H diff --git a/src/i2p.h b/src/i2p.h index 6183885435..f9619de9c5 100644 --- a/src/i2p.h +++ b/src/i2p.h @@ -286,4 +286,4 @@ class Session } // namespace sam } // namespace i2p -#endif /* BITCOIN_I2P_H */ +#endif // BITCOIN_I2P_H diff --git a/src/llmq/context.cpp b/src/llmq/context.cpp index 31857000d5..2d58262f80 100644 --- a/src/llmq/context.cpp +++ b/src/llmq/context.cpp @@ -46,9 +46,9 @@ LLMQContext::LLMQContext(ChainstateManager& chainman, CConnman& connman, CDeterm isman{[&]() -> llmq::CInstantSendManager* const { assert(llmq::quorumInstantSendManager == nullptr); llmq::quorumInstantSendManager = std::make_unique(*llmq::chainLocksHandler, - chainman.ActiveChainstate(), - connman, *qman, *sigman, *shareman, - sporkman, mempool, mn_sync, peerman, + chainman.ActiveChainstate(), *qman, + *sigman, *shareman, sporkman, + mempool, mn_sync, peerman, is_masternode, unit_tests, wipe); return llmq::quorumInstantSendManager.get(); }()}, diff --git a/src/llmq/instantsend.cpp b/src/llmq/instantsend.cpp index e175009bb8..598986b55b 100644 --- a/src/llmq/instantsend.cpp +++ b/src/llmq/instantsend.cpp @@ -1047,7 +1047,7 @@ void CInstantSendManager::ProcessInstantSendLock(NodeId from, const uint256& has // bump mempool counter to make sure newly locked txes are picked up by getblocktemplate mempool.AddTransactionsUpdated(1); } else { - AskNodesForLockedTx(islock->txid, connman, *m_peerman, m_is_masternode); + m_peerman->AskPeersForTransaction(islock->txid, m_is_masternode); } } @@ -1321,7 +1321,7 @@ void CInstantSendManager::RemoveMempoolConflictsForLock(const uint256& hash, con for (const auto& p : toDelete) { RemoveConflictedTx(*p.second); } - AskNodesForLockedTx(islock.txid, connman, *m_peerman, m_is_masternode); + m_peerman->AskPeersForTransaction(islock.txid, m_is_masternode); } } @@ -1426,46 +1426,6 @@ void CInstantSendManager::RemoveConflictingLock(const uint256& islockHash, const } } -void CInstantSendManager::AskNodesForLockedTx(const uint256& txid, const CConnman& connman, PeerManager& peerman, - bool is_masternode) -{ - std::vector nodesToAskFor; - nodesToAskFor.reserve(4); - - auto maybe_add_to_nodesToAskFor = [&peerman, &nodesToAskFor, &txid](CNode* pnode) { - if (nodesToAskFor.size() >= 4) { - return; - } - if (peerman.IsInvInFilter(pnode->GetId(), txid)) { - pnode->AddRef(); - nodesToAskFor.emplace_back(pnode); - } - }; - - connman.ForEachNode([&](CNode* pnode) { - // Check masternodes first - if (pnode->m_masternode_connection) maybe_add_to_nodesToAskFor(pnode); - }); - connman.ForEachNode([&](CNode* pnode) { - // Check non-masternodes next - if (!pnode->m_masternode_connection) maybe_add_to_nodesToAskFor(pnode); - }); - { - LOCK(cs_main); - for (const CNode* pnode : nodesToAskFor) { - LogPrintf("CInstantSendManager::%s -- txid=%s: asking other peer %d for correct TX\n", __func__, - txid.ToString(), pnode->GetId()); - - CInv inv(MSG_TX, txid); - peerman.RequestObject(pnode->GetId(), inv, GetTime(), is_masternode, - /* fForce = */ true); - } - } - for (CNode* pnode : nodesToAskFor) { - pnode->Release(); - } -} - void CInstantSendManager::ProcessPendingRetryLockTxs() { const auto retryTxs = WITH_LOCK(cs_pendingRetry, return pendingRetryTxs); diff --git a/src/llmq/instantsend.h b/src/llmq/instantsend.h index c0a27d7494..2b83878e9a 100644 --- a/src/llmq/instantsend.h +++ b/src/llmq/instantsend.h @@ -200,7 +200,6 @@ class CInstantSendManager : public CRecoveredSigsListener CChainLocksHandler& clhandler; CChainState& m_chainstate; - CConnman& connman; CQuorumManager& qman; CSigningManager& sigman; CSigSharesManager& shareman; @@ -255,13 +254,21 @@ class CInstantSendManager : public CRecoveredSigsListener std::unordered_set pendingRetryTxs GUARDED_BY(cs_pendingRetry); public: - explicit CInstantSendManager(CChainLocksHandler& _clhandler, CChainState& chainstate, CConnman& _connman, - CQuorumManager& _qman, CSigningManager& _sigman, CSigSharesManager& _shareman, - CSporkManager& sporkman, CTxMemPool& _mempool, const CMasternodeSync& mn_sync, - const std::unique_ptr& peerman, bool is_masternode, bool unitTests, bool fWipe) : + explicit CInstantSendManager(CChainLocksHandler& _clhandler, CChainState& chainstate, CQuorumManager& _qman, + CSigningManager& _sigman, CSigSharesManager& _shareman, CSporkManager& sporkman, + CTxMemPool& _mempool, const CMasternodeSync& mn_sync, + const std::unique_ptr& peerman, bool is_masternode, bool unitTests, + bool fWipe) : db(unitTests, fWipe), - clhandler(_clhandler), m_chainstate(chainstate), connman(_connman), qman(_qman), sigman(_sigman), - shareman(_shareman), spork_manager(sporkman), mempool(_mempool), m_mn_sync(mn_sync), m_peerman(peerman), + clhandler(_clhandler), + m_chainstate(chainstate), + qman(_qman), + sigman(_sigman), + shareman(_shareman), + spork_manager(sporkman), + mempool(_mempool), + m_mn_sync(mn_sync), + m_peerman(peerman), m_is_masternode{is_masternode} { workInterrupt.reset(); @@ -315,7 +322,6 @@ class CInstantSendManager : public CRecoveredSigsListener EXCLUSIVE_LOCKS_REQUIRED(!cs_inputReqests, !cs_nonLocked, !cs_pendingRetry); void ResolveBlockConflicts(const uint256& islockHash, const CInstantSendLock& islock) EXCLUSIVE_LOCKS_REQUIRED(!cs_inputReqests, !cs_nonLocked, !cs_pendingLocks, !cs_pendingRetry); - static void AskNodesForLockedTx(const uint256& txid, const CConnman& connman, PeerManager& peerman, bool is_masternode); void ProcessPendingRetryLockTxs() EXCLUSIVE_LOCKS_REQUIRED(!cs_creating, !cs_inputReqests, !cs_nonLocked, !cs_pendingRetry); diff --git a/src/llmq/snapshot.h b/src/llmq/snapshot.h index f02698704a..e1e32751c5 100644 --- a/src/llmq/snapshot.h +++ b/src/llmq/snapshot.h @@ -233,4 +233,4 @@ extern std::unique_ptr quorumSnapshotManager; } // namespace llmq -#endif //BITCOIN_LLMQ_SNAPSHOT_H +#endif // BITCOIN_LLMQ_SNAPSHOT_H diff --git a/src/net_processing.cpp b/src/net_processing.cpp index d8e44c7b91..c20c508be8 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -329,6 +329,11 @@ struct Peer { LOCK(m_tx_relay_mutex); return m_can_tx_relay ? m_tx_relay.get() : nullptr; }; + const TxRelay* GetTxRelay() const LOCKS_EXCLUDED(m_tx_relay_mutex) + { + LOCK(m_tx_relay_mutex); + return m_can_tx_relay ? m_tx_relay.get() : nullptr; + }; /** A vector of addresses to send to the peer, limited to MAX_ADDR_TO_SEND. */ std::vector m_addrs_to_send GUARDED_BY(NetEventsInterface::g_msgproc_mutex); @@ -409,7 +414,7 @@ struct Peer { {} private: - Mutex m_tx_relay_mutex; + mutable Mutex m_tx_relay_mutex; /** Transaction relay data. * (Bitcoin) Transaction relay data. May be a nullptr. @@ -633,7 +638,7 @@ class PeerManagerImpl final : public PeerManager bool is_masternode, bool fForce = false) override EXCLUSIVE_LOCKS_REQUIRED(::cs_main); size_t GetRequestedObjectCount(NodeId nodeid) const override EXCLUSIVE_LOCKS_REQUIRED(::cs_main); bool IsInvInFilter(NodeId nodeid, const uint256& hash) const override EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex); - + void AskPeersForTransaction(const uint256& txid, bool is_masternode) override EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex); private: /** Helpers to process result of external handlers of message */ void ProcessPeerMsgRet(const PeerMsgRet& ret, CNode& pfrom) EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex); @@ -648,6 +653,11 @@ class PeerManagerImpl final : public PeerManager /** Retrieve unbroadcast transactions from the mempool and reattempt sending to peers */ void ReattemptInitialBroadcast(CScheduler& scheduler) EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex); + /** + * Private implementation of IsInvInFilter which does not call GetPeerRef; to be prefered when the PeerRef is available. + */ + bool IsInvInFilter(const Peer& peer, const uint256& hash) const; + /** Get a shared pointer to the Peer object. * May return an empty shared_ptr if the Peer object can't be found. */ PeerRef GetPeerRef(NodeId id) const EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex); @@ -2256,12 +2266,47 @@ void PeerManagerImpl::SendPings() for(auto& it : m_peer_map) it.second->m_ping_queued = true; } +void PeerManagerImpl::AskPeersForTransaction(const uint256& txid, bool is_masternode) +{ + std::vector peersToAsk; + peersToAsk.reserve(4); + + { + LOCK(m_peer_mutex); + // TODO consider prioritizing MNs again, once that flag is moved into Peer + for (const auto& [_, peer] : m_peer_map) { + if (peersToAsk.size() >= 4) { + break; + } + if (IsInvInFilter(*peer, txid)) { + peersToAsk.emplace_back(peer); + } + } + } + { + CInv inv(MSG_TX, txid); + LOCK(cs_main); + for (PeerRef& peer : peersToAsk) { + LogPrintf("PeerManagerImpl::%s -- txid=%s: asking other peer %d for correct TX\n", __func__, + txid.ToString(), peer->m_id); + + RequestObject(peer->m_id, inv, GetTime(), is_masternode, + /*fForce=*/true); + } + } +} + bool PeerManagerImpl::IsInvInFilter(NodeId nodeid, const uint256& hash) const { PeerRef peer = GetPeerRef(nodeid); if (peer == nullptr) return false; - if (auto tx_relay = peer->GetTxRelay(); tx_relay != nullptr) { + return IsInvInFilter(*peer, hash); +} + +bool PeerManagerImpl::IsInvInFilter(const Peer& peer, const uint256& hash) const +{ + if (auto tx_relay = peer.GetTxRelay(); tx_relay != nullptr) { LOCK(tx_relay->m_tx_inventory_mutex); return tx_relay->m_tx_inventory_known_filter.contains(hash); } diff --git a/src/net_processing.h b/src/net_processing.h index 2a302ee9dc..0bd94f7729 100644 --- a/src/net_processing.h +++ b/src/net_processing.h @@ -91,6 +91,9 @@ class PeerManager : public CValidationInterface, public NetEventsInterface /** Is an inventory in the known inventory filter. Used by InstantSend. */ virtual bool IsInvInFilter(NodeId nodeid, const uint256& hash) const = 0; + /** Ask a number of our peers, which have a transaction in their inventory, for the transaction. */ + virtual void AskPeersForTransaction(const uint256& txid, bool is_masternode) = 0; + /** Broadcast inventory message to a specific peer. */ virtual void PushInventory(NodeId nodeid, const CInv& inv) = 0; diff --git a/src/node/psbt.h b/src/node/psbt.h index fd2f063351..8f41ec2291 100644 --- a/src/node/psbt.h +++ b/src/node/psbt.h @@ -50,4 +50,4 @@ struct PSBTAnalysis { */ PSBTAnalysis AnalyzePSBT(PartiallySignedTransaction psbtx); -#endif // BITCOIN_PSBT_H +#endif // BITCOIN_NODE_PSBT_H diff --git a/src/policy/feerate.h b/src/policy/feerate.h index df700dfed5..f0d82a7402 100644 --- a/src/policy/feerate.h +++ b/src/policy/feerate.h @@ -66,4 +66,4 @@ class CFeeRate SERIALIZE_METHODS(CFeeRate, obj) { READWRITE(obj.nSatoshisPerK); } }; -#endif // BITCOIN_POLICY_FEERATE_H +#endif // BITCOIN_POLICY_FEERATE_H diff --git a/src/randomenv.h b/src/randomenv.h index 46cea6f6f2..746516b79b 100644 --- a/src/randomenv.h +++ b/src/randomenv.h @@ -14,4 +14,4 @@ void RandAddDynamicEnv(CSHA512& hasher); /** Gather non-cryptographic environment data that does not change over time. */ void RandAddStaticEnv(CSHA512& hasher); -#endif +#endif // BITCOIN_RANDOMENV_H diff --git a/src/rpc/blockchain.h b/src/rpc/blockchain.h index e383aba436..15a1d0e723 100644 --- a/src/rpc/blockchain.h +++ b/src/rpc/blockchain.h @@ -64,4 +64,4 @@ void TxToUniv(const CTransaction& tx, const uint256& hashBlock, UniValue& entry, */ UniValue CreateUTXOSnapshot(NodeContext& node, CChainState& chainstate, CAutoFile& afile); -#endif +#endif // BITCOIN_RPC_BLOCKCHAIN_H diff --git a/src/rpc/index_util.h b/src/rpc/index_util.h index 6bd3b73124..a2a632c8d1 100644 --- a/src/rpc/index_util.h +++ b/src/rpc/index_util.h @@ -42,4 +42,4 @@ bool GetTimestampIndex(CBlockTreeDB& block_tree_db, const uint32_t high, const u std::vector& hashes) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); -#endif // BITCOIN_RPC_CLIENT_H +#endif // BITCOIN_RPC_INDEX_UTIL_H diff --git a/src/scheduler.h b/src/scheduler.h index 438fb8c126..727270e195 100644 --- a/src/scheduler.h +++ b/src/scheduler.h @@ -152,4 +152,4 @@ class SingleThreadedSchedulerClient size_t CallbacksPending() EXCLUSIVE_LOCKS_REQUIRED(!m_callbacks_mutex); }; -#endif +#endif // BITCOIN_SCHEDULER_H diff --git a/src/shutdown.h b/src/shutdown.h index 31a0b441a4..3e22bfd0f0 100644 --- a/src/shutdown.h +++ b/src/shutdown.h @@ -36,4 +36,4 @@ bool RestartRequested(); */ void WaitForShutdown(); -#endif +#endif // BITCOIN_SHUTDOWN_H diff --git a/src/span.h b/src/span.h index 84a3f4b140..a7ebd48cc7 100644 --- a/src/span.h +++ b/src/span.h @@ -295,4 +295,4 @@ template return span.end(); } -#endif +#endif // BITCOIN_SPAN_H diff --git a/src/stacktraces.h b/src/stacktraces.h index 01ae135ee8..e6f577a818 100644 --- a/src/stacktraces.h +++ b/src/stacktraces.h @@ -37,4 +37,4 @@ inline std::string GetExceptionWhat(const T& e) void RegisterPrettyTerminateHander(); void RegisterPrettySignalHandlers(); -#endif//BITCOIN_STACKTRACES_H +#endif // BITCOIN_STACKTRACES_H diff --git a/src/test/scriptnum10.h b/src/test/scriptnum10.h index 352797f18d..3937366f01 100644 --- a/src/test/scriptnum10.h +++ b/src/test/scriptnum10.h @@ -179,4 +179,4 @@ class CScriptNum10 }; -#endif // BITCOIN_TEST_BIGNUM_H +#endif // BITCOIN_TEST_SCRIPTNUM10_H diff --git a/src/test/util/setup_common.h b/src/test/util/setup_common.h index f21205cff2..639c96ffe7 100644 --- a/src/test/util/setup_common.h +++ b/src/test/util/setup_common.h @@ -250,4 +250,4 @@ class HasReason // define an implicit conversion here so that uint256 may be used directly in BOOST_CHECK_* std::ostream& operator<<(std::ostream& os, const uint256& num); -#endif +#endif // BITCOIN_TEST_UTIL_SETUP_COMMON_H diff --git a/src/threadinterrupt.h b/src/threadinterrupt.h index c053c01499..bf7945a0a1 100644 --- a/src/threadinterrupt.h +++ b/src/threadinterrupt.h @@ -32,4 +32,4 @@ class CThreadInterrupt std::atomic flag; }; -#endif //BITCOIN_THREADINTERRUPT_H +#endif // BITCOIN_THREADINTERRUPT_H diff --git a/src/torcontrol.h b/src/torcontrol.h index 75464b148e..97dadd88fd 100644 --- a/src/torcontrol.h +++ b/src/torcontrol.h @@ -159,4 +159,4 @@ class TorController static void reconnect_cb(evutil_socket_t fd, short what, void *arg); }; -#endif /* BITCOIN_TORCONTROL_H */ +#endif // BITCOIN_TORCONTROL_H diff --git a/src/util/edge.h b/src/util/edge.h index 1137ce624a..8e3ec357a1 100644 --- a/src/util/edge.h +++ b/src/util/edge.h @@ -59,4 +59,4 @@ class EdgeTriggeredEvents int m_fd{-1}; }; -#endif /* BITCOIN_UTIL_EDGE_H */ +#endif // BITCOIN_UTIL_EDGE_H diff --git a/src/util/readwritefile.h b/src/util/readwritefile.h index 1dab874b38..a59d0be131 100644 --- a/src/util/readwritefile.h +++ b/src/util/readwritefile.h @@ -25,4 +25,4 @@ std::pair ReadBinaryFile(const fs::path &filename, size_t maxs */ bool WriteBinaryFile(const fs::path &filename, const std::string &data); -#endif /* BITCOIN_UTIL_READWRITEFILE_H */ +#endif // BITCOIN_UTIL_READWRITEFILE_H diff --git a/src/util/trace.h b/src/util/trace.h index ec468330d7..d0fd841edf 100644 --- a/src/util/trace.h +++ b/src/util/trace.h @@ -55,4 +55,4 @@ #endif -#endif /* BITCOIN_UTIL_TRACE_H */ +#endif // BITCOIN_UTIL_TRACE_H diff --git a/src/util/underlying.h b/src/util/underlying.h index aa036b18fb..55821bb742 100644 --- a/src/util/underlying.h +++ b/src/util/underlying.h @@ -11,4 +11,4 @@ template return static_cast::type>(e); } -#endif //BITCOIN_UTIL_UNDERLYING_H +#endif // BITCOIN_UTIL_UNDERLYING_H diff --git a/src/util/wpipe.h b/src/util/wpipe.h index c4fff558ec..30912149b4 100644 --- a/src/util/wpipe.h +++ b/src/util/wpipe.h @@ -56,4 +56,4 @@ class WakeupPipe EdgeTriggeredEvents* m_edge_trig_events{nullptr}; }; -#endif /* BITCOIN_UTIL_WPIPE_H */ +#endif // BITCOIN_UTIL_WPIPE_H diff --git a/src/wallet/rpcwallet.h b/src/wallet/rpcwallet.h index 5c8c643a43..ce3d5a659f 100644 --- a/src/wallet/rpcwallet.h +++ b/src/wallet/rpcwallet.h @@ -41,4 +41,4 @@ RPCHelpMan getaddressinfo(); RPCHelpMan getrawchangeaddress(); RPCHelpMan addmultisigaddress(); RPCHelpMan signrawtransactionwithwallet(); -#endif //BITCOIN_WALLET_RPCWALLET_H +#endif // BITCOIN_WALLET_RPCWALLET_H diff --git a/src/warnings.h b/src/warnings.h index c38edb4570..7ab0a93e3f 100644 --- a/src/warnings.h +++ b/src/warnings.h @@ -20,4 +20,4 @@ void SetfLargeWorkInvalidChainFound(bool flag); */ bilingual_str GetWarnings(bool verbose); -#endif // BITCOIN_WARNINGS_H +#endif // BITCOIN_WARNINGS_H diff --git a/src/zmq/zmqrpc.h b/src/zmq/zmqrpc.h index 5a810a16fb..8538adf9d3 100644 --- a/src/zmq/zmqrpc.h +++ b/src/zmq/zmqrpc.h @@ -9,4 +9,4 @@ class CRPCTable; void RegisterZMQRPCCommands(CRPCTable& t); -#endif // BITCOIN_ZMQ_ZMRRPC_H +#endif // BITCOIN_ZMQ_ZMQRPC_H diff --git a/test/README.md b/test/README.md index 3e69765074..4802ed7007 100644 --- a/test/README.md +++ b/test/README.md @@ -310,11 +310,11 @@ Use the `-v` option for verbose output. | Lint test | Dependency | |-----------|:----------:| -| [`lint-python.sh`](lint/lint-python.sh) | [flake8](https://gitlab.com/pycqa/flake8) -| [`lint-python.sh`](lint/lint-python.sh) | [mypy](https://github.com/python/mypy) -| [`lint-python.sh`](lint/lint-python.sh) | [pyzmq](https://github.com/zeromq/pyzmq) +| [`lint-python.py`](lint/lint-python.py) | [flake8](https://gitlab.com/pycqa/flake8) +| [`lint-python.py`](lint/lint-python.py) | [mypy](https://github.com/python/mypy) +| [`lint-python.py`](lint/lint-python.py) | [pyzmq](https://github.com/zeromq/pyzmq) | [`lint-python-dead-code.py`](lint/lint-python-dead-code.py) | [vulture](https://github.com/jendrikseipp/vulture) -| [`lint-shell.sh`](lint/lint-shell.sh) | [ShellCheck](https://github.com/koalaman/shellcheck) +| [`lint-shell.py`](lint/lint-shell.py) | [ShellCheck](https://github.com/koalaman/shellcheck) | [`lint-spelling.py`](lint/lint-spelling.py) | [codespell](https://github.com/codespell-project/codespell) In use versions and install instructions are available in the [CI setup](../ci/lint/04_install.sh). @@ -332,7 +332,7 @@ test/lint/lint-files.py You can run all the shell-based lint tests by running: ``` -test/lint/lint-all.sh +test/lint/all-lint.py ``` # Writing functional tests diff --git a/test/lint/README.md b/test/lint/README.md index 177356abc4..748a845e59 100644 --- a/test/lint/README.md +++ b/test/lint/README.md @@ -39,6 +39,6 @@ To do so, add the upstream repository as remote: git remote add --fetch secp256k1 https://github.com/bitcoin-core/secp256k1.git ``` -lint-all.sh +all-lint.py =========== Calls other scripts with the `lint-` prefix. diff --git a/test/lint/all-lint.py b/test/lint/all-lint.py new file mode 100755 index 0000000000..40274fcc41 --- /dev/null +++ b/test/lint/all-lint.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2017-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# This script runs all test/lint/lint-* files, and fails if any exit +# with a non-zero status code. + +from glob import glob +from os import path as os_path, remove +from pathlib import Path +from shutil import which +from subprocess import run + +exit_code = 0 +mod_path = Path(__file__).parent +lints = glob(f"{mod_path}/lint-*") +if which("parallel") and which("column"): + logfile = "parallel_out.log" + command = ["parallel", "--jobs", "100%", "--will-cite", "--joblog", logfile, ":::"] + lints + result = run(command) + if result.returncode != 0: + print(f"^---- failure generated") + exit_code = result.returncode + result = run(["column", "-t", logfile]) + if os_path.isfile(logfile): + remove(logfile) +else: + for lint in lints: + result = run([lint]) + if result.returncode != 0: + print(f"^---- failure generated from {lint.split('/')[-1]}") + exit_code |= result.returncode + +exit(exit_code) diff --git a/test/lint/lint-all.sh b/test/lint/lint-all.sh deleted file mode 100755 index 3472a09908..0000000000 --- a/test/lint/lint-all.sh +++ /dev/null @@ -1,46 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2017-2019 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# This script runs all contrib/devtools/lint-* files, and fails if any exit -# with a non-zero status code. - -# This script is intentionally locale dependent by not setting "export LC_ALL=C" -# in order to allow for the executed lint scripts to opt in or opt out of locale -# dependence themselves. - -set -u - -SCRIPTDIR=$(dirname "${BASH_SOURCE[0]}") -LINTALL=$(basename "${BASH_SOURCE[0]}") - -EXIT_CODE=0 - -if ! command -v parallel > /dev/null; then - for f in "${SCRIPTDIR}"/lint-*; do - if [ "$(basename "$f")" != "$LINTALL" ]; then - if ! "$f"; then - echo "^---- failure generated from $f" - EXIT_CODE=1 - fi - fi - done -else - SCRIPTS=() - - for f in "${SCRIPTDIR}"/lint-*; do - if [ "$(basename "$f")" != "$LINTALL" ]; then - SCRIPTS+=("$f") - fi - done - - if ! parallel --jobs 100% --will-cite --joblog parallel_out.log ::: "${SCRIPTS[@]}"; then - echo "^---- failure generated" - EXIT_CODE=1 - fi - column -t parallel_out.log && rm parallel_out.log -fi - -exit ${EXIT_CODE} diff --git a/test/lint/lint-circular-dependencies.py b/test/lint/lint-circular-dependencies.py new file mode 100755 index 0000000000..74782e401e --- /dev/null +++ b/test/lint/lint-circular-dependencies.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2020-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Check for circular dependencies + +import glob +import os +import re +import subprocess +import sys + +EXPECTED_CIRCULAR_DEPENDENCIES = ( + "chainparamsbase -> util/system -> chainparamsbase", + "node/blockstorage -> validation -> node/blockstorage", + "index/coinstatsindex -> node/coinstats -> index/coinstatsindex", + "policy/fees -> txmempool -> policy/fees", + "qt/addresstablemodel -> qt/walletmodel -> qt/addresstablemodel", + "qt/recentrequeststablemodel -> qt/walletmodel -> qt/recentrequeststablemodel", + "qt/transactiontablemodel -> qt/walletmodel -> qt/transactiontablemodel", + "wallet/fees -> wallet/wallet -> wallet/fees", + "wallet/wallet -> wallet/walletdb -> wallet/wallet", + "node/coinstats -> validation -> node/coinstats", + # Dash + "banman -> common/bloom -> evo/assetlocktx -> llmq/quorums -> net -> banman", + "banman -> common/bloom -> evo/assetlocktx -> llmq/signing -> net_processing -> banman", + "coinjoin/client -> net_processing -> coinjoin/client", + "coinjoin/client -> net_processing -> coinjoin/context -> coinjoin/client", + "coinjoin/coinjoin -> llmq/chainlocks -> net -> coinjoin/coinjoin", + "coinjoin/context -> coinjoin/server -> net_processing -> coinjoin/context", + "coinjoin/server -> net_processing -> coinjoin/server", + "common/bloom -> evo/assetlocktx -> llmq/quorums -> net -> common/bloom", + "common/bloom -> evo/assetlocktx -> llmq/signing -> net_processing -> merkleblock -> common/bloom", + "consensus/tx_verify -> evo/assetlocktx -> validation -> consensus/tx_verify", + "consensus/tx_verify -> evo/assetlocktx -> validation -> txmempool -> consensus/tx_verify", + "core_io -> evo/cbtx -> evo/simplifiedmns -> core_io", + "dsnotificationinterface -> llmq/chainlocks -> node/blockstorage -> dsnotificationinterface", + "evo/assetlocktx -> validation -> txmempool -> evo/assetlocktx", + "evo/cbtx -> evo/simplifiedmns -> evo/cbtx", + "evo/chainhelper -> evo/specialtxman -> validation -> evo/chainhelper", + "evo/deterministicmns -> llmq/commitment -> evo/deterministicmns", + "evo/deterministicmns -> llmq/utils -> evo/deterministicmns", + "evo/deterministicmns -> llmq/utils -> llmq/snapshot -> evo/simplifiedmns -> evo/deterministicmns", + "evo/deterministicmns -> llmq/utils -> net -> evo/deterministicmns", + "evo/deterministicmns -> validation -> evo/deterministicmns", + "evo/deterministicmns -> validation -> txmempool -> evo/deterministicmns", + "evo/deterministicmns -> validationinterface -> evo/deterministicmns", + "evo/deterministicmns -> validationinterface -> governance/vote -> evo/deterministicmns", + "evo/mnhftx -> validation -> evo/mnhftx", + "evo/simplifiedmns -> llmq/blockprocessor -> llmq/utils -> llmq/snapshot -> evo/simplifiedmns", + "evo/specialtxman -> validation -> evo/specialtxman", + "governance/governance -> governance/object -> governance/governance", + "governance/governance -> masternode/sync -> governance/governance", + "governance/governance -> net_processing -> governance/governance", + "governance/governance -> validation -> governance/governance", + "governance/vote -> masternode/node -> validationinterface -> governance/vote", + "llmq/blockprocessor -> llmq/utils -> llmq/snapshot -> llmq/blockprocessor", + "llmq/chainlocks -> llmq/instantsend -> llmq/chainlocks", + "llmq/chainlocks -> llmq/instantsend -> net_processing -> llmq/chainlocks", + "llmq/chainlocks -> validation -> llmq/chainlocks", + "llmq/commitment -> llmq/utils -> llmq/snapshot -> llmq/commitment", + "llmq/context -> llmq/instantsend -> net_processing -> llmq/context", + "llmq/dkgsession -> llmq/dkgsessionmgr -> llmq/dkgsessionhandler -> llmq/dkgsession", + "llmq/dkgsessionhandler -> net_processing -> llmq/dkgsessionmgr -> llmq/dkgsessionhandler", + "llmq/instantsend -> net_processing -> llmq/instantsend", + "llmq/instantsend -> txmempool -> llmq/instantsend", + "llmq/instantsend -> validation -> llmq/instantsend", + "llmq/signing -> llmq/signing_shares -> llmq/signing", + "llmq/signing -> masternode/node -> validationinterface -> llmq/signing", + "llmq/signing -> net_processing -> llmq/signing", + "llmq/signing_shares -> net_processing -> llmq/signing_shares", + "logging -> util/system -> logging", + "logging -> util/system -> stacktraces -> logging", + "logging -> util/system -> sync -> logging", + "logging -> util/system -> sync -> logging/timer -> logging", + "logging -> util/system -> util/getuniquepath -> random -> logging", + "masternode/payments -> validation -> masternode/payments", + "masternode/sync -> validation -> masternode/sync", + "net -> netmessagemaker -> net", + "net_processing -> spork -> net_processing", + "netaddress -> netbase -> netaddress", + "policy/policy -> policy/settings -> policy/policy", + "qt/appearancewidget -> qt/guiutil -> qt/appearancewidget", + "qt/appearancewidget -> qt/guiutil -> qt/optionsdialog -> qt/appearancewidget", + "qt/bitcoinaddressvalidator -> qt/guiutil -> qt/bitcoinaddressvalidator", + "qt/bitcoingui -> qt/guiutil -> qt/bitcoingui", + "qt/guiutil -> qt/optionsdialog -> qt/guiutil", + "qt/guiutil -> qt/optionsdialog -> qt/optionsmodel -> qt/guiutil", + "qt/guiutil -> qt/qvalidatedlineedit -> qt/guiutil", + "rpc/blockchain -> rpc/server -> rpc/blockchain" +) + +CODE_DIR = "src" + + +def main(): + circular_dependencies = [] + exit_code = 0 + os.chdir( + CODE_DIR + ) # We change dir before globbing since glob.glob's root_dir option is only available in Python 3.10 + + # Using glob.glob since subprocess.run's globbing won't work without shell=True + files = [] + for path in ["*", "*/*", "*/*/*"]: + for extension in ["h", "cpp"]: + files.extend(glob.glob(f"{path}.{extension}")) + + command = ["python3", "../contrib/devtools/circular-dependencies.py", *files] + dependencies_output = subprocess.run( + command, + stdout=subprocess.PIPE, + universal_newlines=True, + ) + + for dependency_str in dependencies_output.stdout.rstrip().split("\n"): + circular_dependencies.append( + re.sub("^Circular dependency: ", "", dependency_str) + ) + + # Check for an unexpected dependencies + for dependency in circular_dependencies: + if dependency not in EXPECTED_CIRCULAR_DEPENDENCIES: + exit_code = 1 + print( + f'A new circular dependency in the form of "{dependency}" appears to have been introduced.\n', + file=sys.stderr, + ) + + # Check for missing expected dependencies + for expected_dependency in EXPECTED_CIRCULAR_DEPENDENCIES: + if expected_dependency not in circular_dependencies: + exit_code = 1 + print( + f'Good job! The circular dependency "{expected_dependency}" is no longer present.', + ) + print( + f"Please remove it from EXPECTED_CIRCULAR_DEPENDENCIES in {__file__}", + ) + print( + "to make sure this circular dependency is not accidentally reintroduced.\n", + ) + + sys.exit(exit_code) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-circular-dependencies.sh b/test/lint/lint-circular-dependencies.sh deleted file mode 100755 index 3516033753..0000000000 --- a/test/lint/lint-circular-dependencies.sh +++ /dev/null @@ -1,133 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Check for circular dependencies - -export LC_ALL=C - -EXPECTED_CIRCULAR_DEPENDENCIES=( - "chainparamsbase -> util/system -> chainparamsbase" - "node/blockstorage -> validation -> node/blockstorage" - "index/coinstatsindex -> node/coinstats -> index/coinstatsindex" - "policy/fees -> txmempool -> policy/fees" - "qt/addresstablemodel -> qt/walletmodel -> qt/addresstablemodel" - "qt/recentrequeststablemodel -> qt/walletmodel -> qt/recentrequeststablemodel" - "qt/transactiontablemodel -> qt/walletmodel -> qt/transactiontablemodel" - "wallet/fees -> wallet/wallet -> wallet/fees" - "wallet/wallet -> wallet/walletdb -> wallet/wallet" - "node/coinstats -> validation -> node/coinstats" - # Dash - "dsnotificationinterface -> llmq/chainlocks -> node/blockstorage -> dsnotificationinterface" - "evo/cbtx -> evo/simplifiedmns -> evo/cbtx" - "evo/deterministicmns -> llmq/commitment -> evo/deterministicmns" - "evo/deterministicmns -> llmq/utils -> evo/deterministicmns" - "governance/governance -> governance/object -> governance/governance" - "governance/governance -> masternode/sync -> governance/governance" - "llmq/chainlocks -> llmq/instantsend -> llmq/chainlocks" - "llmq/dkgsessionhandler -> net_processing -> llmq/dkgsessionmgr -> llmq/dkgsessionhandler" - "llmq/instantsend -> net_processing -> llmq/instantsend" - "llmq/instantsend -> txmempool -> llmq/instantsend" - "llmq/instantsend -> validation -> llmq/instantsend" - "llmq/signing -> llmq/signing_shares -> llmq/signing" - "llmq/signing -> net_processing -> llmq/signing" - "llmq/signing_shares -> net_processing -> llmq/signing_shares" - "logging -> util/system -> logging" - "masternode/payments -> validation -> masternode/payments" - "masternode/sync -> validation -> masternode/sync" - "net -> netmessagemaker -> net" - "netaddress -> netbase -> netaddress" - "qt/appearancewidget -> qt/guiutil -> qt/appearancewidget" - "qt/bitcoinaddressvalidator -> qt/guiutil -> qt/bitcoinaddressvalidator" - "qt/bitcoingui -> qt/guiutil -> qt/bitcoingui" - "qt/guiutil -> qt/optionsdialog -> qt/guiutil" - "qt/guiutil -> qt/qvalidatedlineedit -> qt/guiutil" - "core_io -> evo/cbtx -> evo/simplifiedmns -> core_io" - "llmq/dkgsession -> llmq/dkgsessionmgr -> llmq/dkgsessionhandler -> llmq/dkgsession" - "logging -> util/system -> sync -> logging" - "logging -> util/system -> stacktraces -> logging" - "logging -> util/system -> util/getuniquepath -> random -> logging" - "qt/appearancewidget -> qt/guiutil -> qt/optionsdialog -> qt/appearancewidget" - "qt/guiutil -> qt/optionsdialog -> qt/optionsmodel -> qt/guiutil" - - "common/bloom -> evo/assetlocktx -> llmq/quorums -> net -> common/bloom" - "common/bloom -> evo/assetlocktx -> llmq/signing -> net_processing -> merkleblock -> common/bloom" - "banman -> common/bloom -> evo/assetlocktx -> llmq/quorums -> net -> banman" - "banman -> common/bloom -> evo/assetlocktx -> llmq/signing -> net_processing -> banman" - - "llmq/chainlocks -> validation -> llmq/chainlocks" - "coinjoin/coinjoin -> llmq/chainlocks -> net -> coinjoin/coinjoin" - "evo/assetlocktx -> validation -> txmempool -> evo/assetlocktx" - "evo/deterministicmns -> llmq/utils -> llmq/snapshot -> evo/simplifiedmns -> evo/deterministicmns" - "evo/deterministicmns -> llmq/utils -> net -> evo/deterministicmns" - "evo/deterministicmns -> validation -> txmempool -> evo/deterministicmns" - "policy/policy -> policy/settings -> policy/policy" - "consensus/tx_verify -> evo/assetlocktx -> validation -> consensus/tx_verify" - "consensus/tx_verify -> evo/assetlocktx -> validation -> txmempool -> consensus/tx_verify" - - "evo/simplifiedmns -> llmq/blockprocessor -> llmq/utils -> llmq/snapshot -> evo/simplifiedmns" - "llmq/blockprocessor -> llmq/utils -> llmq/snapshot -> llmq/blockprocessor" - "llmq/commitment -> llmq/utils -> llmq/snapshot -> llmq/commitment" - "governance/governance -> validation -> governance/governance" - "evo/deterministicmns -> validationinterface -> governance/vote -> evo/deterministicmns" - "governance/vote -> masternode/node -> validationinterface -> governance/vote" - "llmq/signing -> masternode/node -> validationinterface -> llmq/signing" - "evo/mnhftx -> validation -> evo/mnhftx" - "evo/deterministicmns -> validation -> evo/deterministicmns" - "evo/specialtxman -> validation -> evo/specialtxman" - "evo/chainhelper -> evo/specialtxman -> validation -> evo/chainhelper" - "evo/deterministicmns -> validationinterface -> evo/deterministicmns" - "logging -> util/system -> sync -> logging/timer -> logging" - - "coinjoin/client -> net_processing -> coinjoin/client" - "coinjoin/client -> net_processing -> coinjoin/context -> coinjoin/client" - "coinjoin/context -> coinjoin/server -> net_processing -> coinjoin/context" - "coinjoin/server -> net_processing -> coinjoin/server" - "llmq/context -> llmq/instantsend -> net_processing -> llmq/context" - "llmq/chainlocks -> llmq/instantsend -> net_processing -> llmq/chainlocks" - "net_processing -> spork -> net_processing" - "governance/governance -> net_processing -> governance/governance" - "rpc/blockchain -> rpc/server -> rpc/blockchain" -) - -EXIT_CODE=0 - -CIRCULAR_DEPENDENCIES=() - -IFS=$'\n' -for CIRC in $(cd src && ../contrib/devtools/circular-dependencies.py {*,*/*,*/*/*}.{h,cpp} | sed -e 's/^Circular dependency: //'); do - CIRCULAR_DEPENDENCIES+=( "$CIRC" ) - IS_EXPECTED_CIRC=0 - for EXPECTED_CIRC in "${EXPECTED_CIRCULAR_DEPENDENCIES[@]}"; do - if [[ "${CIRC}" == "${EXPECTED_CIRC}" ]]; then - IS_EXPECTED_CIRC=1 - break - fi - done - if [[ ${IS_EXPECTED_CIRC} == 0 ]]; then - echo "A new circular dependency in the form of \"${CIRC}\" appears to have been introduced." - echo - EXIT_CODE=1 - fi -done - -for EXPECTED_CIRC in "${EXPECTED_CIRCULAR_DEPENDENCIES[@]}"; do - IS_PRESENT_EXPECTED_CIRC=0 - for CIRC in "${CIRCULAR_DEPENDENCIES[@]}"; do - if [[ "${CIRC}" == "${EXPECTED_CIRC}" ]]; then - IS_PRESENT_EXPECTED_CIRC=1 - break - fi - done - if [[ ${IS_PRESENT_EXPECTED_CIRC} == 0 ]]; then - echo "Good job! The circular dependency \"${EXPECTED_CIRC}\" is no longer present." - echo "Please remove it from EXPECTED_CIRCULAR_DEPENDENCIES in $0" - echo "to make sure this circular dependency is not accidentally reintroduced." - echo - EXIT_CODE=1 - fi -done - -exit ${EXIT_CODE} diff --git a/test/lint/lint-include-guards.py b/test/lint/lint-include-guards.py new file mode 100755 index 0000000000..8fd4c7d636 --- /dev/null +++ b/test/lint/lint-include-guards.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" +Check include guards. +""" + +import re +import sys +from subprocess import check_output +from typing import List + + +HEADER_ID_PREFIX = 'BITCOIN_' +HEADER_ID_SUFFIX = '_H' + +EXCLUDE_FILES_WITH_PREFIX = ['src/crypto/ctaes', + 'src/leveldb', + 'src/crc32c', + 'src/secp256k1', + 'src/minisketch', + 'src/univalue', + 'src/tinyformat.h', + 'src/bench/nanobench.h', + 'src/test/fuzz/FuzzedDataProvider.h', + 'src/bls', + 'src/crypto/x11/sph', + 'src/ctpl_stl.h', + 'src/dashbls', + 'src/gsl', + 'src/immer', + 'src/util/expected.h'] + + +def _get_header_file_lst() -> List[str]: + """ Helper function to get a list of header filepaths to be + checked for include guards. + """ + git_cmd_lst = ['git', 'ls-files', '--', '*.h'] + header_file_lst = check_output( + git_cmd_lst).decode('utf-8').splitlines() + + header_file_lst = [hf for hf in header_file_lst + if not any(ef in hf for ef + in EXCLUDE_FILES_WITH_PREFIX)] + + return header_file_lst + + +def _get_header_id(header_file: str) -> str: + """ Helper function to get the header id from a header file + string. + + eg: 'src/wallet/walletdb.h' -> 'BITCOIN_WALLET_WALLETDB_H' + + Args: + header_file: Filepath to header file. + + Returns: + The header id. + """ + header_id_base = header_file.split('/')[1:] + header_id_base = '_'.join(header_id_base) + header_id_base = header_id_base.replace('.h', '').replace('-', '_') + header_id_base = header_id_base.upper() + + header_id = f'{HEADER_ID_PREFIX}{header_id_base}{HEADER_ID_SUFFIX}' + + return header_id + + +def main(): + exit_code = 0 + + header_file_lst = _get_header_file_lst() + for header_file in header_file_lst: + header_id = _get_header_id(header_file) + + regex_pattern = f'^#(ifndef|define|endif //) {header_id}' + + with open(header_file, 'r', encoding='utf-8') as f: + header_file_contents = f.readlines() + + count = 0 + for header_file_contents_string in header_file_contents: + include_guard_lst = re.findall( + regex_pattern, header_file_contents_string) + + count += len(include_guard_lst) + + if count != 3: + print(f'{header_file} seems to be missing the expected ' + 'include guard:') + print(f' #ifndef {header_id}') + print(f' #define {header_id}') + print(' ...') + print(f' #endif // {header_id}\n') + exit_code = 1 + + sys.exit(exit_code) + + +if __name__ == '__main__': + main() diff --git a/test/lint/lint-include-guards.sh b/test/lint/lint-include-guards.sh deleted file mode 100755 index e88680aec0..0000000000 --- a/test/lint/lint-include-guards.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Check include guards. - -export LC_ALL=C -HEADER_ID_PREFIX="BITCOIN_" -HEADER_ID_SUFFIX="_H" - -REGEXP_EXCLUDE_FILES_WITH_PREFIX="src/(crypto/ctaes/|dashbls/|immer/|leveldb/|crc32c/|secp256k1/|minisketch/|test/fuzz/FuzzedDataProvider.h|tinyformat.h|bench/nanobench.h|univalue/|ctpl_stl.h|bls/|crypto/x11/sph|gsl|util/expected.h)" - -EXIT_CODE=0 -for HEADER_FILE in $(git ls-files -- "*.h" | grep -vE "^${REGEXP_EXCLUDE_FILES_WITH_PREFIX}") -do - HEADER_ID_BASE=$(cut -f2- -d/ <<< "${HEADER_FILE}" | sed "s/\.h$//g" | tr / _ | tr - _ | tr "[:lower:]" "[:upper:]") - HEADER_ID="${HEADER_ID_PREFIX}${HEADER_ID_BASE}${HEADER_ID_SUFFIX}" - if [[ $(grep -cE "^#(ifndef|define) ${HEADER_ID}" "${HEADER_FILE}") != 2 ]]; then - echo "${HEADER_FILE} seems to be missing the expected include guard:" - echo " #ifndef ${HEADER_ID}" - echo " #define ${HEADER_ID}" - echo " ..." - echo " #endif // ${HEADER_ID}" - echo - EXIT_CODE=1 - fi -done -exit ${EXIT_CODE} diff --git a/test/lint/lint-python-utf8-encoding.py b/test/lint/lint-python-utf8-encoding.py new file mode 100755 index 0000000000..4f8b307c04 --- /dev/null +++ b/test/lint/lint-python-utf8-encoding.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. +# +# Make sure we explicitly open all text files using UTF-8 (or ASCII) encoding to +# avoid potential issues on the BSDs where the locale is not always set. + +import sys +import re + +from subprocess import check_output, CalledProcessError + +EXCLUDED_DIRS = ["src/crc32c/", + "src/secp256k1/"] + + +def get_exclude_args(): + return [":(exclude)" + dir for dir in EXCLUDED_DIRS] + + +def check_fileopens(): + fileopens = list() + + try: + fileopens = check_output(["git", "grep", r" open(", "--", "*.py"] + get_exclude_args(), universal_newlines=True, encoding="utf8").splitlines() + except CalledProcessError as e: + if e.returncode > 1: + raise e + + filtered_fileopens = [fileopen for fileopen in fileopens if not re.search(r"encoding=.(ascii|utf8|utf-8).|open\([^,]*, ['\"][^'\"]*b[^'\"]*['\"]", fileopen)] + + return filtered_fileopens + + +def check_checked_outputs(): + checked_outputs = list() + + try: + checked_outputs = check_output(["git", "grep", "check_output(", "--", "*.py"] + get_exclude_args(), universal_newlines=True, encoding="utf8").splitlines() + except CalledProcessError as e: + if e.returncode > 1: + raise e + + filtered_checked_outputs = [checked_output for checked_output in checked_outputs if re.search(r"universal_newlines=True", checked_output) and not re.search(r"encoding=.(ascii|utf8|utf-8).", checked_output)] + + return filtered_checked_outputs + + +def main(): + exit_code = 0 + + nonexplicit_utf8_fileopens = check_fileopens() + if nonexplicit_utf8_fileopens: + print("Python's open(...) seems to be used to open text files without explicitly specifying encoding='utf8':\n") + for fileopen in nonexplicit_utf8_fileopens: + print(fileopen) + exit_code = 1 + + nonexplicit_utf8_checked_outputs = check_checked_outputs() + if nonexplicit_utf8_checked_outputs: + if nonexplicit_utf8_fileopens: + print("\n") + print("Python's check_output(...) seems to be used to get program outputs without explicitly specifying encoding='utf8':\n") + for checked_output in nonexplicit_utf8_checked_outputs: + print(checked_output) + exit_code = 1 + + sys.exit(exit_code) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-python-utf8-encoding.sh b/test/lint/lint-python-utf8-encoding.sh deleted file mode 100755 index 83c34bef7f..0000000000 --- a/test/lint/lint-python-utf8-encoding.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Make sure we explicitly open all text files using UTF-8 (or ASCII) encoding to -# avoid potential issues on the BSDs where the locale is not always set. - -export LC_ALL=C -EXIT_CODE=0 -OUTPUT=$(git grep " open(" -- "*.py" ":(exclude)src/crc32c/" ":(exclude)src/secp256k1/" | grep -vE "encoding=.(ascii|utf8|utf-8)." | grep -vE "open\([^,]*, ['\"][^'\"]*b[^'\"]*['\"]") -if [[ ${OUTPUT} != "" ]]; then - echo "Python's open(...) seems to be used to open text files without explicitly" - echo "specifying encoding=\"utf8\":" - echo - echo "${OUTPUT}" - EXIT_CODE=1 -fi -OUTPUT=$(git grep "check_output(" -- "*.py" ":(exclude)src/crc32c/" ":(exclude)src/secp256k1/" | grep "universal_newlines=True" | grep -vE "encoding=.(ascii|utf8|utf-8).") -if [[ ${OUTPUT} != "" ]]; then - echo "Python's check_output(...) seems to be used to get program outputs without explicitly" - echo "specifying encoding=\"utf8\":" - echo - echo "${OUTPUT}" - EXIT_CODE=1 -fi -exit ${EXIT_CODE} diff --git a/test/lint/lint-python.py b/test/lint/lint-python.py new file mode 100755 index 0000000000..61d2c49a46 --- /dev/null +++ b/test/lint/lint-python.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" +Check for specified flake8 and mypy warnings in python files. +""" + +import os +import pkg_resources +import subprocess +import sys + +DEPS = ['flake8', 'mypy', 'pyzmq'] +MYPY_CACHE_DIR = f"{os.getenv('BASE_ROOT_DIR', '')}/test/.mypy_cache" +FILES_ARGS = ['git', 'ls-files', '--','test/functional/*.py', 'contrib/devtools/*.py', ':(exclude)contrib/devtools/github-merge.py'] +EXCLUDE_DIRS = ['src/dashbls/', + 'src/immer/'] + +ENABLED = ( + 'E101,' # indentation contains mixed spaces and tabs + 'E112,' # expected an indented block + 'E113,' # unexpected indentation + 'E115,' # expected an indented block (comment) + 'E116,' # unexpected indentation (comment) + 'E125,' # continuation line with same indent as next logical line + 'E129,' # visually indented line with same indent as next logical line + 'E131,' # continuation line unaligned for hanging indent + 'E133,' # closing bracket is missing indentation + 'E223,' # tab before operator + 'E224,' # tab after operator + 'E242,' # tab after ',' + 'E266,' # too many leading '#' for block comment + 'E271,' # multiple spaces after keyword + 'E272,' # multiple spaces before keyword + 'E273,' # tab after keyword + 'E274,' # tab before keyword + # TODO: enable it after bitcoin/bitcoin#26257 - too many warnings with newer flake + # 'E275,' # missing whitespace after keyword + 'E304,' # blank lines found after function decorator + 'E306,' # expected 1 blank line before a nested definition + 'E401,' # multiple imports on one line + 'E402,' # module level import not at top of file + 'E502,' # the backslash is redundant between brackets + 'E701,' # multiple statements on one line (colon) + 'E702,' # multiple statements on one line (semicolon) + 'E703,' # statement ends with a semicolon + 'E711,' # comparison to None should be 'if cond is None:' + 'E714,' # test for object identity should be "is not" + 'E721,' # do not compare types, use "isinstance()" + 'E742,' # do not define classes named "l", "O", or "I" + 'E743,' # do not define functions named "l", "O", or "I" + 'E901,' # SyntaxError: invalid syntax + 'E902,' # TokenError: EOF in multi-line string + 'F401,' # module imported but unused + 'F402,' # import module from line N shadowed by loop variable + 'F403,' # 'from foo_module import *' used; unable to detect undefined names + 'F404,' # future import(s) name after other statements + 'F405,' # foo_function may be undefined, or defined from star imports: bar_module + 'F406,' # "from module import *" only allowed at module level + 'F407,' # an undefined __future__ feature name was imported + 'F601,' # dictionary key name repeated with different values + 'F602,' # dictionary key variable name repeated with different values + 'F621,' # too many expressions in an assignment with star-unpacking + 'F622,' # two or more starred expressions in an assignment (a, *b, *c = d) + 'F631,' # assertion test is a tuple, which are always True + 'F632,' # use ==/!= to compare str, bytes, and int literals + 'F701,' # a break statement outside of a while or for loop + 'F702,' # a continue statement outside of a while or for loop + 'F703,' # a continue statement in a finally block in a loop + 'F704,' # a yield or yield from statement outside of a function + 'F705,' # a return statement with arguments inside a generator + 'F706,' # a return statement outside of a function/method + 'F707,' # an except: block as not the last exception handler + 'F811,' # redefinition of unused name from line N + 'F812,' # list comprehension redefines 'foo' from line N + 'F821,' # undefined name 'Foo' + 'F822,' # undefined name name in __all__ + 'F823,' # local variable name … referenced before assignment + 'F831,' # duplicate argument name in function definition + 'F841,' # local variable 'foo' is assigned to but never used + 'W191,' # indentation contains tabs + 'W291,' # trailing whitespace + 'W292,' # no newline at end of file + 'W293,' # blank line contains whitespace + 'W601,' # .has_key() is deprecated, use "in" + 'W602,' # deprecated form of raising exception + 'W603,' # "<>" is deprecated, use "!=" + 'W604,' # backticks are deprecated, use "repr()" + # 'W605,' # invalid escape sequence "x" + 'W606,' # 'async' and 'await' are reserved keywords starting with Python 3.7 +) + + +def check_dependencies(): + working_set = {pkg.key for pkg in pkg_resources.working_set} + + for dep in DEPS: + if dep not in working_set: + print(f"Skipping Python linting since {dep} is not installed.") + exit(0) + + +def main(): + check_dependencies() + + if len(sys.argv) > 1: + flake8_files = sys.argv[1:] + else: + files_args = ['git', 'ls-files', '--', '*.py'] + for dir in EXCLUDE_DIRS: + files_args += [f':(exclude){dir}'] + flake8_files = subprocess.check_output(files_args).decode("utf-8").splitlines() + + flake8_args = ['flake8', '--ignore=B,C,E,F,I,N,W', f'--select={ENABLED}'] + flake8_files + flake8_env = os.environ.copy() + flake8_env["PYTHONWARNINGS"] = "ignore" + + try: + subprocess.check_call(flake8_args, env=flake8_env) + except subprocess.CalledProcessError: + exit(1) + + mypy_files = subprocess.check_output(FILES_ARGS).decode("utf-8").splitlines() + mypy_args = ['mypy', '--ignore-missing-imports', '--show-error-codes'] + mypy_files + + try: + subprocess.check_call(mypy_args) + except subprocess.CalledProcessError: + exit(1) + + +if __name__ == "__main__": + main() diff --git a/test/lint/lint-python.sh b/test/lint/lint-python.sh deleted file mode 100755 index c7ecb11ea0..0000000000 --- a/test/lint/lint-python.sh +++ /dev/null @@ -1,120 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2017-2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Check for specified flake8 warnings in python files. - -export LC_ALL=C -export MYPY_CACHE_DIR="${BASE_ROOT_DIR}/test/.mypy_cache" - -enabled=( - E101 # indentation contains mixed spaces and tabs - E112 # expected an indented block - E113 # unexpected indentation - E115 # expected an indented block (comment) - E116 # unexpected indentation (comment) - E125 # continuation line with same indent as next logical line - E129 # visually indented line with same indent as next logical line - E131 # continuation line unaligned for hanging indent - E133 # closing bracket is missing indentation - E223 # tab before operator - E224 # tab after operator - E242 # tab after ',' - E266 # too many leading '#' for block comment - E271 # multiple spaces after keyword - E272 # multiple spaces before keyword - E273 # tab after keyword - E274 # tab before keyword - # TODO: enable it after bitcoin/bitcoin#26257 - too many warnings with newer flake - #E275 # missing whitespace after keyword - E304 # blank lines found after function decorator - E306 # expected 1 blank line before a nested definition - E401 # multiple imports on one line - E402 # module level import not at top of file - E502 # the backslash is redundant between brackets - E701 # multiple statements on one line (colon) - E702 # multiple statements on one line (semicolon) - E703 # statement ends with a semicolon - E711 # comparison to None should be 'if cond is None:' - E714 # test for object identity should be "is not" - E721 # do not compare types, use "isinstance()" - E742 # do not define classes named "l", "O", or "I" - E743 # do not define functions named "l", "O", or "I" - E901 # SyntaxError: invalid syntax - E902 # TokenError: EOF in multi-line string - F401 # module imported but unused - F402 # import module from line N shadowed by loop variable - F403 # 'from foo_module import *' used; unable to detect undefined names - F404 # future import(s) name after other statements - F405 # foo_function may be undefined, or defined from star imports: bar_module - F406 # "from module import *" only allowed at module level - F407 # an undefined __future__ feature name was imported - F601 # dictionary key name repeated with different values - F602 # dictionary key variable name repeated with different values - F621 # too many expressions in an assignment with star-unpacking - F622 # two or more starred expressions in an assignment (a, *b, *c = d) - F631 # assertion test is a tuple, which are always True - F632 # use ==/!= to compare str, bytes, and int literals - F701 # a break statement outside of a while or for loop - F702 # a continue statement outside of a while or for loop - F703 # a continue statement in a finally block in a loop - F704 # a yield or yield from statement outside of a function - F705 # a return statement with arguments inside a generator - F706 # a return statement outside of a function/method - F707 # an except: block as not the last exception handler - F811 # redefinition of unused name from line N - F812 # list comprehension redefines 'foo' from line N - F821 # undefined name 'Foo' - F822 # undefined name name in __all__ - F823 # local variable name … referenced before assignment - F831 # duplicate argument name in function definition - F841 # local variable 'foo' is assigned to but never used - W191 # indentation contains tabs - W291 # trailing whitespace - W292 # no newline at end of file - W293 # blank line contains whitespace - W601 # .has_key() is deprecated, use "in" - W602 # deprecated form of raising exception - W603 # "<>" is deprecated, use "!=" - W604 # backticks are deprecated, use "repr()" - # W605 # invalid escape sequence "x" - W606 # 'async' and 'await' are reserved keywords starting with Python 3.7 -) - -if ! command -v flake8 > /dev/null; then - echo "Skipping Python linting since flake8 is not installed." - exit 0 -elif PYTHONWARNINGS="ignore" flake8 --version | grep -q "Python 2"; then - echo "Skipping Python linting since flake8 is running under Python 2. Install the Python 3 version of flake8." - exit 0 -fi - -FLAKECMD=flake8 - -if command -v flake8-cached > /dev/null; then - FLAKECMD=flake8-cached -else - echo "Consider install flake8-cached for cached flake8 results." -fi - -EXIT_CODE=0 - -# shellcheck disable=SC2046 -if ! PYTHONWARNINGS="ignore" $FLAKECMD --ignore=B,C,E,F,I,N,W --select=$(IFS=","; echo "${enabled[*]}") $( - if [[ $# == 0 ]]; then - git ls-files "*.py" | grep -vE "src/(immer)/" - else - echo "$@" - fi -); then - EXIT_CODE=1 -fi - -mapfile -t FILES < <(git ls-files "test/functional/*.py" "contrib/devtools/*.py" | grep -v contrib/devtools/github-merge.py) -if ! mypy --ignore-missing-imports --show-error-codes "${FILES[@]}"; then - EXIT_CODE=1 -fi - -exit $EXIT_CODE diff --git a/test/lint/lint-shell-locale.py b/test/lint/lint-shell-locale.py new file mode 100755 index 0000000000..52928041cb --- /dev/null +++ b/test/lint/lint-shell-locale.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" +Make sure all shell scripts are: +a.) explicitly opt out of locale dependence using + "export LC_ALL=C" or "export LC_ALL=C.UTF-8", or +b.) explicitly opt in to locale dependence using the annotation below. +""" + +import subprocess +import sys +import re + +OPT_IN_LINE = '# This script is intentionally locale dependent by not setting \"export LC_ALL=C\"' + +OPT_OUT_LINES = [ + 'export LC_ALL=C', + 'export LC_ALL=C.UTF-8', +] + +def get_shell_files_list(): + command = [ + 'git', + 'ls-files', + '--', + '*.sh', + ] + try: + return subprocess.check_output(command, stderr = subprocess.STDOUT).decode('utf-8').splitlines() + except subprocess.CalledProcessError as e: + if e.returncode > 1: # return code is 1 when match is empty + print(e.output.decode('utf-8'), end='') + sys.exit(1) + return [] + +def main(): + exit_code = 0 + shell_files = get_shell_files_list() + for file_path in shell_files: + if re.search('src/(dashbls|secp256k1|minisketch|univalue)/', file_path): + continue + + with open(file_path, 'r', encoding='utf-8') as file_obj: + contents = file_obj.read() + + if OPT_IN_LINE in contents: + continue + + non_comment_pattern = re.compile(r'^\s*((?!#).+)$', re.MULTILINE) + non_comment_lines = re.findall(non_comment_pattern, contents) + if not non_comment_lines: + continue + + first_non_comment_line = non_comment_lines[0] + if first_non_comment_line not in OPT_OUT_LINES: + print(f'Missing "export LC_ALL=C" (to avoid locale dependence) as first non-comment non-empty line in {file_path}') + exit_code = 1 + + return sys.exit(exit_code) + +if __name__ == '__main__': + main() + diff --git a/test/lint/lint-shell-locale.sh b/test/lint/lint-shell-locale.sh deleted file mode 100755 index b21b5d9b8d..0000000000 --- a/test/lint/lint-shell-locale.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Make sure all shell scripts: -# a.) explicitly opt out of locale dependence using -# "export LC_ALL=C" or "export LC_ALL=C.UTF-8", or -# b.) explicitly opt in to locale dependence using the annotation below. - -export LC_ALL=C - -EXIT_CODE=0 -for SHELL_SCRIPT in $(git ls-files -- "*.sh" | grep -vE "src/(dashbls|secp256k1|minisketch|univalue)/"); do - if grep -q "# This script is intentionally locale dependent by not setting \"export LC_ALL=C\"" "${SHELL_SCRIPT}"; then - continue - fi - FIRST_NON_COMMENT_LINE=$(grep -vE '^(#.*)?$' "${SHELL_SCRIPT}" | head -1) - if [[ ${FIRST_NON_COMMENT_LINE} != "export LC_ALL=C" && ${FIRST_NON_COMMENT_LINE} != "export LC_ALL=C.UTF-8" ]]; then - echo "Missing \"export LC_ALL=C\" (to avoid locale dependence) as first non-comment non-empty line in ${SHELL_SCRIPT}" - EXIT_CODE=1 - fi -done -exit ${EXIT_CODE} diff --git a/test/lint/lint-shell.py b/test/lint/lint-shell.py new file mode 100755 index 0000000000..2b13bbb06e --- /dev/null +++ b/test/lint/lint-shell.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +# +# Copyright (c) 2018-2022 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +""" +Check for shellcheck warnings in shell scripts. +""" + +import subprocess +import re +import sys + +# Disabled warnings: +DISABLED = [ + 'SC2046', # Quote this to prevent word splitting. + 'SC2086', # Double quote to prevent globbing and word splitting. + 'SC2162', # read without -r will mangle backslashes. +] + +def check_shellcheck_install(): + try: + subprocess.run(['shellcheck', '--version'], stdout=subprocess.DEVNULL, check=True) + except FileNotFoundError: + print('Skipping shell linting since shellcheck is not installed.') + sys.exit(0) + +def get_files(command): + output = subprocess.run(command, stdout=subprocess.PIPE, universal_newlines=True) + files = output.stdout.split('\n') + + # remove whitespace element + files = list(filter(None, files)) + return files + +def main(): + check_shellcheck_install() + + # build the `exclude` flag + exclude = '--exclude=' + ','.join(DISABLED) + + # build the `sourced files` list + sourced_files_cmd = [ + 'git', + 'grep', + '-El', + r'^# shellcheck shell=', + ] + sourced_files = get_files(sourced_files_cmd) + + # build the `guix files` list + guix_files_cmd = [ + 'git', + 'grep', + '-El', + r'^#!\/usr\/bin\/env bash', + '--', + 'contrib/guix', + 'contrib/shell', + ] + guix_files = get_files(guix_files_cmd) + + # build the other script files list + files_cmd = [ + 'git', + 'ls-files', + '--', + '*.sh', + ] + files = get_files(files_cmd) + # remove everything that doesn't match this regex + reg = re.compile(r'src/[dashbls,immer,leveldb,secp256k1,minisketch,univalue]') + files[:] = [file for file in files if not reg.match(file)] + + # build the `shellcheck` command + shellcheck_cmd = [ + 'shellcheck', + '--external-sources', + '--check-sourced', + '--source-path=SCRIPTDIR', + ] + shellcheck_cmd.append(exclude) + shellcheck_cmd.extend(sourced_files) + shellcheck_cmd.extend(guix_files) + shellcheck_cmd.extend(files) + + # run the `shellcheck` command + try: + subprocess.check_call(shellcheck_cmd) + except subprocess.CalledProcessError: + sys.exit(1) + +if __name__ == '__main__': + main() diff --git a/test/lint/lint-shell.sh b/test/lint/lint-shell.sh deleted file mode 100755 index 7741345453..0000000000 --- a/test/lint/lint-shell.sh +++ /dev/null @@ -1,39 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2020 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. -# -# Check for shellcheck warnings in shell scripts. - -export LC_ALL=C - -# Disabled warnings: -disabled=( - SC2046 # Quote this to prevent word splitting. - SC2086 # Double quote to prevent globbing and word splitting. - SC2162 # read without -r will mangle backslashes. -) - -EXIT_CODE=0 - -if ! command -v shellcheck > /dev/null; then - echo "Skipping shell linting since shellcheck is not installed." - exit $EXIT_CODE -fi - -if ! command -v gawk > /dev/null; then - echo "Skipping shell linting since gawk is not installed." - exit $EXIT_CODE -fi - -SHELLCHECK_CMD=(shellcheck --external-sources --check-sourced) -EXCLUDE="--exclude=$(IFS=','; echo "${disabled[*]}")" -# Check shellcheck directive used for sourced files -mapfile -t SOURCED_FILES < <(git ls-files | xargs gawk '/^# shellcheck shell=/ {print FILENAME} {nextfile}') -mapfile -t FILES < <(git ls-files -- '*.sh' | grep -vE 'src/(dashbls|immer|leveldb|secp256k1|minisketch|univalue)/') -if ! "${SHELLCHECK_CMD[@]}" "$EXCLUDE" "${SOURCED_FILES[@]}" "${FILES[@]}"; then - EXIT_CODE=1 -fi - -exit $EXIT_CODE