Merge pull request #5144 from vijaydasmp/bp21_11

backport: Merge bitcoin#18664,18917,18901,18939,18875,19452,19548,19595,20300,20375

.gitignore

@@ -11,7 +11,7 @@ src/dashd
 src/dash-cli
 src/dash-tx
 src/dash-wallet
-src/test/fuzz
+src/test/fuzz/*
 !src/test/fuzz/*.*
 src/test/test_dash
 src/qt/test/test_dash-qt

ci/test/00_setup_env_native_fuzz.sh

@@ -15,4 +15,4 @@ export RUN_UNIT_TESTS=false
 export RUN_INTEGRATION_TESTS=false
 export RUN_FUZZ_TESTS=true
 export GOAL="install"
-export BITCOIN_CONFIG="--enable-zmq --disable-ccache --enable-fuzz --with-sanitizers=fuzzer,address,undefined CC=clang CXX=clang++"
+export BITCOIN_CONFIG="--enable-zmq --disable-ccache --enable-fuzz --with-sanitizers=fuzzer,address,undefined --enable-c++17 CC=clang CXX=clang++"

ci/test/00_setup_env_native_fuzz_with_valgrind.sh

@@ -14,5 +14,5 @@ export RUN_FUNCTIONAL_TESTS=false
 export RUN_FUZZ_TESTS=true
 export FUZZ_TESTS_CONFIG="--valgrind"
 export GOAL="install"
-export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer CC=clang-8 CXX=clang++-8"
+export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer --enable-c++17 CC=clang-8 CXX=clang++-8"
 # Use clang-8, instead of default clang on bionic, which is clang-6 and does not come with libfuzzer on aarch64

doc/fuzzing.md

@@ -8,7 +8,7 @@ To quickly get started fuzzing Dash Core using [libFuzzer](https://llvm.org/docs
 $ git clone https://github.com/dashpay/dash
 $ cd dash/
 $ ./autogen.sh
-$ CC=clang CXX=clang++ ./configure --enable-fuzz --with-sanitizers=address,fuzzer,undefined
+$ CC=clang CXX=clang++ ./configure --enable-fuzz --with-sanitizers=address,fuzzer,undefined --enable-c++17
 # macOS users: If you have problem with this step then make sure to read "macOS hints for
 # libFuzzer" on https://github.com/dashpay/dash/blob/develop/doc/fuzzing.md#macos-hints-for-libfuzzer
 $ make
@@ -103,7 +103,7 @@ You may also need to take care of giving the correct path for `clang` and
 Full configure that was tested on macOS Catalina with `brew` installed `llvm`:
 
 ```sh
-./configure --enable-fuzz --with-sanitizers=fuzzer,address,undefined CC=/usr/local/opt/llvm/bin/clang CXX=/usr/local/opt/llvm/bin/clang++ --disable-asm
+./configure --enable-fuzz --with-sanitizers=fuzzer,address,undefined CC=/usr/local/opt/llvm/bin/clang CXX=/usr/local/opt/llvm/bin/clang++ --disable-asm --enable-c++17
 ```
 
 Read the [libFuzzer documentation](https://llvm.org/docs/LibFuzzer.html) for more information. This [libFuzzer tutorial](https://github.com/google/fuzzing/blob/master/tutorial/libFuzzerTutorial.md) might also be of interest.
@@ -121,7 +121,9 @@ $ git clone https://github.com/google/afl
 $ make -C afl/
 $ make -C afl/llvm_mode/
 $ ./autogen.sh
-$ CC=$(pwd)/afl/afl-clang-fast CXX=$(pwd)/afl/afl-clang-fast++ ./configure --enable-fuzz
+# It is possible to compile with afl-gcc and afl-g++ instead of afl-clang. However, running afl-fuzz
+# may require more memory via the -m flag.
+$ CC=$(pwd)/afl/afl-clang-fast CXX=$(pwd)/afl/afl-clang-fast++ ./configure --enable-fuzz --enable-c++17
 $ make
 # For macOS you may need to ignore x86 compilation checks when running "make". If so,
 # try compiling using: AFL_NO_X86=1 make
@@ -148,7 +150,7 @@ $ git clone https://github.com/google/honggfuzz
 $ cd honggfuzz/
 $ make
 $ cd ..
-$ CC=$(pwd)/honggfuzz/hfuzz_cc/hfuzz-clang CXX=$(pwd)/honggfuzz/hfuzz_cc/hfuzz-clang++ ./configure --enable-fuzz --with-sanitizers=address,undefined
+$ CC=$(pwd)/honggfuzz/hfuzz_cc/hfuzz-clang CXX=$(pwd)/honggfuzz/hfuzz_cc/hfuzz-clang++ ./configure --enable-fuzz --with-sanitizers=address,undefined --enable-c++17
 $ make
 $ mkdir -p inputs/
 $ FUZZ=process_message honggfuzz/honggfuzz -i inputs/ -- src/test/fuzz/fuzz

src/test/fuzz/asmap_direct.cpp

@@ -7,6 +7,7 @@
 #include <test/fuzz/fuzz.h>
 
 #include <cstddef>
+#include <optional>
 #include <vector>
 
 #include <assert.h>
@@ -14,20 +15,19 @@
 FUZZ_TARGET(asmap_direct)
 {
     // Encoding: [asmap using 1 bit / byte] 0xFF [addr using 1 bit / byte]
-    bool have_sep = false;
+    std::optional<size_t> sep_pos_opt;
-    size_t sep_pos;
     for (size_t pos = 0; pos < buffer.size(); ++pos) {
         uint8_t x = buffer[pos];
         if ((x & 0xFE) == 0) continue;
         if (x == 0xFF) {
-            if (have_sep) return;
+            if (sep_pos_opt) return;
-            have_sep = true;
+            sep_pos_opt = pos;
-            sep_pos = pos;
         } else {
             return;
         }
     }
-    if (!have_sep) return; // Needs exactly 1 separator
+    if (!sep_pos_opt) return; // Needs exactly 1 separator
+    const size_t sep_pos{sep_pos_opt.value()};
     if (buffer.size() - sep_pos - 1 > 128) return; // At most 128 bits in IP address
 
     // Checks on asmap

src/test/fuzz/descriptor_parse.cpp

@@ -12,7 +12,8 @@
 void initialize_descriptor_parse()
 {
     static const ECCVerifyHandle verify_handle;
-    SelectParams(CBaseChainParams::REGTEST);
+    ECC_Start();
+    SelectParams(CBaseChainParams::MAIN);
 }
 
 FUZZ_TARGET_INIT(descriptor_parse, initialize_descriptor_parse)

src/test/fuzz/merkleblock.cpp

@@ -16,12 +16,36 @@
 FUZZ_TARGET(merkleblock)
 {
     FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size());
-    std::optional<CPartialMerkleTree> partial_merkle_tree = ConsumeDeserializable<CPartialMerkleTree>(fuzzed_data_provider);
+    CPartialMerkleTree partial_merkle_tree;
-    if (!partial_merkle_tree) {
+    switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 1)) {
-        return;
+    case 0: {
+        const std::optional<CPartialMerkleTree> opt_partial_merkle_tree = ConsumeDeserializable<CPartialMerkleTree>(fuzzed_data_provider);
+        if (opt_partial_merkle_tree) {
+            partial_merkle_tree = *opt_partial_merkle_tree;
+        }
+        break;
     }
-    (void)partial_merkle_tree->GetNumTransactions();
+    case 1: {
+        CMerkleBlock merkle_block;
+        const std::optional<CBlock> opt_block = ConsumeDeserializable<CBlock>(fuzzed_data_provider);
+        CBloomFilter bloom_filter;
+        std::set<uint256> txids;
+        if (opt_block && !opt_block->vtx.empty()) {
+            if (fuzzed_data_provider.ConsumeBool()) {
+                merkle_block = CMerkleBlock{*opt_block, bloom_filter};
+            } else if (fuzzed_data_provider.ConsumeBool()) {
+                while (fuzzed_data_provider.ConsumeBool()) {
+                    txids.insert(ConsumeUInt256(fuzzed_data_provider));
+                }
+                merkle_block = CMerkleBlock{*opt_block, txids};
+            }
+        }
+        partial_merkle_tree = merkle_block.txn;
+        break;
+    }
+    }
+    (void)partial_merkle_tree.GetNumTransactions();
     std::vector<uint256> matches;
     std::vector<unsigned int> indices;
-    (void)partial_merkle_tree->ExtractMatches(matches, indices);
+    (void)partial_merkle_tree.ExtractMatches(matches, indices);
 }

src/test/fuzz/script.cpp

@@ -46,7 +46,7 @@ FUZZ_TARGET_INIT(script, initialize_script)
     std::vector<unsigned char> compressed;
     if (CompressScript(script, compressed)) {
         const unsigned int size = compressed[0];
-        assert(size >= 0 && size <= 5);
+        assert(size <= 5);
         CScript decompressed_script;
         const bool ok = DecompressScript(decompressed_script, size, compressed);
         assert(ok);

src/test/fuzz/scriptnum_ops.cpp

@@ -33,7 +33,7 @@ FUZZ_TARGET(scriptnum_ops)
         case 0: {
             const int64_t i = fuzzed_data_provider.ConsumeIntegral<int64_t>();
             assert((script_num == i) != (script_num != i));
-            assert((script_num <= i) != script_num > i);
+            assert((script_num <= i) != (script_num > i));
             assert((script_num >= i) != (script_num < i));
             // Avoid signed integer overflow:
             // script/script.h:264:93: runtime error: signed integer overflow: -2261405121394637306 + -9223372036854775802 cannot be represented in type 'long'

src/test/fuzz/signature_checker.cpp

@@ -27,17 +27,17 @@ public:
     {
     }
 
-    virtual bool CheckSig(const std::vector<unsigned char>& scriptSig, const std::vector<unsigned char>& vchPubKey, const CScript& scriptCode, SigVersion sigversion) const
+    bool CheckSig(const std::vector<unsigned char>& scriptSig, const std::vector<unsigned char>& vchPubKey, const CScript& scriptCode, SigVersion sigversion) const override
     {
         return m_fuzzed_data_provider.ConsumeBool();
     }
 
-    virtual bool CheckLockTime(const CScriptNum& nLockTime) const
+    bool CheckLockTime(const CScriptNum& nLockTime) const override
     {
         return m_fuzzed_data_provider.ConsumeBool();
     }
 
-    virtual bool CheckSequence(const CScriptNum& nSequence) const
+    bool CheckSequence(const CScriptNum& nSequence) const override
     {
         return m_fuzzed_data_provider.ConsumeBool();
     }

src/test/fuzz/system.cpp

@@ -85,7 +85,7 @@ FUZZ_TARGET(system)
         case 7: {
             const std::vector<std::string> random_arguments = ConsumeRandomLengthStringVector(fuzzed_data_provider);
             std::vector<const char*> argv;
-            argv.resize(random_arguments.size());
+            argv.reserve(random_arguments.size());
             for (const std::string& random_argument : random_arguments) {
                 argv.push_back(random_argument.c_str());
             }