merge bitcoin#23114: Add minisketch subtree and integrate into build/test

excludes: - b6487dc4ef47ec9ea894eceac25f37d0b806f8aa (subtree manipulation done in previous commits) - 07f0a61ef711a2f75ded3d73545bfabdf2a64fef (see above) - 0eb7928ab8d9dcb840e4965bfa81deb752b00dfa (we don't support MSVC build method)
2024-12-24 19:42:46 +01:00 · 2021-07-20 18:18:43 +00:00 · 2021-07-20 18:18:43 +00:00 · a6b26b5dc1
commit a6b26b5dc1
parent 57ef0db047
23 changed files with 243 additions and 17 deletions
--- a/Makefile.am
+++ b/Makefile.am
@ -177,6 +177,7 @@ LCOV_FILTER_PATTERN = \
 	-p "src/bench/" \
 	-p "src/univalue" \
 	-p "src/crypto/ctaes" \
+	-p "src/minisketch" \
 	-p "src/secp256k1" \
 	-p "depends"

--- a/ci/lint/06_script.sh
+++ b/ci/lint/06_script.sh
@ -17,6 +17,7 @@ export COMMIT_RANGE
 # check with -r to not have to fetch all the remotes.
 test/lint/git-subtree-check.sh src/crypto/ctaes
 test/lint/git-subtree-check.sh src/secp256k1
+test/lint/git-subtree-check.sh src/minisketch
 test/lint/git-subtree-check.sh src/univalue
 test/lint/git-subtree-check.sh src/leveldb
 test/lint/check-doc.py
--- a/ci/test/wrap-wine.sh
+++ b/ci/test/wrap-wine.sh
@ -6,7 +6,7 @@

 export LC_ALL=C.UTF-8

-for b_name in {"${BASE_OUTDIR}/bin"/*,src/secp256k1/*tests,src/univalue/{no_nul,test_json,unitester,object}}.exe; do
+for b_name in {"${BASE_OUTDIR}/bin"/*,src/secp256k1/*tests,src/minisketch/test{,-verify},src/univalue/{no_nul,test_json,unitester,object}}.exe; do
    # shellcheck disable=SC2044
    for b in $(find "${BASE_ROOT_DIR}" -executable -type f -name "$(basename $b_name)"); do
      if (file "$b" | grep "Windows"); then
--- a/configure.ac
+++ b/configure.ac
@ -537,6 +537,24 @@ AX_CHECK_COMPILE_FLAG([-msse4.1],[[SSE41_CXXFLAGS="-msse4.1"]],,[[$CXXFLAG_WERRO
 AX_CHECK_COMPILE_FLAG([-mavx -mavx2],[[AVX2_CXXFLAGS="-mavx -mavx2"]],,[[$CXXFLAG_WERROR]])
 AX_CHECK_COMPILE_FLAG([-msse4 -msha],[[X86_SHANI_CXXFLAGS="-msse4 -msha"]],,[[$CXXFLAG_WERROR]])

+enable_clmul=
+AX_CHECK_COMPILE_FLAG([-mpclmul], [enable_clmul=yes], [], [$CXXFLAG_WERROR], [AC_LANG_PROGRAM([
+  #include <stdint.h>
+  #include <x86intrin.h>
+], [
+  __m128i a = _mm_cvtsi64_si128((uint64_t)7);
+  __m128i b = _mm_clmulepi64_si128(a, a, 37);
+  __m128i c = _mm_srli_epi64(b, 41);
+  __m128i d = _mm_xor_si128(b, c);
+  uint64_t e = _mm_cvtsi128_si64(d);
+  return e == 0;
+])])
+
+if test x$enable_clmul = xyes; then
+  CLMUL_CXXFLAGS="-mpclmul"
+  AC_DEFINE(HAVE_CLMUL, 1, [Define this symbol if clmul instructions can be used])
+fi
+
 TEMP_CXXFLAGS="$CXXFLAGS"
 CXXFLAGS="$CXXFLAGS $SSE42_CXXFLAGS"
 AC_MSG_CHECKING(for SSE4.2 intrinsics)
@ -1030,19 +1048,20 @@ AC_CHECK_DECLS([bswap_16, bswap_32, bswap_64],,,
                 #endif])

 AC_MSG_CHECKING(for __builtin_clzl)
+
 AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ ]], [[
 (void) __builtin_clzl(0);
  ]])],
- [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_BUILTIN_CLZL, 1, [Define this symbol if you have __builtin_clzl])],
- [ AC_MSG_RESULT(no)]
+ [ AC_MSG_RESULT(yes); have_clzl=yes; AC_DEFINE(HAVE_BUILTIN_CLZL, 1, [Define this symbol if you have __builtin_clzl])],
+ [ AC_MSG_RESULT(no); have_clzl=no;]
 )

 AC_MSG_CHECKING(for __builtin_clzll)
 AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ ]], [[
  (void) __builtin_clzll(0);
  ]])],
- [ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_BUILTIN_CLZLL, 1, [Define this symbol if you have __builtin_clzll])],
- [ AC_MSG_RESULT(no)]
+ [ AC_MSG_RESULT(yes); have_clzll=yes; AC_DEFINE(HAVE_BUILTIN_CLZLL, 1, [Define this symbol if you have __builtin_clzll])],
+ [ AC_MSG_RESULT(no); have_clzll=no;]
 )

 dnl Check for mallopt(M_ARENA_MAX) (to set glibc arenas)
@ -1815,6 +1834,10 @@ AM_CONDITIONAL([WORDS_BIGENDIAN],[test x$ac_cv_c_bigendian = xyes])
 AM_CONDITIONAL([USE_NATPMP],[test x$use_natpmp = xyes])
 AM_CONDITIONAL([USE_UPNP],[test x$use_upnp = xyes])

+dnl for minisketch
+AM_CONDITIONAL([ENABLE_CLMUL],[test x$enable_clmul = xyes])
+AM_CONDITIONAL([HAVE_CLZ],[test x$have_clzl$have_clzll = xyesyes])
+
 AC_DEFINE(CLIENT_VERSION_MAJOR, _CLIENT_VERSION_MAJOR, [Major version])
 AC_DEFINE(CLIENT_VERSION_MINOR, _CLIENT_VERSION_MINOR, [Minor version])
 AC_DEFINE(CLIENT_VERSION_BUILD, _CLIENT_VERSION_BUILD, [Version Build])
@ -1857,6 +1880,7 @@ AC_SUBST(SANITIZER_CXXFLAGS)
 AC_SUBST(SANITIZER_LDFLAGS)
 AC_SUBST(SSE42_CXXFLAGS)
 AC_SUBST(SSE41_CXXFLAGS)
+AC_SUBST(CLMUL_CXXFLAGS)
 AC_SUBST(AVX2_CXXFLAGS)
 AC_SUBST(X86_SHANI_CXXFLAGS)
 AC_SUBST(ARM_CRC_CXXFLAGS)
--- a/contrib/devtools/copyright_header.py
+++ b/contrib/devtools/copyright_header.py
@ -44,6 +44,7 @@ EXCLUDE_DIRS = [
    "src/gsl/",
    "src/immer/",
    "src/leveldb/",
+    "src/minisketch",
    "src/secp256k1/",
    "src/univalue/",
 ]
--- a/doc/developer-notes.md
+++ b/doc/developer-notes.md
@ -1063,6 +1063,9 @@ Current subtrees include:
 - src/univalue
  - Upstream at https://github.com/bitcoin-core/univalue ; actively maintained by Core contributors, deviates from upstream https://github.com/jgarzik/univalue

+- src/minisketch
+  - Upstream at https://github.com/sipa/minisketch ; maintained by Core contributors.
+
 Upgrading LevelDB
 ---------------------

--- a/src/Makefile.am
+++ b/src/Makefile.am
@ -36,7 +36,7 @@ BACKTRACE_LIB = -lbacktrace
 endif
 endif #ENABLE_STACKTRACES

-BITCOIN_INCLUDES=-I$(builddir) -I$(srcdir)/secp256k1/include -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) $(BDB_CPPFLAGS) $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS)
+BITCOIN_INCLUDES=-I$(builddir) -I$(srcdir)/$(MINISKETCH_INCLUDE_DIR_INT) -I$(srcdir)/secp256k1/include -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) $(BDB_CPPFLAGS) $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS)
 BITCOIN_INCLUDES+=-isystem$(srcdir)/dashbls/include -isystem$(srcdir)/dashbls/depends/relic/include -isystem$(srcdir)/dashbls/depends/minialloc/include
 BITCOIN_INCLUDES+=-isystem$(srcdir)/immer

@ -259,6 +259,7 @@ BITCOIN_CORE_H = \
  merkleblock.h \
  messagesigner.h \
  miner.h \
+  minisketchwrapper.h \
  net.h \
  net_permissions.h \
  net_processing.h \
@ -494,6 +495,7 @@ libbitcoin_server_a_SOURCES = \
  masternode/sync.cpp \
  masternode/utils.cpp \
  miner.cpp \
+  minisketchwrapper.cpp \
  net.cpp \
  netfulfilledman.cpp \
  netgroup.cpp \
@ -1059,8 +1061,11 @@ nodist_libbitcoin_ipc_a_SOURCES = $(libbitcoin_ipc_mpgen_output)
 CLEANFILES += $(libbitcoin_ipc_mpgen_output)
 endif

+include Makefile.minisketch.include
+
 include Makefile.crc32c.include
 include Makefile.leveldb.include
+
 include Makefile.test_util.include
 include Makefile.test_fuzz.include

--- a/src/Makefile.minisketch.include
+++ b/src/Makefile.minisketch.include
@ -0,0 +1,43 @@
+include minisketch/sources.mk
+
+LIBMINISKETCH_CPPFLAGS=
+LIBMINISKETCH_CPPFLAGS += -DDISABLE_DEFAULT_FIELDS -DENABLE_FIELD_32
+
+LIBMINISKETCH = minisketch/libminisketch.a
+MINISKETCH_LIBS = $(LIBMINISKETCH)
+
+if ENABLE_CLMUL
+LIBMINISKETCH_CLMUL = minisketch/libminisketch_clmul.a
+LIBMINISKETCH_CPPFLAGS += -DHAVE_CLMUL
+MINISKETCH_LIBS += $(LIBMINISKETCH_CLMUL)
+endif
+
+if HAVE_CLZ
+LIBMINISKETCH_CPPFLAGS += -DHAVE_CLZ
+endif
+
+EXTRA_LIBRARIES += $(MINISKETCH_LIBS)
+
+minisketch_libminisketch_clmul_a_SOURCES = $(MINISKETCH_FIELD_CLMUL_SOURCES_INT) $(MINISKETCH_FIELD_CLMUL_HEADERS_INT)
+minisketch_libminisketch_clmul_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) $(CLMUL_CXXFLAGS)
+minisketch_libminisketch_clmul_a_CPPFLAGS = $(AM_CPPFLAGS) $(LIBMINISKETCH_CPPFLAGS)
+
+minisketch_libminisketch_a_SOURCES = $(MINISKETCH_FIELD_GENERIC_SOURCES_INT) $(MINISKETCH_LIB_SOURCES_INT)
+minisketch_libminisketch_a_SOURCES += $(MINISKETCH_FIELD_GENERIC_HEADERS_INT) $(MINISKETCH_LIB_HEADERS_INT) $(MINISKETCH_DIST_HEADERS_INT)
+minisketch_libminisketch_a_CPPFLAGS = $(AM_CPPFLAGS) $(LIBMINISKETCH_CPPFLAGS)
+minisketch_libminisketch_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS)
+
+if ENABLE_TESTS
+if !ENABLE_FUZZ
+MINISKETCH_TEST = minisketch/test
+TESTS += $(MINISKETCH_TEST)
+noinst_PROGRAMS += $(MINISKETCH_TEST)
+
+minisketch_test_SOURCES = $(MINISKETCH_TEST_SOURCES_INT)
+minisketch_test_CPPFLAGS = $(AM_CPPFLAGS) $(LIBMINISKETCH_CPPFLAGS)
+minisketch_test_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS)
+minisketch_test_LDADD = $(MINISKETCH_LIBS)
+minisketch_test_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS)
+
+endif
+endif
--- a/src/Makefile.test.include
+++ b/src/Makefile.test.include
@ -130,6 +130,7 @@ BITCOIN_TESTS =\
  test/mempool_tests.cpp \
  test/merkle_tests.cpp \
  test/merkleblock_tests.cpp \
+  test/minisketch_tests.cpp \
  test/miner_tests.cpp \
  test/multisig_tests.cpp \
  test/net_peer_eviction_tests.cpp \
@ -219,7 +220,7 @@ if ENABLE_WALLET
 test_test_dash_LDADD += $(LIBBITCOIN_WALLET)
 endif
 test_test_dash_LDADD += $(LIBBITCOIN_SERVER) $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) \
-  $(LIBDASHBLS) $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) $(BACKTRACE_LIB) $(LIBSECP256K1) $(EVENT_LIBS) $(EVENT_PTHREADS_LIBS)
+  $(LIBDASHBLS) $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) $(BACKTRACE_LIB) $(LIBSECP256K1) $(EVENT_LIBS) $(EVENT_PTHREADS_LIBS) $(MINISKETCH_LIBS)
 test_test_dash_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS)

 test_test_dash_LDADD += $(BDB_LIBS) $(MINIUPNPC_LIBS) $(SQLITE_LIBS) $(NATPMP_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(GMP_LIBS)
--- a/src/minisketchwrapper.cpp
+++ b/src/minisketchwrapper.cpp
@ -0,0 +1,77 @@
+// Copyright (c) 2021 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <minisketchwrapper.h>
+
+#include <logging.h>
+#include <util/time.h>
+
+#include <minisketch.h>
+
+#include <algorithm>
+#include <cstddef>
+#include <cstdint>
+#include <optional>
+#include <utility>
+#include <vector>
+
+namespace {
+
+static constexpr uint32_t BITS = 32;
+
+uint32_t FindBestImplementation()
+{
+    std::optional<std::pair<int64_t, uint32_t>> best;
+
+    uint32_t max_impl = Minisketch::MaxImplementation();
+    for (uint32_t impl = 0; impl <= max_impl; ++impl) {
+        std::vector<int64_t> benches;
+        uint64_t offset = 0;
+        /* Run a little benchmark with capacity 32, adding 184 entries, and decoding 11 of them once. */
+        for (int b = 0; b < 11; ++b) {
+            if (!Minisketch::ImplementationSupported(BITS, impl)) break;
+            Minisketch sketch(BITS, impl, 32);
+            auto start = GetTimeMicros();
+            for (uint64_t e = 0; e < 100; ++e) {
+                sketch.Add(e*1337 + b*13337 + offset);
+            }
+            for (uint64_t e = 0; e < 84; ++e) {
+                sketch.Add(e*1337 + b*13337 + offset);
+            }
+            offset += (*sketch.Decode(32))[0];
+            auto stop = GetTimeMicros();
+            benches.push_back(stop - start);
+        }
+        /* Remember which implementation has the best median benchmark time. */
+        if (!benches.empty()) {
+            std::sort(benches.begin(), benches.end());
+            if (!best || best->first > benches[5]) {
+                best = std::make_pair(benches[5], impl);
+            }
+        }
+    }
+    assert(best.has_value());
+    LogPrintf("Using Minisketch implementation number %i\n", best->second);
+    return best->second;
+}
+
+uint32_t Minisketch32Implementation()
+{
+    // Fast compute-once idiom.
+    static uint32_t best = FindBestImplementation();
+    return best;
+}
+
+} // namespace
+
+
+Minisketch MakeMinisketch32(size_t capacity)
+{
+    return Minisketch(BITS, Minisketch32Implementation(), capacity);
+}
+
+Minisketch MakeMinisketch32FP(size_t max_elements, uint32_t fpbits)
+{
+    return Minisketch::CreateFP(BITS, Minisketch32Implementation(), max_elements, fpbits);
+}
--- a/src/minisketchwrapper.h
+++ b/src/minisketchwrapper.h
@ -0,0 +1,17 @@
+// Copyright (c) 2021 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#ifndef BITCOIN_MINISKETCHWRAPPER_H
+#define BITCOIN_MINISKETCHWRAPPER_H
+
+#include <minisketch.h>
+#include <cstddef>
+#include <cstdint>
+
+/** Wrapper around Minisketch::Minisketch(32, implementation, capacity). */
+Minisketch MakeMinisketch32(size_t capacity);
+/** Wrapper around Minisketch::CreateFP. */
+Minisketch MakeMinisketch32FP(size_t max_elements, uint32_t fpbits);
+
+#endif // BITCOIN_DBWRAPPER_H
--- a/src/test/minisketch_tests.cpp
+++ b/src/test/minisketch_tests.cpp
@ -0,0 +1,49 @@
+// Copyright (c) 2021 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <minisketch.h>
+#include <minisketchwrapper.h>
+#include <random.h>
+#include <test/util/setup_common.h>
+
+#include <boost/test/unit_test.hpp>
+
+#include <utility>
+
+BOOST_AUTO_TEST_SUITE(minisketch_tests)
+
+BOOST_AUTO_TEST_CASE(minisketch_test)
+{
+    for (int i = 0; i < 100; ++i) {
+        uint32_t errors = 0 + InsecureRandRange(11);
+        uint32_t start_a = 1 + InsecureRandRange(1000000000);
+        uint32_t a_not_b = InsecureRandRange(errors + 1);
+        uint32_t b_not_a = errors - a_not_b;
+        uint32_t both = InsecureRandRange(10000);
+        uint32_t end_a = start_a + a_not_b + both;
+        uint32_t start_b = start_a + a_not_b;
+        uint32_t end_b = start_b + both + b_not_a;
+
+        Minisketch sketch_a = MakeMinisketch32(10);
+        for (uint32_t a = start_a; a < end_a; ++a) sketch_a.Add(a);
+        Minisketch sketch_b = MakeMinisketch32(10);
+        for (uint32_t b = start_b; b < end_b; ++b) sketch_b.Add(b);
+
+        Minisketch sketch_ar = MakeMinisketch32(10);
+        Minisketch sketch_br = MakeMinisketch32(10);
+        sketch_ar.Deserialize(sketch_a.Serialize());
+        sketch_br.Deserialize(sketch_b.Serialize());
+
+        Minisketch sketch_c = std::move(sketch_ar);
+        sketch_c.Merge(sketch_br);
+        auto dec = sketch_c.Decode(errors);
+        BOOST_CHECK(dec.has_value());
+        auto sols = std::move(*dec);
+        std::sort(sols.begin(), sols.end());
+        for (uint32_t i = 0; i < a_not_b; ++i) BOOST_CHECK_EQUAL(sols[i], start_a + i);
+        for (uint32_t i = 0; i < b_not_a; ++i) BOOST_CHECK_EQUAL(sols[i + a_not_b], start_b + both + i);
+    }
+}
+
+BOOST_AUTO_TEST_SUITE_END()
--- a/test/lint/README.md
+++ b/test/lint/README.md
@ -31,6 +31,7 @@ maintained:
 * for `src/univalue`: https://github.com/bitcoin-core/univalue.git (branch master)
 * for `src/crypto/ctaes`: https://github.com/bitcoin-core/ctaes.git (branch master)
 * for `src/crc32c`: https://github.com/google/crc32c.git (branch master)
+* for `src/minisketch`: https://github.com/sipa/minisketch.git (branch master)

 To do so, add the upstream repository as remote:

--- a/test/lint/lint-files.py
+++ b/test/lint/lint-files.py
@ -21,7 +21,7 @@ ALL_SOURCE_FILENAMES_REGEXP = r"^.*\.(cpp|h|py|sh)$"
 ALLOWED_FILENAME_REGEXP = "^[a-zA-Z0-9/_.@][a-zA-Z0-9/_.@-]*$"
 ALLOWED_SOURCE_FILENAME_REGEXP = "^[a-z0-9_./-]+$"
 ALLOWED_SOURCE_FILENAME_EXCEPTION_REGEXP = (
-    "^src/(dashbls/|immer/|secp256k1/|univalue/|test/fuzz/FuzzedDataProvider.h)"
+    "^src/(dashbls/|immer/|secp256k1/|minisketch/|univalue/|test/fuzz/FuzzedDataProvider.h)"
 )
 ALLOWED_PERMISSION_NON_EXECUTABLES = 0o644
 ALLOWED_PERMISSION_EXECUTABLES = 0o755
--- a/test/lint/lint-format-strings.sh
+++ b/test/lint/lint-format-strings.sh
@ -34,7 +34,7 @@ if ! python3 -m doctest "test/lint/run-lint-format-strings.py"; then
 fi
 for S in "${FUNCTION_NAMES_AND_NUMBER_OF_LEADING_ARGUMENTS[@]}"; do
    IFS="," read -r FUNCTION_NAME SKIP_ARGUMENTS <<< "${S}"
-    for MATCHING_FILE in $(git grep --full-name -l "${FUNCTION_NAME}" -- "*.c" "*.cpp" "*.h" | sort | grep -vE "^src/(dashbls|leveldb|secp256k1|tinyformat|univalue|test/fuzz/strprintf.cpp)"); do
+    for MATCHING_FILE in $(git grep --full-name -l "${FUNCTION_NAME}" -- "*.c" "*.cpp" "*.h" | sort | grep -vE "^src/(dashbls|leveldb|secp256k1|minisketch|tinyformat|univalue|test/fuzz/strprintf.cpp)"); do
        MATCHING_FILES+=("${MATCHING_FILE}")
    done
    if ! "test/lint/run-lint-format-strings.py" --skip-arguments "${SKIP_ARGUMENTS}" "${FUNCTION_NAME}" "${MATCHING_FILES[@]}"; then
--- a/test/lint/lint-include-guards.sh
+++ b/test/lint/lint-include-guards.sh
@ -10,7 +10,7 @@ export LC_ALL=C
 HEADER_ID_PREFIX="BITCOIN_"
 HEADER_ID_SUFFIX="_H"

-REGEXP_EXCLUDE_FILES_WITH_PREFIX="src/(crypto/ctaes/|dashbls/|immer/|leveldb/|crc32c/|secp256k1/|test/fuzz/FuzzedDataProvider.h|tinyformat.h|bench/nanobench.h|univalue/|ctpl_stl.h|bls/|crypto/x11/sph|gsl|util/expected.h)"
+REGEXP_EXCLUDE_FILES_WITH_PREFIX="src/(crypto/ctaes/|dashbls/|immer/|leveldb/|crc32c/|secp256k1/|minisketch/|test/fuzz/FuzzedDataProvider.h|tinyformat.h|bench/nanobench.h|univalue/|ctpl_stl.h|bls/|crypto/x11/sph|gsl|util/expected.h)"

 EXIT_CODE=0
 for HEADER_FILE in $(git ls-files -- "*.h" | grep -vE "^${REGEXP_EXCLUDE_FILES_WITH_PREFIX}")
--- a/test/lint/lint-includes.sh
+++ b/test/lint/lint-includes.sh
@ -9,7 +9,7 @@
 # Check includes: Check for duplicate includes. Enforce bracket syntax includes.

 export LC_ALL=C
-IGNORE_REGEXP="/(dashbls|immer|leveldb|secp256k1|univalue|crc32c|crypto/x11)/"
+IGNORE_REGEXP="/(dashbls|immer|leveldb|secp256k1|minisketch|univalue|crc32c|crypto/x11)/"

 # cd to root folder of git repo for git ls-files to work properly
 cd "$(dirname $0)/../.." || exit 1
--- a/test/lint/lint-locale-dependence.sh
+++ b/test/lint/lint-locale-dependence.sh
@ -51,7 +51,7 @@ KNOWN_VIOLATIONS=(
    "src/util/system.cpp:.*fprintf"
 )

-REGEXP_IGNORE_EXTERNAL_DEPENDENCIES="^src/(dashbls/|immer/|crypto/ctaes/|leveldb/|secp256k1/|tinyformat.h|univalue/)"
+REGEXP_IGNORE_EXTERNAL_DEPENDENCIES="^src/(dashbls/|immer/|crypto/ctaes/|leveldb/|secp256k1/|minisketch/|tinyformat.h|univalue/)"

 LOCALE_DEPENDENT_FUNCTIONS=(
    alphasort    # LC_COLLATE (via strcoll)
--- a/test/lint/lint-shell-locale.sh
+++ b/test/lint/lint-shell-locale.sh
@ -12,7 +12,7 @@
 export LC_ALL=C

 EXIT_CODE=0
-for SHELL_SCRIPT in $(git ls-files -- "*.sh" | grep -vE "src/(dashbls|secp256k1|univalue)/"); do
+for SHELL_SCRIPT in $(git ls-files -- "*.sh" | grep -vE "src/(dashbls|secp256k1|minisketch|univalue)/"); do
    if grep -q "# This script is intentionally locale dependent by not setting \"export LC_ALL=C\"" "${SHELL_SCRIPT}"; then
        continue
    fi
--- a/test/lint/lint-shell.sh
+++ b/test/lint/lint-shell.sh
@ -30,7 +30,7 @@ fi
 SHELLCHECK_CMD=(shellcheck --external-sources --check-sourced)
 EXCLUDE="--exclude=$(IFS=','; echo "${disabled[*]}")"
 SOURCED_FILES=$(git ls-files | xargs gawk '/^# shellcheck shell=/ {print FILENAME} {nextfile}')  # Check shellcheck directive used for sourced files
-if ! "${SHELLCHECK_CMD[@]}" "$EXCLUDE" $SOURCED_FILES $(git ls-files -- '*.sh' | grep -vE 'src/(dashbls|immer|leveldb|secp256k1|univalue)/'); then
+if ! "${SHELLCHECK_CMD[@]}" "$EXCLUDE" $SOURCED_FILES $(git ls-files -- '*.sh' | grep -vE 'src/(dashbls|immer|leveldb|secp256k1|minisketch|univalue)/'); then
    EXIT_CODE=1
 fi

--- a/test/lint/lint-spelling.sh
+++ b/test/lint/lint-spelling.sh
@ -15,6 +15,6 @@ if ! command -v codespell > /dev/null; then
 fi

 IGNORE_WORDS_FILE="test/lint/spelling.ignore-words.txt"
-if ! codespell --check-filenames --disable-colors --quiet-level=7 --ignore-words=${IGNORE_WORDS_FILE} $(git ls-files -- ":(exclude)build-aux/m4/" ":(exclude)contrib/seeds/*.txt" ":(exclude)depends/" ":(exclude)doc/release-notes/" ":(exclude)src/bip39_english.h" ":(exclude)src/dashbls/" ":(exclude)src/crc32c/" ":(exclude)src/crypto/" ":(exclude)src/ctpl_stl.h" ":(exclude)src/cxxtimer.hpp" ":(exclude)src/util/expected.h" ":(exclude)src/immer/" ":(exclude)src/leveldb/" ":(exclude)src/qt/locale/" ":(exclude)src/qt/*.qrc" ":(exclude)src/secp256k1/" ":(exclude)src/univalue/" ":(exclude)contrib/builder-keys/" ":(exclude)contrib/guix/patches"); then
+if ! codespell --check-filenames --disable-colors --quiet-level=7 --ignore-words=${IGNORE_WORDS_FILE} $(git ls-files -- ":(exclude)build-aux/m4/" ":(exclude)contrib/seeds/*.txt" ":(exclude)depends/" ":(exclude)doc/release-notes/" ":(exclude)src/bip39_english.h" ":(exclude)src/dashbls/" ":(exclude)src/crc32c/" ":(exclude)src/crypto/" ":(exclude)src/ctpl_stl.h" ":(exclude)src/cxxtimer.hpp" ":(exclude)src/util/expected.h" ":(exclude)src/immer/" ":(exclude)src/leveldb/" ":(exclude)src/qt/locale/" ":(exclude)src/qt/*.qrc" ":(exclude)src/secp256k1/" ":(exclude)src/minisketch/" ":(exclude)src/univalue/" ":(exclude)contrib/builder-keys/" ":(exclude)contrib/guix/patches"); then
    echo "^ Warning: codespell identified likely spelling errors. Any false positives? Add them to the list of ignored words in ${IGNORE_WORDS_FILE}"
 fi
--- a/test/lint/lint-whitespace.sh
+++ b/test/lint/lint-whitespace.sh
@ -33,14 +33,14 @@ if [ -z "${COMMIT_RANGE}" ]; then
 fi

 showdiff() {
-  if ! git diff -U0 "${COMMIT_RANGE}" -- "." ":(exclude)depends/patches/" ":(exclude)contrib/guix/patches/" ":(exclude)src/dashbls/" ":(exclude)src/util/expected.h" ":(exclude)src/immer/" ":(exclude)src/leveldb/" ":(exclude)src/crc32c/" ":(exclude)src/secp256k1/" ":(exclude)src/univalue/" ":(exclude)doc/release-notes/"; then
+  if ! git diff -U0 "${COMMIT_RANGE}" -- "." ":(exclude)depends/patches/" ":(exclude)contrib/guix/patches/" ":(exclude)src/dashbls/" ":(exclude)src/util/expected.h" ":(exclude)src/immer/" ":(exclude)src/leveldb/" ":(exclude)src/crc32c/" ":(exclude)src/secp256k1/" ":(exclude)src/minisketch/" ":(exclude)src/univalue/" ":(exclude)doc/release-notes/"; then
    echo "Failed to get a diff"
    exit 1
  fi
 }

 showcodediff() {
-  if ! git diff -U0 "${COMMIT_RANGE}" -- *.cpp *.h *.md *.py *.sh ":(exclude)src/dashbls/" ":(exclude)src/util/expected.h" ":(exclude)src/immer/" ":(exclude)src/leveldb/" ":(exclude)src/crc32c/" ":(exclude)src/secp256k1/" ":(exclude)src/univalue/" ":(exclude)doc/release-notes/"; then
+  if ! git diff -U0 "${COMMIT_RANGE}" -- *.cpp *.h *.md *.py *.sh ":(exclude)src/dashbls/" ":(exclude)src/util/expected.h" ":(exclude)src/immer/" ":(exclude)src/leveldb/" ":(exclude)src/crc32c/" ":(exclude)src/secp256k1/" ":(exclude)src/minisketch/" ":(exclude)src/univalue/" ":(exclude)doc/release-notes/"; then
    echo "Failed to get a diff"
    exit 1
  fi
--- a/test/sanitizer_suppressions/ubsan
+++ b/test/sanitizer_suppressions/ubsan
@ -32,6 +32,7 @@ unsigned-integer-overflow:crypto/
 unsigned-integer-overflow:FuzzedDataProvider.h
 unsigned-integer-overflow:hash.cpp
 unsigned-integer-overflow:leveldb/
+unsigned-integer-overflow:minisketch/
 unsigned-integer-overflow:policy/fees.cpp
 unsigned-integer-overflow:prevector.h
 unsigned-integer-overflow:pubkey.h
@ -58,6 +59,7 @@ implicit-integer-sign-change:crypto/
 # implicit-integer-sign-change in FuzzedDataProvider's ConsumeIntegralInRange
 implicit-integer-sign-change:FuzzedDataProvider.h
 implicit-integer-sign-change:key.cpp
+implicit-integer-sign-change:minisketch/
 implicit-integer-sign-change:policy/fees.cpp
 implicit-integer-sign-change:prevector.h
 implicit-integer-sign-change:script/bitcoinconsensus.cpp
@ -96,6 +98,7 @@ shift-base:arith_uint256.cpp
 shift-base:crypto/
 shift-base:hash.cpp
 shift-base:leveldb/
+shift-base:minisketch/
 shift-base:streams.h
 shift-base:test/fuzz/crypto_diff_fuzz_chacha20.cpp
 shift-base:util/bip32.cpp