mirror of
https://github.com/dashpay/dash.git
synced 2024-12-25 12:02:48 +01:00
merge bitcoin#23114: Add minisketch subtree and integrate into build/test
excludes: - b6487dc4ef47ec9ea894eceac25f37d0b806f8aa (subtree manipulation done in previous commits) - 07f0a61ef711a2f75ded3d73545bfabdf2a64fef (see above) - 0eb7928ab8d9dcb840e4965bfa81deb752b00dfa (we don't support MSVC build method)
This commit is contained in:
parent
57ef0db047
commit
a6b26b5dc1
@ -177,6 +177,7 @@ LCOV_FILTER_PATTERN = \
|
|||||||
-p "src/bench/" \
|
-p "src/bench/" \
|
||||||
-p "src/univalue" \
|
-p "src/univalue" \
|
||||||
-p "src/crypto/ctaes" \
|
-p "src/crypto/ctaes" \
|
||||||
|
-p "src/minisketch" \
|
||||||
-p "src/secp256k1" \
|
-p "src/secp256k1" \
|
||||||
-p "depends"
|
-p "depends"
|
||||||
|
|
||||||
|
@ -17,6 +17,7 @@ export COMMIT_RANGE
|
|||||||
# check with -r to not have to fetch all the remotes.
|
# check with -r to not have to fetch all the remotes.
|
||||||
test/lint/git-subtree-check.sh src/crypto/ctaes
|
test/lint/git-subtree-check.sh src/crypto/ctaes
|
||||||
test/lint/git-subtree-check.sh src/secp256k1
|
test/lint/git-subtree-check.sh src/secp256k1
|
||||||
|
test/lint/git-subtree-check.sh src/minisketch
|
||||||
test/lint/git-subtree-check.sh src/univalue
|
test/lint/git-subtree-check.sh src/univalue
|
||||||
test/lint/git-subtree-check.sh src/leveldb
|
test/lint/git-subtree-check.sh src/leveldb
|
||||||
test/lint/check-doc.py
|
test/lint/check-doc.py
|
||||||
|
@ -6,7 +6,7 @@
|
|||||||
|
|
||||||
export LC_ALL=C.UTF-8
|
export LC_ALL=C.UTF-8
|
||||||
|
|
||||||
for b_name in {"${BASE_OUTDIR}/bin"/*,src/secp256k1/*tests,src/univalue/{no_nul,test_json,unitester,object}}.exe; do
|
for b_name in {"${BASE_OUTDIR}/bin"/*,src/secp256k1/*tests,src/minisketch/test{,-verify},src/univalue/{no_nul,test_json,unitester,object}}.exe; do
|
||||||
# shellcheck disable=SC2044
|
# shellcheck disable=SC2044
|
||||||
for b in $(find "${BASE_ROOT_DIR}" -executable -type f -name "$(basename $b_name)"); do
|
for b in $(find "${BASE_ROOT_DIR}" -executable -type f -name "$(basename $b_name)"); do
|
||||||
if (file "$b" | grep "Windows"); then
|
if (file "$b" | grep "Windows"); then
|
||||||
|
32
configure.ac
32
configure.ac
@ -537,6 +537,24 @@ AX_CHECK_COMPILE_FLAG([-msse4.1],[[SSE41_CXXFLAGS="-msse4.1"]],,[[$CXXFLAG_WERRO
|
|||||||
AX_CHECK_COMPILE_FLAG([-mavx -mavx2],[[AVX2_CXXFLAGS="-mavx -mavx2"]],,[[$CXXFLAG_WERROR]])
|
AX_CHECK_COMPILE_FLAG([-mavx -mavx2],[[AVX2_CXXFLAGS="-mavx -mavx2"]],,[[$CXXFLAG_WERROR]])
|
||||||
AX_CHECK_COMPILE_FLAG([-msse4 -msha],[[X86_SHANI_CXXFLAGS="-msse4 -msha"]],,[[$CXXFLAG_WERROR]])
|
AX_CHECK_COMPILE_FLAG([-msse4 -msha],[[X86_SHANI_CXXFLAGS="-msse4 -msha"]],,[[$CXXFLAG_WERROR]])
|
||||||
|
|
||||||
|
enable_clmul=
|
||||||
|
AX_CHECK_COMPILE_FLAG([-mpclmul], [enable_clmul=yes], [], [$CXXFLAG_WERROR], [AC_LANG_PROGRAM([
|
||||||
|
#include <stdint.h>
|
||||||
|
#include <x86intrin.h>
|
||||||
|
], [
|
||||||
|
__m128i a = _mm_cvtsi64_si128((uint64_t)7);
|
||||||
|
__m128i b = _mm_clmulepi64_si128(a, a, 37);
|
||||||
|
__m128i c = _mm_srli_epi64(b, 41);
|
||||||
|
__m128i d = _mm_xor_si128(b, c);
|
||||||
|
uint64_t e = _mm_cvtsi128_si64(d);
|
||||||
|
return e == 0;
|
||||||
|
])])
|
||||||
|
|
||||||
|
if test x$enable_clmul = xyes; then
|
||||||
|
CLMUL_CXXFLAGS="-mpclmul"
|
||||||
|
AC_DEFINE(HAVE_CLMUL, 1, [Define this symbol if clmul instructions can be used])
|
||||||
|
fi
|
||||||
|
|
||||||
TEMP_CXXFLAGS="$CXXFLAGS"
|
TEMP_CXXFLAGS="$CXXFLAGS"
|
||||||
CXXFLAGS="$CXXFLAGS $SSE42_CXXFLAGS"
|
CXXFLAGS="$CXXFLAGS $SSE42_CXXFLAGS"
|
||||||
AC_MSG_CHECKING(for SSE4.2 intrinsics)
|
AC_MSG_CHECKING(for SSE4.2 intrinsics)
|
||||||
@ -1030,19 +1048,20 @@ AC_CHECK_DECLS([bswap_16, bswap_32, bswap_64],,,
|
|||||||
#endif])
|
#endif])
|
||||||
|
|
||||||
AC_MSG_CHECKING(for __builtin_clzl)
|
AC_MSG_CHECKING(for __builtin_clzl)
|
||||||
|
|
||||||
AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ ]], [[
|
AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ ]], [[
|
||||||
(void) __builtin_clzl(0);
|
(void) __builtin_clzl(0);
|
||||||
]])],
|
]])],
|
||||||
[ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_BUILTIN_CLZL, 1, [Define this symbol if you have __builtin_clzl])],
|
[ AC_MSG_RESULT(yes); have_clzl=yes; AC_DEFINE(HAVE_BUILTIN_CLZL, 1, [Define this symbol if you have __builtin_clzl])],
|
||||||
[ AC_MSG_RESULT(no)]
|
[ AC_MSG_RESULT(no); have_clzl=no;]
|
||||||
)
|
)
|
||||||
|
|
||||||
AC_MSG_CHECKING(for __builtin_clzll)
|
AC_MSG_CHECKING(for __builtin_clzll)
|
||||||
AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ ]], [[
|
AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ ]], [[
|
||||||
(void) __builtin_clzll(0);
|
(void) __builtin_clzll(0);
|
||||||
]])],
|
]])],
|
||||||
[ AC_MSG_RESULT(yes); AC_DEFINE(HAVE_BUILTIN_CLZLL, 1, [Define this symbol if you have __builtin_clzll])],
|
[ AC_MSG_RESULT(yes); have_clzll=yes; AC_DEFINE(HAVE_BUILTIN_CLZLL, 1, [Define this symbol if you have __builtin_clzll])],
|
||||||
[ AC_MSG_RESULT(no)]
|
[ AC_MSG_RESULT(no); have_clzll=no;]
|
||||||
)
|
)
|
||||||
|
|
||||||
dnl Check for mallopt(M_ARENA_MAX) (to set glibc arenas)
|
dnl Check for mallopt(M_ARENA_MAX) (to set glibc arenas)
|
||||||
@ -1815,6 +1834,10 @@ AM_CONDITIONAL([WORDS_BIGENDIAN],[test x$ac_cv_c_bigendian = xyes])
|
|||||||
AM_CONDITIONAL([USE_NATPMP],[test x$use_natpmp = xyes])
|
AM_CONDITIONAL([USE_NATPMP],[test x$use_natpmp = xyes])
|
||||||
AM_CONDITIONAL([USE_UPNP],[test x$use_upnp = xyes])
|
AM_CONDITIONAL([USE_UPNP],[test x$use_upnp = xyes])
|
||||||
|
|
||||||
|
dnl for minisketch
|
||||||
|
AM_CONDITIONAL([ENABLE_CLMUL],[test x$enable_clmul = xyes])
|
||||||
|
AM_CONDITIONAL([HAVE_CLZ],[test x$have_clzl$have_clzll = xyesyes])
|
||||||
|
|
||||||
AC_DEFINE(CLIENT_VERSION_MAJOR, _CLIENT_VERSION_MAJOR, [Major version])
|
AC_DEFINE(CLIENT_VERSION_MAJOR, _CLIENT_VERSION_MAJOR, [Major version])
|
||||||
AC_DEFINE(CLIENT_VERSION_MINOR, _CLIENT_VERSION_MINOR, [Minor version])
|
AC_DEFINE(CLIENT_VERSION_MINOR, _CLIENT_VERSION_MINOR, [Minor version])
|
||||||
AC_DEFINE(CLIENT_VERSION_BUILD, _CLIENT_VERSION_BUILD, [Version Build])
|
AC_DEFINE(CLIENT_VERSION_BUILD, _CLIENT_VERSION_BUILD, [Version Build])
|
||||||
@ -1857,6 +1880,7 @@ AC_SUBST(SANITIZER_CXXFLAGS)
|
|||||||
AC_SUBST(SANITIZER_LDFLAGS)
|
AC_SUBST(SANITIZER_LDFLAGS)
|
||||||
AC_SUBST(SSE42_CXXFLAGS)
|
AC_SUBST(SSE42_CXXFLAGS)
|
||||||
AC_SUBST(SSE41_CXXFLAGS)
|
AC_SUBST(SSE41_CXXFLAGS)
|
||||||
|
AC_SUBST(CLMUL_CXXFLAGS)
|
||||||
AC_SUBST(AVX2_CXXFLAGS)
|
AC_SUBST(AVX2_CXXFLAGS)
|
||||||
AC_SUBST(X86_SHANI_CXXFLAGS)
|
AC_SUBST(X86_SHANI_CXXFLAGS)
|
||||||
AC_SUBST(ARM_CRC_CXXFLAGS)
|
AC_SUBST(ARM_CRC_CXXFLAGS)
|
||||||
|
@ -44,6 +44,7 @@ EXCLUDE_DIRS = [
|
|||||||
"src/gsl/",
|
"src/gsl/",
|
||||||
"src/immer/",
|
"src/immer/",
|
||||||
"src/leveldb/",
|
"src/leveldb/",
|
||||||
|
"src/minisketch",
|
||||||
"src/secp256k1/",
|
"src/secp256k1/",
|
||||||
"src/univalue/",
|
"src/univalue/",
|
||||||
]
|
]
|
||||||
|
@ -1063,6 +1063,9 @@ Current subtrees include:
|
|||||||
- src/univalue
|
- src/univalue
|
||||||
- Upstream at https://github.com/bitcoin-core/univalue ; actively maintained by Core contributors, deviates from upstream https://github.com/jgarzik/univalue
|
- Upstream at https://github.com/bitcoin-core/univalue ; actively maintained by Core contributors, deviates from upstream https://github.com/jgarzik/univalue
|
||||||
|
|
||||||
|
- src/minisketch
|
||||||
|
- Upstream at https://github.com/sipa/minisketch ; maintained by Core contributors.
|
||||||
|
|
||||||
Upgrading LevelDB
|
Upgrading LevelDB
|
||||||
---------------------
|
---------------------
|
||||||
|
|
||||||
|
@ -36,7 +36,7 @@ BACKTRACE_LIB = -lbacktrace
|
|||||||
endif
|
endif
|
||||||
endif #ENABLE_STACKTRACES
|
endif #ENABLE_STACKTRACES
|
||||||
|
|
||||||
BITCOIN_INCLUDES=-I$(builddir) -I$(srcdir)/secp256k1/include -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) $(BDB_CPPFLAGS) $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS)
|
BITCOIN_INCLUDES=-I$(builddir) -I$(srcdir)/$(MINISKETCH_INCLUDE_DIR_INT) -I$(srcdir)/secp256k1/include -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) $(BDB_CPPFLAGS) $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS)
|
||||||
BITCOIN_INCLUDES+=-isystem$(srcdir)/dashbls/include -isystem$(srcdir)/dashbls/depends/relic/include -isystem$(srcdir)/dashbls/depends/minialloc/include
|
BITCOIN_INCLUDES+=-isystem$(srcdir)/dashbls/include -isystem$(srcdir)/dashbls/depends/relic/include -isystem$(srcdir)/dashbls/depends/minialloc/include
|
||||||
BITCOIN_INCLUDES+=-isystem$(srcdir)/immer
|
BITCOIN_INCLUDES+=-isystem$(srcdir)/immer
|
||||||
|
|
||||||
@ -259,6 +259,7 @@ BITCOIN_CORE_H = \
|
|||||||
merkleblock.h \
|
merkleblock.h \
|
||||||
messagesigner.h \
|
messagesigner.h \
|
||||||
miner.h \
|
miner.h \
|
||||||
|
minisketchwrapper.h \
|
||||||
net.h \
|
net.h \
|
||||||
net_permissions.h \
|
net_permissions.h \
|
||||||
net_processing.h \
|
net_processing.h \
|
||||||
@ -494,6 +495,7 @@ libbitcoin_server_a_SOURCES = \
|
|||||||
masternode/sync.cpp \
|
masternode/sync.cpp \
|
||||||
masternode/utils.cpp \
|
masternode/utils.cpp \
|
||||||
miner.cpp \
|
miner.cpp \
|
||||||
|
minisketchwrapper.cpp \
|
||||||
net.cpp \
|
net.cpp \
|
||||||
netfulfilledman.cpp \
|
netfulfilledman.cpp \
|
||||||
netgroup.cpp \
|
netgroup.cpp \
|
||||||
@ -1059,8 +1061,11 @@ nodist_libbitcoin_ipc_a_SOURCES = $(libbitcoin_ipc_mpgen_output)
|
|||||||
CLEANFILES += $(libbitcoin_ipc_mpgen_output)
|
CLEANFILES += $(libbitcoin_ipc_mpgen_output)
|
||||||
endif
|
endif
|
||||||
|
|
||||||
|
include Makefile.minisketch.include
|
||||||
|
|
||||||
include Makefile.crc32c.include
|
include Makefile.crc32c.include
|
||||||
include Makefile.leveldb.include
|
include Makefile.leveldb.include
|
||||||
|
|
||||||
include Makefile.test_util.include
|
include Makefile.test_util.include
|
||||||
include Makefile.test_fuzz.include
|
include Makefile.test_fuzz.include
|
||||||
|
|
||||||
|
43
src/Makefile.minisketch.include
Normal file
43
src/Makefile.minisketch.include
Normal file
@ -0,0 +1,43 @@
|
|||||||
|
include minisketch/sources.mk
|
||||||
|
|
||||||
|
LIBMINISKETCH_CPPFLAGS=
|
||||||
|
LIBMINISKETCH_CPPFLAGS += -DDISABLE_DEFAULT_FIELDS -DENABLE_FIELD_32
|
||||||
|
|
||||||
|
LIBMINISKETCH = minisketch/libminisketch.a
|
||||||
|
MINISKETCH_LIBS = $(LIBMINISKETCH)
|
||||||
|
|
||||||
|
if ENABLE_CLMUL
|
||||||
|
LIBMINISKETCH_CLMUL = minisketch/libminisketch_clmul.a
|
||||||
|
LIBMINISKETCH_CPPFLAGS += -DHAVE_CLMUL
|
||||||
|
MINISKETCH_LIBS += $(LIBMINISKETCH_CLMUL)
|
||||||
|
endif
|
||||||
|
|
||||||
|
if HAVE_CLZ
|
||||||
|
LIBMINISKETCH_CPPFLAGS += -DHAVE_CLZ
|
||||||
|
endif
|
||||||
|
|
||||||
|
EXTRA_LIBRARIES += $(MINISKETCH_LIBS)
|
||||||
|
|
||||||
|
minisketch_libminisketch_clmul_a_SOURCES = $(MINISKETCH_FIELD_CLMUL_SOURCES_INT) $(MINISKETCH_FIELD_CLMUL_HEADERS_INT)
|
||||||
|
minisketch_libminisketch_clmul_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) $(CLMUL_CXXFLAGS)
|
||||||
|
minisketch_libminisketch_clmul_a_CPPFLAGS = $(AM_CPPFLAGS) $(LIBMINISKETCH_CPPFLAGS)
|
||||||
|
|
||||||
|
minisketch_libminisketch_a_SOURCES = $(MINISKETCH_FIELD_GENERIC_SOURCES_INT) $(MINISKETCH_LIB_SOURCES_INT)
|
||||||
|
minisketch_libminisketch_a_SOURCES += $(MINISKETCH_FIELD_GENERIC_HEADERS_INT) $(MINISKETCH_LIB_HEADERS_INT) $(MINISKETCH_DIST_HEADERS_INT)
|
||||||
|
minisketch_libminisketch_a_CPPFLAGS = $(AM_CPPFLAGS) $(LIBMINISKETCH_CPPFLAGS)
|
||||||
|
minisketch_libminisketch_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS)
|
||||||
|
|
||||||
|
if ENABLE_TESTS
|
||||||
|
if !ENABLE_FUZZ
|
||||||
|
MINISKETCH_TEST = minisketch/test
|
||||||
|
TESTS += $(MINISKETCH_TEST)
|
||||||
|
noinst_PROGRAMS += $(MINISKETCH_TEST)
|
||||||
|
|
||||||
|
minisketch_test_SOURCES = $(MINISKETCH_TEST_SOURCES_INT)
|
||||||
|
minisketch_test_CPPFLAGS = $(AM_CPPFLAGS) $(LIBMINISKETCH_CPPFLAGS)
|
||||||
|
minisketch_test_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS)
|
||||||
|
minisketch_test_LDADD = $(MINISKETCH_LIBS)
|
||||||
|
minisketch_test_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) $(PTHREAD_FLAGS)
|
||||||
|
|
||||||
|
endif
|
||||||
|
endif
|
@ -130,6 +130,7 @@ BITCOIN_TESTS =\
|
|||||||
test/mempool_tests.cpp \
|
test/mempool_tests.cpp \
|
||||||
test/merkle_tests.cpp \
|
test/merkle_tests.cpp \
|
||||||
test/merkleblock_tests.cpp \
|
test/merkleblock_tests.cpp \
|
||||||
|
test/minisketch_tests.cpp \
|
||||||
test/miner_tests.cpp \
|
test/miner_tests.cpp \
|
||||||
test/multisig_tests.cpp \
|
test/multisig_tests.cpp \
|
||||||
test/net_peer_eviction_tests.cpp \
|
test/net_peer_eviction_tests.cpp \
|
||||||
@ -219,7 +220,7 @@ if ENABLE_WALLET
|
|||||||
test_test_dash_LDADD += $(LIBBITCOIN_WALLET)
|
test_test_dash_LDADD += $(LIBBITCOIN_WALLET)
|
||||||
endif
|
endif
|
||||||
test_test_dash_LDADD += $(LIBBITCOIN_SERVER) $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) \
|
test_test_dash_LDADD += $(LIBBITCOIN_SERVER) $(LIBBITCOIN_CLI) $(LIBBITCOIN_COMMON) $(LIBBITCOIN_UTIL) $(LIBBITCOIN_CONSENSUS) $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) \
|
||||||
$(LIBDASHBLS) $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) $(BACKTRACE_LIB) $(LIBSECP256K1) $(EVENT_LIBS) $(EVENT_PTHREADS_LIBS)
|
$(LIBDASHBLS) $(LIBLEVELDB) $(LIBLEVELDB_SSE42) $(LIBMEMENV) $(BACKTRACE_LIB) $(LIBSECP256K1) $(EVENT_LIBS) $(EVENT_PTHREADS_LIBS) $(MINISKETCH_LIBS)
|
||||||
test_test_dash_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS)
|
test_test_dash_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS)
|
||||||
|
|
||||||
test_test_dash_LDADD += $(BDB_LIBS) $(MINIUPNPC_LIBS) $(SQLITE_LIBS) $(NATPMP_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(GMP_LIBS)
|
test_test_dash_LDADD += $(BDB_LIBS) $(MINIUPNPC_LIBS) $(SQLITE_LIBS) $(NATPMP_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS) $(GMP_LIBS)
|
||||||
|
77
src/minisketchwrapper.cpp
Normal file
77
src/minisketchwrapper.cpp
Normal file
@ -0,0 +1,77 @@
|
|||||||
|
// Copyright (c) 2021 The Bitcoin Core developers
|
||||||
|
// Distributed under the MIT software license, see the accompanying
|
||||||
|
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
|
||||||
|
|
||||||
|
#include <minisketchwrapper.h>
|
||||||
|
|
||||||
|
#include <logging.h>
|
||||||
|
#include <util/time.h>
|
||||||
|
|
||||||
|
#include <minisketch.h>
|
||||||
|
|
||||||
|
#include <algorithm>
|
||||||
|
#include <cstddef>
|
||||||
|
#include <cstdint>
|
||||||
|
#include <optional>
|
||||||
|
#include <utility>
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
|
namespace {
|
||||||
|
|
||||||
|
static constexpr uint32_t BITS = 32;
|
||||||
|
|
||||||
|
uint32_t FindBestImplementation()
|
||||||
|
{
|
||||||
|
std::optional<std::pair<int64_t, uint32_t>> best;
|
||||||
|
|
||||||
|
uint32_t max_impl = Minisketch::MaxImplementation();
|
||||||
|
for (uint32_t impl = 0; impl <= max_impl; ++impl) {
|
||||||
|
std::vector<int64_t> benches;
|
||||||
|
uint64_t offset = 0;
|
||||||
|
/* Run a little benchmark with capacity 32, adding 184 entries, and decoding 11 of them once. */
|
||||||
|
for (int b = 0; b < 11; ++b) {
|
||||||
|
if (!Minisketch::ImplementationSupported(BITS, impl)) break;
|
||||||
|
Minisketch sketch(BITS, impl, 32);
|
||||||
|
auto start = GetTimeMicros();
|
||||||
|
for (uint64_t e = 0; e < 100; ++e) {
|
||||||
|
sketch.Add(e*1337 + b*13337 + offset);
|
||||||
|
}
|
||||||
|
for (uint64_t e = 0; e < 84; ++e) {
|
||||||
|
sketch.Add(e*1337 + b*13337 + offset);
|
||||||
|
}
|
||||||
|
offset += (*sketch.Decode(32))[0];
|
||||||
|
auto stop = GetTimeMicros();
|
||||||
|
benches.push_back(stop - start);
|
||||||
|
}
|
||||||
|
/* Remember which implementation has the best median benchmark time. */
|
||||||
|
if (!benches.empty()) {
|
||||||
|
std::sort(benches.begin(), benches.end());
|
||||||
|
if (!best || best->first > benches[5]) {
|
||||||
|
best = std::make_pair(benches[5], impl);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert(best.has_value());
|
||||||
|
LogPrintf("Using Minisketch implementation number %i\n", best->second);
|
||||||
|
return best->second;
|
||||||
|
}
|
||||||
|
|
||||||
|
uint32_t Minisketch32Implementation()
|
||||||
|
{
|
||||||
|
// Fast compute-once idiom.
|
||||||
|
static uint32_t best = FindBestImplementation();
|
||||||
|
return best;
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace
|
||||||
|
|
||||||
|
|
||||||
|
Minisketch MakeMinisketch32(size_t capacity)
|
||||||
|
{
|
||||||
|
return Minisketch(BITS, Minisketch32Implementation(), capacity);
|
||||||
|
}
|
||||||
|
|
||||||
|
Minisketch MakeMinisketch32FP(size_t max_elements, uint32_t fpbits)
|
||||||
|
{
|
||||||
|
return Minisketch::CreateFP(BITS, Minisketch32Implementation(), max_elements, fpbits);
|
||||||
|
}
|
17
src/minisketchwrapper.h
Normal file
17
src/minisketchwrapper.h
Normal file
@ -0,0 +1,17 @@
|
|||||||
|
// Copyright (c) 2021 The Bitcoin Core developers
|
||||||
|
// Distributed under the MIT software license, see the accompanying
|
||||||
|
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
|
||||||
|
|
||||||
|
#ifndef BITCOIN_MINISKETCHWRAPPER_H
|
||||||
|
#define BITCOIN_MINISKETCHWRAPPER_H
|
||||||
|
|
||||||
|
#include <minisketch.h>
|
||||||
|
#include <cstddef>
|
||||||
|
#include <cstdint>
|
||||||
|
|
||||||
|
/** Wrapper around Minisketch::Minisketch(32, implementation, capacity). */
|
||||||
|
Minisketch MakeMinisketch32(size_t capacity);
|
||||||
|
/** Wrapper around Minisketch::CreateFP. */
|
||||||
|
Minisketch MakeMinisketch32FP(size_t max_elements, uint32_t fpbits);
|
||||||
|
|
||||||
|
#endif // BITCOIN_DBWRAPPER_H
|
49
src/test/minisketch_tests.cpp
Normal file
49
src/test/minisketch_tests.cpp
Normal file
@ -0,0 +1,49 @@
|
|||||||
|
// Copyright (c) 2021 The Bitcoin Core developers
|
||||||
|
// Distributed under the MIT software license, see the accompanying
|
||||||
|
// file COPYING or http://www.opensource.org/licenses/mit-license.php.
|
||||||
|
|
||||||
|
#include <minisketch.h>
|
||||||
|
#include <minisketchwrapper.h>
|
||||||
|
#include <random.h>
|
||||||
|
#include <test/util/setup_common.h>
|
||||||
|
|
||||||
|
#include <boost/test/unit_test.hpp>
|
||||||
|
|
||||||
|
#include <utility>
|
||||||
|
|
||||||
|
BOOST_AUTO_TEST_SUITE(minisketch_tests)
|
||||||
|
|
||||||
|
BOOST_AUTO_TEST_CASE(minisketch_test)
|
||||||
|
{
|
||||||
|
for (int i = 0; i < 100; ++i) {
|
||||||
|
uint32_t errors = 0 + InsecureRandRange(11);
|
||||||
|
uint32_t start_a = 1 + InsecureRandRange(1000000000);
|
||||||
|
uint32_t a_not_b = InsecureRandRange(errors + 1);
|
||||||
|
uint32_t b_not_a = errors - a_not_b;
|
||||||
|
uint32_t both = InsecureRandRange(10000);
|
||||||
|
uint32_t end_a = start_a + a_not_b + both;
|
||||||
|
uint32_t start_b = start_a + a_not_b;
|
||||||
|
uint32_t end_b = start_b + both + b_not_a;
|
||||||
|
|
||||||
|
Minisketch sketch_a = MakeMinisketch32(10);
|
||||||
|
for (uint32_t a = start_a; a < end_a; ++a) sketch_a.Add(a);
|
||||||
|
Minisketch sketch_b = MakeMinisketch32(10);
|
||||||
|
for (uint32_t b = start_b; b < end_b; ++b) sketch_b.Add(b);
|
||||||
|
|
||||||
|
Minisketch sketch_ar = MakeMinisketch32(10);
|
||||||
|
Minisketch sketch_br = MakeMinisketch32(10);
|
||||||
|
sketch_ar.Deserialize(sketch_a.Serialize());
|
||||||
|
sketch_br.Deserialize(sketch_b.Serialize());
|
||||||
|
|
||||||
|
Minisketch sketch_c = std::move(sketch_ar);
|
||||||
|
sketch_c.Merge(sketch_br);
|
||||||
|
auto dec = sketch_c.Decode(errors);
|
||||||
|
BOOST_CHECK(dec.has_value());
|
||||||
|
auto sols = std::move(*dec);
|
||||||
|
std::sort(sols.begin(), sols.end());
|
||||||
|
for (uint32_t i = 0; i < a_not_b; ++i) BOOST_CHECK_EQUAL(sols[i], start_a + i);
|
||||||
|
for (uint32_t i = 0; i < b_not_a; ++i) BOOST_CHECK_EQUAL(sols[i + a_not_b], start_b + both + i);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
BOOST_AUTO_TEST_SUITE_END()
|
@ -31,6 +31,7 @@ maintained:
|
|||||||
* for `src/univalue`: https://github.com/bitcoin-core/univalue.git (branch master)
|
* for `src/univalue`: https://github.com/bitcoin-core/univalue.git (branch master)
|
||||||
* for `src/crypto/ctaes`: https://github.com/bitcoin-core/ctaes.git (branch master)
|
* for `src/crypto/ctaes`: https://github.com/bitcoin-core/ctaes.git (branch master)
|
||||||
* for `src/crc32c`: https://github.com/google/crc32c.git (branch master)
|
* for `src/crc32c`: https://github.com/google/crc32c.git (branch master)
|
||||||
|
* for `src/minisketch`: https://github.com/sipa/minisketch.git (branch master)
|
||||||
|
|
||||||
To do so, add the upstream repository as remote:
|
To do so, add the upstream repository as remote:
|
||||||
|
|
||||||
|
@ -21,7 +21,7 @@ ALL_SOURCE_FILENAMES_REGEXP = r"^.*\.(cpp|h|py|sh)$"
|
|||||||
ALLOWED_FILENAME_REGEXP = "^[a-zA-Z0-9/_.@][a-zA-Z0-9/_.@-]*$"
|
ALLOWED_FILENAME_REGEXP = "^[a-zA-Z0-9/_.@][a-zA-Z0-9/_.@-]*$"
|
||||||
ALLOWED_SOURCE_FILENAME_REGEXP = "^[a-z0-9_./-]+$"
|
ALLOWED_SOURCE_FILENAME_REGEXP = "^[a-z0-9_./-]+$"
|
||||||
ALLOWED_SOURCE_FILENAME_EXCEPTION_REGEXP = (
|
ALLOWED_SOURCE_FILENAME_EXCEPTION_REGEXP = (
|
||||||
"^src/(dashbls/|immer/|secp256k1/|univalue/|test/fuzz/FuzzedDataProvider.h)"
|
"^src/(dashbls/|immer/|secp256k1/|minisketch/|univalue/|test/fuzz/FuzzedDataProvider.h)"
|
||||||
)
|
)
|
||||||
ALLOWED_PERMISSION_NON_EXECUTABLES = 0o644
|
ALLOWED_PERMISSION_NON_EXECUTABLES = 0o644
|
||||||
ALLOWED_PERMISSION_EXECUTABLES = 0o755
|
ALLOWED_PERMISSION_EXECUTABLES = 0o755
|
||||||
|
@ -34,7 +34,7 @@ if ! python3 -m doctest "test/lint/run-lint-format-strings.py"; then
|
|||||||
fi
|
fi
|
||||||
for S in "${FUNCTION_NAMES_AND_NUMBER_OF_LEADING_ARGUMENTS[@]}"; do
|
for S in "${FUNCTION_NAMES_AND_NUMBER_OF_LEADING_ARGUMENTS[@]}"; do
|
||||||
IFS="," read -r FUNCTION_NAME SKIP_ARGUMENTS <<< "${S}"
|
IFS="," read -r FUNCTION_NAME SKIP_ARGUMENTS <<< "${S}"
|
||||||
for MATCHING_FILE in $(git grep --full-name -l "${FUNCTION_NAME}" -- "*.c" "*.cpp" "*.h" | sort | grep -vE "^src/(dashbls|leveldb|secp256k1|tinyformat|univalue|test/fuzz/strprintf.cpp)"); do
|
for MATCHING_FILE in $(git grep --full-name -l "${FUNCTION_NAME}" -- "*.c" "*.cpp" "*.h" | sort | grep -vE "^src/(dashbls|leveldb|secp256k1|minisketch|tinyformat|univalue|test/fuzz/strprintf.cpp)"); do
|
||||||
MATCHING_FILES+=("${MATCHING_FILE}")
|
MATCHING_FILES+=("${MATCHING_FILE}")
|
||||||
done
|
done
|
||||||
if ! "test/lint/run-lint-format-strings.py" --skip-arguments "${SKIP_ARGUMENTS}" "${FUNCTION_NAME}" "${MATCHING_FILES[@]}"; then
|
if ! "test/lint/run-lint-format-strings.py" --skip-arguments "${SKIP_ARGUMENTS}" "${FUNCTION_NAME}" "${MATCHING_FILES[@]}"; then
|
||||||
|
@ -10,7 +10,7 @@ export LC_ALL=C
|
|||||||
HEADER_ID_PREFIX="BITCOIN_"
|
HEADER_ID_PREFIX="BITCOIN_"
|
||||||
HEADER_ID_SUFFIX="_H"
|
HEADER_ID_SUFFIX="_H"
|
||||||
|
|
||||||
REGEXP_EXCLUDE_FILES_WITH_PREFIX="src/(crypto/ctaes/|dashbls/|immer/|leveldb/|crc32c/|secp256k1/|test/fuzz/FuzzedDataProvider.h|tinyformat.h|bench/nanobench.h|univalue/|ctpl_stl.h|bls/|crypto/x11/sph|gsl|util/expected.h)"
|
REGEXP_EXCLUDE_FILES_WITH_PREFIX="src/(crypto/ctaes/|dashbls/|immer/|leveldb/|crc32c/|secp256k1/|minisketch/|test/fuzz/FuzzedDataProvider.h|tinyformat.h|bench/nanobench.h|univalue/|ctpl_stl.h|bls/|crypto/x11/sph|gsl|util/expected.h)"
|
||||||
|
|
||||||
EXIT_CODE=0
|
EXIT_CODE=0
|
||||||
for HEADER_FILE in $(git ls-files -- "*.h" | grep -vE "^${REGEXP_EXCLUDE_FILES_WITH_PREFIX}")
|
for HEADER_FILE in $(git ls-files -- "*.h" | grep -vE "^${REGEXP_EXCLUDE_FILES_WITH_PREFIX}")
|
||||||
|
@ -9,7 +9,7 @@
|
|||||||
# Check includes: Check for duplicate includes. Enforce bracket syntax includes.
|
# Check includes: Check for duplicate includes. Enforce bracket syntax includes.
|
||||||
|
|
||||||
export LC_ALL=C
|
export LC_ALL=C
|
||||||
IGNORE_REGEXP="/(dashbls|immer|leveldb|secp256k1|univalue|crc32c|crypto/x11)/"
|
IGNORE_REGEXP="/(dashbls|immer|leveldb|secp256k1|minisketch|univalue|crc32c|crypto/x11)/"
|
||||||
|
|
||||||
# cd to root folder of git repo for git ls-files to work properly
|
# cd to root folder of git repo for git ls-files to work properly
|
||||||
cd "$(dirname $0)/../.." || exit 1
|
cd "$(dirname $0)/../.." || exit 1
|
||||||
|
@ -51,7 +51,7 @@ KNOWN_VIOLATIONS=(
|
|||||||
"src/util/system.cpp:.*fprintf"
|
"src/util/system.cpp:.*fprintf"
|
||||||
)
|
)
|
||||||
|
|
||||||
REGEXP_IGNORE_EXTERNAL_DEPENDENCIES="^src/(dashbls/|immer/|crypto/ctaes/|leveldb/|secp256k1/|tinyformat.h|univalue/)"
|
REGEXP_IGNORE_EXTERNAL_DEPENDENCIES="^src/(dashbls/|immer/|crypto/ctaes/|leveldb/|secp256k1/|minisketch/|tinyformat.h|univalue/)"
|
||||||
|
|
||||||
LOCALE_DEPENDENT_FUNCTIONS=(
|
LOCALE_DEPENDENT_FUNCTIONS=(
|
||||||
alphasort # LC_COLLATE (via strcoll)
|
alphasort # LC_COLLATE (via strcoll)
|
||||||
|
@ -12,7 +12,7 @@
|
|||||||
export LC_ALL=C
|
export LC_ALL=C
|
||||||
|
|
||||||
EXIT_CODE=0
|
EXIT_CODE=0
|
||||||
for SHELL_SCRIPT in $(git ls-files -- "*.sh" | grep -vE "src/(dashbls|secp256k1|univalue)/"); do
|
for SHELL_SCRIPT in $(git ls-files -- "*.sh" | grep -vE "src/(dashbls|secp256k1|minisketch|univalue)/"); do
|
||||||
if grep -q "# This script is intentionally locale dependent by not setting \"export LC_ALL=C\"" "${SHELL_SCRIPT}"; then
|
if grep -q "# This script is intentionally locale dependent by not setting \"export LC_ALL=C\"" "${SHELL_SCRIPT}"; then
|
||||||
continue
|
continue
|
||||||
fi
|
fi
|
||||||
|
@ -30,7 +30,7 @@ fi
|
|||||||
SHELLCHECK_CMD=(shellcheck --external-sources --check-sourced)
|
SHELLCHECK_CMD=(shellcheck --external-sources --check-sourced)
|
||||||
EXCLUDE="--exclude=$(IFS=','; echo "${disabled[*]}")"
|
EXCLUDE="--exclude=$(IFS=','; echo "${disabled[*]}")"
|
||||||
SOURCED_FILES=$(git ls-files | xargs gawk '/^# shellcheck shell=/ {print FILENAME} {nextfile}') # Check shellcheck directive used for sourced files
|
SOURCED_FILES=$(git ls-files | xargs gawk '/^# shellcheck shell=/ {print FILENAME} {nextfile}') # Check shellcheck directive used for sourced files
|
||||||
if ! "${SHELLCHECK_CMD[@]}" "$EXCLUDE" $SOURCED_FILES $(git ls-files -- '*.sh' | grep -vE 'src/(dashbls|immer|leveldb|secp256k1|univalue)/'); then
|
if ! "${SHELLCHECK_CMD[@]}" "$EXCLUDE" $SOURCED_FILES $(git ls-files -- '*.sh' | grep -vE 'src/(dashbls|immer|leveldb|secp256k1|minisketch|univalue)/'); then
|
||||||
EXIT_CODE=1
|
EXIT_CODE=1
|
||||||
fi
|
fi
|
||||||
|
|
||||||
|
@ -15,6 +15,6 @@ if ! command -v codespell > /dev/null; then
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
IGNORE_WORDS_FILE="test/lint/spelling.ignore-words.txt"
|
IGNORE_WORDS_FILE="test/lint/spelling.ignore-words.txt"
|
||||||
if ! codespell --check-filenames --disable-colors --quiet-level=7 --ignore-words=${IGNORE_WORDS_FILE} $(git ls-files -- ":(exclude)build-aux/m4/" ":(exclude)contrib/seeds/*.txt" ":(exclude)depends/" ":(exclude)doc/release-notes/" ":(exclude)src/bip39_english.h" ":(exclude)src/dashbls/" ":(exclude)src/crc32c/" ":(exclude)src/crypto/" ":(exclude)src/ctpl_stl.h" ":(exclude)src/cxxtimer.hpp" ":(exclude)src/util/expected.h" ":(exclude)src/immer/" ":(exclude)src/leveldb/" ":(exclude)src/qt/locale/" ":(exclude)src/qt/*.qrc" ":(exclude)src/secp256k1/" ":(exclude)src/univalue/" ":(exclude)contrib/builder-keys/" ":(exclude)contrib/guix/patches"); then
|
if ! codespell --check-filenames --disable-colors --quiet-level=7 --ignore-words=${IGNORE_WORDS_FILE} $(git ls-files -- ":(exclude)build-aux/m4/" ":(exclude)contrib/seeds/*.txt" ":(exclude)depends/" ":(exclude)doc/release-notes/" ":(exclude)src/bip39_english.h" ":(exclude)src/dashbls/" ":(exclude)src/crc32c/" ":(exclude)src/crypto/" ":(exclude)src/ctpl_stl.h" ":(exclude)src/cxxtimer.hpp" ":(exclude)src/util/expected.h" ":(exclude)src/immer/" ":(exclude)src/leveldb/" ":(exclude)src/qt/locale/" ":(exclude)src/qt/*.qrc" ":(exclude)src/secp256k1/" ":(exclude)src/minisketch/" ":(exclude)src/univalue/" ":(exclude)contrib/builder-keys/" ":(exclude)contrib/guix/patches"); then
|
||||||
echo "^ Warning: codespell identified likely spelling errors. Any false positives? Add them to the list of ignored words in ${IGNORE_WORDS_FILE}"
|
echo "^ Warning: codespell identified likely spelling errors. Any false positives? Add them to the list of ignored words in ${IGNORE_WORDS_FILE}"
|
||||||
fi
|
fi
|
||||||
|
@ -33,14 +33,14 @@ if [ -z "${COMMIT_RANGE}" ]; then
|
|||||||
fi
|
fi
|
||||||
|
|
||||||
showdiff() {
|
showdiff() {
|
||||||
if ! git diff -U0 "${COMMIT_RANGE}" -- "." ":(exclude)depends/patches/" ":(exclude)contrib/guix/patches/" ":(exclude)src/dashbls/" ":(exclude)src/util/expected.h" ":(exclude)src/immer/" ":(exclude)src/leveldb/" ":(exclude)src/crc32c/" ":(exclude)src/secp256k1/" ":(exclude)src/univalue/" ":(exclude)doc/release-notes/"; then
|
if ! git diff -U0 "${COMMIT_RANGE}" -- "." ":(exclude)depends/patches/" ":(exclude)contrib/guix/patches/" ":(exclude)src/dashbls/" ":(exclude)src/util/expected.h" ":(exclude)src/immer/" ":(exclude)src/leveldb/" ":(exclude)src/crc32c/" ":(exclude)src/secp256k1/" ":(exclude)src/minisketch/" ":(exclude)src/univalue/" ":(exclude)doc/release-notes/"; then
|
||||||
echo "Failed to get a diff"
|
echo "Failed to get a diff"
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
showcodediff() {
|
showcodediff() {
|
||||||
if ! git diff -U0 "${COMMIT_RANGE}" -- *.cpp *.h *.md *.py *.sh ":(exclude)src/dashbls/" ":(exclude)src/util/expected.h" ":(exclude)src/immer/" ":(exclude)src/leveldb/" ":(exclude)src/crc32c/" ":(exclude)src/secp256k1/" ":(exclude)src/univalue/" ":(exclude)doc/release-notes/"; then
|
if ! git diff -U0 "${COMMIT_RANGE}" -- *.cpp *.h *.md *.py *.sh ":(exclude)src/dashbls/" ":(exclude)src/util/expected.h" ":(exclude)src/immer/" ":(exclude)src/leveldb/" ":(exclude)src/crc32c/" ":(exclude)src/secp256k1/" ":(exclude)src/minisketch/" ":(exclude)src/univalue/" ":(exclude)doc/release-notes/"; then
|
||||||
echo "Failed to get a diff"
|
echo "Failed to get a diff"
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
@ -32,6 +32,7 @@ unsigned-integer-overflow:crypto/
|
|||||||
unsigned-integer-overflow:FuzzedDataProvider.h
|
unsigned-integer-overflow:FuzzedDataProvider.h
|
||||||
unsigned-integer-overflow:hash.cpp
|
unsigned-integer-overflow:hash.cpp
|
||||||
unsigned-integer-overflow:leveldb/
|
unsigned-integer-overflow:leveldb/
|
||||||
|
unsigned-integer-overflow:minisketch/
|
||||||
unsigned-integer-overflow:policy/fees.cpp
|
unsigned-integer-overflow:policy/fees.cpp
|
||||||
unsigned-integer-overflow:prevector.h
|
unsigned-integer-overflow:prevector.h
|
||||||
unsigned-integer-overflow:pubkey.h
|
unsigned-integer-overflow:pubkey.h
|
||||||
@ -58,6 +59,7 @@ implicit-integer-sign-change:crypto/
|
|||||||
# implicit-integer-sign-change in FuzzedDataProvider's ConsumeIntegralInRange
|
# implicit-integer-sign-change in FuzzedDataProvider's ConsumeIntegralInRange
|
||||||
implicit-integer-sign-change:FuzzedDataProvider.h
|
implicit-integer-sign-change:FuzzedDataProvider.h
|
||||||
implicit-integer-sign-change:key.cpp
|
implicit-integer-sign-change:key.cpp
|
||||||
|
implicit-integer-sign-change:minisketch/
|
||||||
implicit-integer-sign-change:policy/fees.cpp
|
implicit-integer-sign-change:policy/fees.cpp
|
||||||
implicit-integer-sign-change:prevector.h
|
implicit-integer-sign-change:prevector.h
|
||||||
implicit-integer-sign-change:script/bitcoinconsensus.cpp
|
implicit-integer-sign-change:script/bitcoinconsensus.cpp
|
||||||
@ -96,6 +98,7 @@ shift-base:arith_uint256.cpp
|
|||||||
shift-base:crypto/
|
shift-base:crypto/
|
||||||
shift-base:hash.cpp
|
shift-base:hash.cpp
|
||||||
shift-base:leveldb/
|
shift-base:leveldb/
|
||||||
|
shift-base:minisketch/
|
||||||
shift-base:streams.h
|
shift-base:streams.h
|
||||||
shift-base:test/fuzz/crypto_diff_fuzz_chacha20.cpp
|
shift-base:test/fuzz/crypto_diff_fuzz_chacha20.cpp
|
||||||
shift-base:util/bip32.cpp
|
shift-base:util/bip32.cpp
|
||||||
|
Loading…
Reference in New Issue
Block a user