Merge pull request #6733

7072c54 Support very-fast-running benchmarks (Gavin Andresen)
535ed92 Simple benchmarking framework (Gavin Andresen)

configure.ac

@@ -91,6 +91,11 @@ AC_ARG_ENABLE(tests,
     [use_tests=$enableval],
     [use_tests=yes])
 
+AC_ARG_ENABLE(bench,
+    AS_HELP_STRING([--enable-bench],[compile benchmarks (default is yes)]),
+    [use_bench=$enableval],
+    [use_bench=yes])
+
 AC_ARG_WITH([comparison-tool],
     AS_HELP_STRING([--with-comparison-tool],[path to java comparison tool (requires --enable-tests)]),
     [use_comparison_tool=$withval],
@@ -876,6 +881,7 @@ AM_CONDITIONAL([ENABLE_WALLET],[test x$enable_wallet = xyes])
 AM_CONDITIONAL([ENABLE_TESTS],[test x$use_tests = xyes])
 AM_CONDITIONAL([ENABLE_QT],[test x$bitcoin_enable_qt = xyes])
 AM_CONDITIONAL([ENABLE_QT_TESTS],[test x$use_tests$bitcoin_enable_qt_test = xyesyes])
+AM_CONDITIONAL([ENABLE_BENCH],[test x$use_bench = xyes])
 AM_CONDITIONAL([USE_QRCODE], [test x$use_qr = xyes])
 AM_CONDITIONAL([USE_LCOV],[test x$use_lcov = xyes])
 AM_CONDITIONAL([USE_COMPARISON_TOOL],[test x$use_comparison_tool != xno])

src/Makefile.am

@@ -64,6 +64,7 @@ endif
 
 bin_PROGRAMS =
 TESTS =
+BENCHMARKS =
 
 if BUILD_BITCOIND
   bin_PROGRAMS += bitcoind
@@ -439,6 +440,10 @@ if ENABLE_TESTS
 include Makefile.test.include
 endif
 
+if ENABLE_BENCH
+include Makefile.bench.include
+endif
+
 if ENABLE_QT
 include Makefile.qt.include
 endif

src/Makefile.bench.include

@@ -0,0 +1,45 @@
+bin_PROGRAMS += bench/bench_bitcoin
+BENCH_SRCDIR = bench
+BENCH_BINARY = bench/bench_bitcoin$(EXEEXT)
+
+
+bench_bench_bitcoin_SOURCES = \
+  bench/bench_bitcoin.cpp \
+  bench/bench.cpp \
+  bench/bench.h \
+  bench/Examples.cpp
+
+bench_bench_bitcoin_CPPFLAGS = $(BITCOIN_INCLUDES) $(EVENT_CLFAGS) $(EVENT_PTHREADS_CFLAGS) -I$(builddir)/bench/
+bench_bench_bitcoin_LDADD = \
+  $(LIBBITCOIN_SERVER) \
+  $(LIBBITCOIN_COMMON) \
+  $(LIBBITCOIN_UNIVALUE) \
+  $(LIBBITCOIN_UTIL) \
+  $(LIBBITCOIN_CRYPTO) \
+  $(LIBLEVELDB) \
+  $(LIBMEMENV) \
+  $(LIBSECP256K1)
+
+if ENABLE_ZMQ
+bench_bench_bitcoin_LDADD += $(LIBBITCOIN_ZMQ) $(ZMQ_LIBS)
+endif
+
+if ENABLE_WALLET
+bench_bench_bitcoin_LDADD += $(LIBBITCOIN_WALLET)
+endif
+
+bench_bench_bitcoin_LDADD += $(BOOST_LIBS) $(BDB_LIBS) $(SSL_LIBS) $(CRYPTO_LIBS) $(MINIUPNPC_LIBS) $(EVENT_PTHREADS_LIBS) $(EVENT_LIBS)
+bench_bench_bitcoin_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS)
+
+
+CLEAN_BITCOIN_BENCH = bench/*.gcda bench/*.gcno
+
+CLEANFILES += $(CLEAN_BITCOIN_BENCH)
+
+bitcoin_bench: $(BENCH_BINARY)
+
+bench: $(BENCH_BINARY) FORCE
+	$(BENCH_BINARY)
+
+bitcoin_bench_clean : FORCE
+	rm -f $(CLEAN_BITCOIN_BENCH) $(bench_bench_bitcoin_OBJECTS) $(BENCH_BINARY)

src/bench/Examples.cpp

@@ -0,0 +1,34 @@
+// Copyright (c) 2015 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include "bench.h"
+#include "main.h"
+#include "utiltime.h"
+
+// Sanity test: this should loop ten times, and
+// min/max/average should be close to 100ms.
+static void Sleep100ms(benchmark::State& state)
+{
+    while (state.KeepRunning()) {
+        MilliSleep(100);
+    }
+}
+
+BENCHMARK(Sleep100ms);
+
+// Extremely fast-running benchmark:
+#include <math.h>
+
+volatile double sum = 0.0; // volatile, global so not optimized away
+
+static void Trig(benchmark::State& state)
+{
+    double d = 0.01;
+    while (state.KeepRunning()) {
+        sum += sin(d);
+        d += 0.000001;
+    }
+}
+
+BENCHMARK(Trig);

src/bench/bench.cpp

@@ -0,0 +1,68 @@
+// Copyright (c) 2015 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+#include "bench.h"
+#include <iostream>
+#include <sys/time.h>
+
+using namespace benchmark;
+
+std::map<std::string, BenchFunction> BenchRunner::benchmarks;
+
+static double gettimedouble(void) {
+    struct timeval tv;
+    gettimeofday(&tv, NULL);
+    return tv.tv_usec * 0.000001 + tv.tv_sec;
+}
+
+BenchRunner::BenchRunner(std::string name, BenchFunction func)
+{
+    benchmarks.insert(std::make_pair(name, func));
+}
+
+void
+BenchRunner::RunAll(double elapsedTimeForOne)
+{
+    std::cout << "Benchmark" << "," << "count" << "," << "min" << "," << "max" << "," << "average" << "\n";
+
+    for (std::map<std::string,BenchFunction>::iterator it = benchmarks.begin();
+         it != benchmarks.end(); ++it) {
+
+        State state(it->first, elapsedTimeForOne);
+        BenchFunction& func = it->second;
+        func(state);
+    }
+}
+
+bool State::KeepRunning()
+{
+    double now;
+    if (count == 0) {
+        beginTime = now = gettimedouble();
+    }
+    else {
+        // timeCheckCount is used to avoid calling gettime most of the time,
+        // so benchmarks that run very quickly get consistent results.
+        if ((count+1)%timeCheckCount != 0) {
+            ++count;
+            return true; // keep going
+        }
+        now = gettimedouble();
+        double elapsedOne = (now - lastTime)/timeCheckCount;
+        if (elapsedOne < minTime) minTime = elapsedOne;
+        if (elapsedOne > maxTime) maxTime = elapsedOne;
+        if (elapsedOne*timeCheckCount < maxElapsed/16) timeCheckCount *= 2;
+    }
+    lastTime = now;
+    ++count;
+
+    if (now - beginTime < maxElapsed) return true; // Keep going
+
+    --count;
+
+    // Output results
+    double average = (now-beginTime)/count;
+    std::cout << name << "," << count << "," << minTime << "," << maxTime << "," << average << "\n";
+
+    return false;
+}

src/bench/bench.h

@@ -0,0 +1,71 @@
+// Copyright (c) 2015 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+#ifndef BITCOIN_BENCH_H
+#define BITCOIN_BENCH_H
+
+// Simple micro-benchmarking framework; API mostly matches a subset of the Google Benchmark
+// framework (see https://github.com/google/benchmark)
+// Wny not use the Google Benchmark framework? Because adding Yet Another Dependency
+// (that uses cmake as its build system and has lots of features we don't need) isn't
+// worth it.
+
+/*
+ * Usage:
+
+static void CODE_TO_TIME(benchmark::State& state)
+{
+    ... do any setup needed...
+    while (state.KeepRunning()) {
+       ... do stuff you want to time...
+    }
+    ... do any cleanup needed...
+}
+
+BENCHMARK(CODE_TO_TIME);
+
+ */
+
+
+#include <boost/function.hpp>
+#include <boost/preprocessor/cat.hpp>
+#include <boost/preprocessor/stringize.hpp>
+#include <map>
+#include <string>
+
+namespace benchmark {
+
+    class State {
+        std::string name;
+        double maxElapsed;
+        double beginTime;
+        double lastTime, minTime, maxTime;
+        int64_t count;
+        int64_t timeCheckCount;
+    public:
+        State(std::string _name, double _maxElapsed) : name(_name), maxElapsed(_maxElapsed), count(0) {
+            minTime = std::numeric_limits<double>::max();
+            maxTime = std::numeric_limits<double>::min();
+            timeCheckCount = 1;
+        }
+        bool KeepRunning();
+    };
+
+    typedef boost::function<void(State&)> BenchFunction;
+
+    class BenchRunner
+    {
+        static std::map<std::string, BenchFunction> benchmarks;
+
+    public:
+        BenchRunner(std::string name, BenchFunction func);
+
+        static void RunAll(double elapsedTimeForOne=1.0);
+    };
+}
+
+// BENCHMARK(foo) expands to:  benchmark::BenchRunner bench_11foo("foo", foo);
+#define BENCHMARK(n) \
+    benchmark::BenchRunner BOOST_PP_CAT(bench_, BOOST_PP_CAT(__LINE__, n))(BOOST_PP_STRINGIZE(n), n);
+
+#endif // BITCOIN_BENCH_H

src/bench/bench_bitcoin.cpp

@@ -0,0 +1,21 @@
+// Copyright (c) 2015 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include "bench.h"
+
+#include "key.h"
+#include "main.h"
+#include "util.h"
+
+int
+main(int argc, char** argv)
+{
+    ECC_Start();
+    SetupEnvironment();
+    fPrintToDebugLog = false; // don't want to write to debug.log file
+
+    benchmark::BenchRunner::RunAll();
+
+    ECC_Stop();
+}