Merge pull request #3132 from codablock/pr_backport_prevector_stuff

Backport prevector optimizations

src/Makefile.bench.include

@@ -32,7 +32,7 @@ bench_bench_dash_SOURCES = \
   bench/poly1305.cpp \
   bench/perf.cpp \
   bench/perf.h \
-  bench/prevector_destructor.cpp \
+  bench/prevector.cpp \
   bench/string_cast.cpp
 
 nodist_bench_bench_dash_SOURCES = $(GENERATED_TEST_FILES)

src/bench/prevector.cpp

@@ -0,0 +1,77 @@
+// Copyright (c) 2015-2017 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include "compat.h"
+#include "prevector.h"
+
+#include "bench/bench.h"
+
+struct nontrivial_t {
+    int x;
+    nontrivial_t() :x(-1) {}
+};
+static_assert(!IS_TRIVIALLY_CONSTRUCTIBLE<nontrivial_t>::value,
+              "expected nontrivial_t to not be trivially constructible");
+
+typedef unsigned char trivial_t;
+static_assert(IS_TRIVIALLY_CONSTRUCTIBLE<trivial_t>::value,
+              "expected trivial_t to be trivially constructible");
+
+template <typename T>
+static void PrevectorDestructor(benchmark::State& state)
+{
+    while (state.KeepRunning()) {
+        for (auto x = 0; x < 1000; ++x) {
+            prevector<28, T> t0;
+            prevector<28, T> t1;
+            t0.resize(28);
+            t1.resize(29);
+        }
+    }
+}
+
+template <typename T>
+static void PrevectorClear(benchmark::State& state)
+{
+
+    while (state.KeepRunning()) {
+        for (auto x = 0; x < 1000; ++x) {
+            prevector<28, T> t0;
+            prevector<28, T> t1;
+            t0.resize(28);
+            t0.clear();
+            t1.resize(29);
+            t0.clear();
+        }
+    }
+}
+
+template <typename T>
+void PrevectorResize(benchmark::State& state)
+{
+    while (state.KeepRunning()) {
+        prevector<28, T> t0;
+        prevector<28, T> t1;
+        for (auto x = 0; x < 1000; ++x) {
+            t0.resize(28);
+            t0.resize(0);
+            t1.resize(29);
+            t1.resize(0);
+        }
+    }
+}
+
+#define PREVECTOR_TEST(name, nontrivops, trivops)                       \
+    static void Prevector ## name ## Nontrivial(benchmark::State& state) { \
+        PrevectorResize<nontrivial_t>(state);                           \
+    }                                                                   \
+    BENCHMARK(Prevector ## name ## Nontrivial/*, nontrivops*/);             \
+    static void Prevector ## name ## Trivial(benchmark::State& state) { \
+        PrevectorResize<trivial_t>(state);                              \
+    }                                                                   \
+    BENCHMARK(Prevector ## name ## Trivial/*, trivops*/);
+
+PREVECTOR_TEST(Clear, 28300, 88600)
+PREVECTOR_TEST(Destructor, 28800, 88900)
+PREVECTOR_TEST(Resize, 28900, 90300)

src/bench/prevector_destructor.cpp

@@ -1,36 +0,0 @@
-// Copyright (c) 2015-2017 The Bitcoin Core developers
-// Distributed under the MIT software license, see the accompanying
-// file COPYING or http://www.opensource.org/licenses/mit-license.php.
-
-#include "bench.h"
-#include "prevector.h"
-
-static void PrevectorDestructor(benchmark::State& state)
-{
-    while (state.KeepRunning()) {
-        for (auto x = 0; x < 1000; ++x) {
-            prevector<28, unsigned char> t0;
-            prevector<28, unsigned char> t1;
-            t0.resize(28);
-            t1.resize(29);
-        }
-    }
-}
-
-static void PrevectorClear(benchmark::State& state)
-{
-
-    while (state.KeepRunning()) {
-        for (auto x = 0; x < 1000; ++x) {
-            prevector<28, unsigned char> t0;
-            prevector<28, unsigned char> t1;
-            t0.resize(28);
-            t0.clear();
-            t1.resize(29);
-            t0.clear();
-        }
-    }
-}
-
-BENCHMARK(PrevectorDestructor);
-BENCHMARK(PrevectorClear);

src/compat.h

@@ -10,6 +10,16 @@
 #include "config/dash-config.h"
 #endif
 
+#include <type_traits>
+
+// GCC 4.8 is missing some C++11 type_traits,
+// https://www.gnu.org/software/gcc/gcc-5/changes.html
+#if defined(__GNUC__) && __GNUC__ < 5
+#define IS_TRIVIALLY_CONSTRUCTIBLE std::is_trivial
+#else
+#define IS_TRIVIALLY_CONSTRUCTIBLE std::is_trivially_constructible
+#endif
+
 #ifdef WIN32
 #ifdef _WIN32_WINNT
 #undef _WIN32_WINNT

src/prevector.h

@@ -10,9 +10,12 @@
 #include <stdint.h>
 #include <string.h>
 
+#include <cstddef>
 #include <iterator>
 #include <type_traits>
 
+#include <compat.h>
+
 #pragma pack(push, 1)
 /** Implements a drop-in replacement for std::vector<T> which stores up to N
  *  elements directly (without heap allocation). The types Size and Diff are
@@ -194,16 +197,42 @@ private:
     T* item_ptr(difference_type pos) { return is_direct() ? direct_ptr(pos) : indirect_ptr(pos); }
     const T* item_ptr(difference_type pos) const { return is_direct() ? direct_ptr(pos) : indirect_ptr(pos); }
 
+    void fill(T* dst, ptrdiff_t count) {
+        if (IS_TRIVIALLY_CONSTRUCTIBLE<T>::value) {
+            // The most common use of prevector is where T=unsigned char. For
+            // trivially constructible types, we can use memset() to avoid
+            // looping.
+            ::memset(dst, 0, count * sizeof(T));
+        } else {
+            for (auto i = 0; i < count; ++i) {
+                new(static_cast<void*>(dst + i)) T();
+            }
+        }
+    }
+
+    void fill(T* dst, ptrdiff_t count, const T& value) {
+        for (auto i = 0; i < count; ++i) {
+            new(static_cast<void*>(dst + i)) T(value);
+        }
+    }
+
+    template<typename InputIterator>
+    void fill(T* dst, InputIterator first, InputIterator last) {
+        while (first != last) {
+            new(static_cast<void*>(dst)) T(*first);
+            ++dst;
+            ++first;
+        }
+    }
+
 public:
     void assign(size_type n, const T& val) {
         clear();
         if (capacity() < n) {
             change_capacity(n);
         }
-        while (size() < n) {
-            _size++;
-            new(static_cast<void*>(item_ptr(size() - 1))) T(val);
-        }
+        _size += n;
+        fill(item_ptr(0), n, val);
     }
 
     template<typename InputIterator>
@@ -213,11 +242,8 @@ public:
         if (capacity() < n) {
             change_capacity(n);
         }
-        while (first != last) {
-            _size++;
-            new(static_cast<void*>(item_ptr(size() - 1))) T(*first);
-            ++first;
-        }
+        _size += n;
+        fill(item_ptr(0), first, last);
     }
 
     prevector() : _size(0), _union{{}} {}
@@ -228,31 +254,23 @@ public:
 
     explicit prevector(size_type n, const T& val = T()) : _size(0) {
         change_capacity(n);
-        while (size() < n) {
-            _size++;
-            new(static_cast<void*>(item_ptr(size() - 1))) T(val);
-        }
+        _size += n;
+        fill(item_ptr(0), n, val);
     }
 
     template<typename InputIterator>
     prevector(InputIterator first, InputIterator last) : _size(0) {
         size_type n = last - first;
         change_capacity(n);
-        while (first != last) {
-            _size++;
-            new(static_cast<void*>(item_ptr(size() - 1))) T(*first);
-            ++first;
-        }
+        _size += n;
+        fill(item_ptr(0), first, last);
     }
 
     prevector(const prevector<N, T, Size, Diff>& other) : _size(0) {
-        change_capacity(other.size());
-        const_iterator it = other.begin();
-        while (it != other.end()) {
-            _size++;
-            new(static_cast<void*>(item_ptr(size() - 1))) T(*it);
-            ++it;
-        }
+        size_type n = other.size();
+        change_capacity(n);
+        _size += n;
+        fill(item_ptr(0), other.begin(),  other.end());
     }
 
     prevector(prevector<N, T, Size, Diff>&& other) : _size(0) {
@@ -263,14 +281,7 @@ public:
         if (&other == this) {
             return *this;
         }
-        resize(0);
-        change_capacity(other.size());
-        const_iterator it = other.begin();
-        while (it != other.end()) {
-            _size++;
-            new(static_cast<void*>(item_ptr(size() - 1))) T(*it);
-            ++it;
-        }
+        assign(other.begin(), other.end());
         return *this;
     }
 
@@ -314,16 +325,20 @@ public:
     }
 
     void resize(size_type new_size) {
-        if (size() > new_size) {
+        size_type cur_size = size();
+        if (cur_size == new_size) {
+            return;
+        }
+        if (cur_size > new_size) {
             erase(item_ptr(new_size), end());
+            return;
         }
         if (new_size > capacity()) {
             change_capacity(new_size);
         }
-        while (size() < new_size) {
-            _size++;
-            new(static_cast<void*>(item_ptr(size() - 1))) T();
-        }
+        ptrdiff_t increase = new_size - cur_size;
+        fill(item_ptr(cur_size), increase);
+        _size += increase;
     }
 
     void reserve(size_type new_capacity) {
@@ -346,10 +361,11 @@ public:
         if (capacity() < new_size) {
             change_capacity(new_size + (new_size >> 1));
         }
-        memmove(item_ptr(p + 1), item_ptr(p), (size() - p) * sizeof(T));
+        T* ptr = item_ptr(p);
+        memmove(ptr + 1, ptr, (size() - p) * sizeof(T));
         _size++;
-        new(static_cast<void*>(item_ptr(p))) T(value);
-        return iterator(item_ptr(p));
+        new(static_cast<void*>(ptr)) T(value);
+        return iterator(ptr);
     }
 
     void insert(iterator pos, size_type count, const T& value) {
@@ -358,11 +374,10 @@ public:
         if (capacity() < new_size) {
             change_capacity(new_size + (new_size >> 1));
         }
-        memmove(item_ptr(p + count), item_ptr(p), (size() - p) * sizeof(T));
+        T* ptr = item_ptr(p);
+        memmove(ptr + count, ptr, (size() - p) * sizeof(T));
         _size += count;
-        for (size_type i = 0; i < count; i++) {
-            new(static_cast<void*>(item_ptr(p + i))) T(value);
-        }
+        fill(item_ptr(p), count, value);
     }
 
     template<typename InputIterator>
@@ -373,12 +388,24 @@ public:
         if (capacity() < new_size) {
             change_capacity(new_size + (new_size >> 1));
         }
-        memmove(item_ptr(p + count), item_ptr(p), (size() - p) * sizeof(T));
+        T* ptr = item_ptr(p);
+        memmove(ptr + count, ptr, (size() - p) * sizeof(T));
         _size += count;
-        while (first != last) {
-            new(static_cast<void*>(item_ptr(p))) T(*first);
-            ++p;
-            ++first;
+        fill(ptr, first, last);
+    }
+
+    inline void resize_uninitialized(size_type new_size) {
+        // resize_uninitialized changes the size of the prevector but does not initialize it.
+        // If size < new_size, the added elements must be initialized explicitly.
+        if (capacity() < new_size) {
+            change_capacity(new_size);
+            _size += new_size - size();
+            return;
+        }
+        if (new_size < size()) {
+            erase(item_ptr(new_size), end());
+        } else {
+            _size += new_size - size();
         }
     }
 

src/serialize.h

@@ -828,7 +828,7 @@ void Unserialize_impl(Stream& is, prevector<N, T>& v, const unsigned char&)
     while (i < nSize)
     {
         unsigned int blk = std::min(nSize - i, (unsigned int)(1 + 4999999 / sizeof(T)));
-        v.resize(i + blk);
+        v.resize_uninitialized(i + blk);
         is.read((char*)&v[i], blk * sizeof(T));
         i += blk;
     }
@@ -846,8 +846,8 @@ void Unserialize_impl(Stream& is, prevector<N, T>& v, const V&)
         nMid += 5000000 / sizeof(T);
         if (nMid > nSize)
             nMid = nSize;
-        v.resize(nMid);
-        for (; i < nMid; i++)
+        v.resize_uninitialized(nMid);
+        for (; i < nMid; ++i)
             Unserialize(is, v[i]);
     }
 }

src/test/prevector_tests.cpp

@@ -183,6 +183,26 @@ public:
         pre_vector = pre_vector_alt;
     }
 
+    void resize_uninitialized(realtype values) {
+        size_t r = values.size();
+        size_t s = real_vector.size() / 2;
+        if (real_vector.capacity() < s + r) {
+            real_vector.reserve(s + r);
+        }
+        real_vector.resize(s);
+        pre_vector.resize_uninitialized(s);
+        for (auto v : values) {
+            real_vector.push_back(v);
+        }
+        auto p = pre_vector.size();
+        pre_vector.resize_uninitialized(p + r);
+        for (auto v : values) {
+            pre_vector[p] = v;
+            ++p;
+        }
+        test();
+    }
+
     ~prevector_tester() {
         BOOST_CHECK_MESSAGE(passed, "insecure_rand: " + rand_seed.ToString());
     }
@@ -260,6 +280,14 @@ BOOST_AUTO_TEST_CASE(PrevectorTestInt)
             if (InsecureRandBits(5) == 18) {
                 test.move();
             }
+            if (InsecureRandBits(5) == 19) {
+                unsigned int num = 1 + (InsecureRandBits(4));
+                std::vector<int> values(num);
+                for (auto &v : values) {
+                    v = InsecureRand32();
+                }
+                test.resize_uninitialized(values);
+            }
         }
     }
 }