bitcoin#14121: Index for BIP 157 block filters

Co-authored-by: UdjinM6 <UdjinM6@users.noreply.github.com>

src/Makefile.am

@@ -177,6 +177,7 @@ BITCOIN_CORE_H = \
   httprpc.h \
   httpserver.h \
   index/base.h \
+  index/blockfilterindex.h \
   index/txindex.h \
   indirectmap.h \
   init.h \
@@ -341,6 +342,7 @@ libdash_server_a_SOURCES = \
   httprpc.cpp \
   httpserver.cpp \
   index/base.cpp \
+  index/blockfilterindex.cpp \
   index/txindex.cpp \
   interfaces/handler.cpp \
   interfaces/node.cpp \

src/Makefile.test.include

@@ -121,6 +121,7 @@ BITCOIN_TESTS =\
   test/blockchain_tests.cpp \
   test/blockencodings_tests.cpp \
   test/blockfilter_tests.cpp \
+  test/blockfilter_index_tests.cpp \
   test/bloom_tests.cpp \
   test/bls_tests.cpp \
   test/bswap_tests.cpp \

src/blockfilter.cpp

@@ -2,6 +2,9 @@
 // Distributed under the MIT software license, see the accompanying
 // file COPYING or http://www.opensource.org/licenses/mit-license.php.
 
+#include <mutex>
+#include <sstream>
+
 #include <blockfilter.h>
 #include <crypto/siphash.h>
 #include <hash.h>
@@ -15,6 +18,10 @@ static constexpr int GCS_SER_TYPE = SER_NETWORK;
 /// Protocol version used to serialize parameters in GCS filter encoding.
 static constexpr int GCS_SER_VERSION = 0;
 
+static const std::map<BlockFilterType, std::string> g_filter_types = {
+    {BlockFilterType::BASIC_FILTER, "basic"},
+};
+
 template <typename OStream>
 static void GolombRiceEncode(BitStreamWriter<OStream>& bitwriter, uint8_t P, uint64_t x)
 {
@@ -197,6 +204,57 @@ bool GCSFilter::MatchAny(const ElementSet& elements) const
     return MatchInternal(queries.data(), queries.size());
 }
 
+const std::string& BlockFilterTypeName(BlockFilterType filter_type)
+{
+    static std::string unknown_retval = "";
+    auto it = g_filter_types.find(filter_type);
+    return it != g_filter_types.end() ? it->second : unknown_retval;
+}
+
+bool BlockFilterTypeByName(const std::string& name, BlockFilterType& filter_type) {
+    for (const auto& entry : g_filter_types) {
+        if (entry.second == name) {
+            filter_type = entry.first;
+            return true;
+        }
+    }
+    return false;
+}
+
+const std::vector<BlockFilterType>& AllBlockFilterTypes()
+{
+    static std::vector<BlockFilterType> types;
+
+    static std::once_flag flag;
+    std::call_once(flag, []() {
+            types.reserve(g_filter_types.size());
+            for (auto entry : g_filter_types) {
+                types.push_back(entry.first);
+            }
+        });
+
+    return types;
+}
+
+const std::string& ListBlockFilterTypes()
+{
+    static std::string type_list;
+
+    static std::once_flag flag;
+    std::call_once(flag, []() {
+            std::stringstream ret;
+            bool first = true;
+            for (auto entry : g_filter_types) {
+                if (!first) ret << ", ";
+                ret << entry.second;
+                first = false;
+            }
+            type_list = ret.str();
+        });
+
+    return type_list;
+}
+
 static GCSFilter::ElementSet BasicFilterElements(const CBlock& block,
                                                  const CBlockUndo& block_undo)
 {

src/blockfilter.h

@@ -6,6 +6,7 @@
 #define BITCOIN_BLOCKFILTER_H
 
 #include <stdint.h>
+#include <string>
 #include <unordered_set>
 #include <vector>
 
@@ -88,6 +89,18 @@ enum BlockFilterType : uint8_t
     BASIC_FILTER = 0,
 };
 
+/** Get the human-readable name for a filter type. Returns empty string for unknown types. */
+const std::string& BlockFilterTypeName(BlockFilterType filter_type);
+
+/** Find a filter type by its human-readable name. */
+bool BlockFilterTypeByName(const std::string& name, BlockFilterType& filter_type);
+
+/** Get a list of known filter types. */
+const std::vector<BlockFilterType>& AllBlockFilterTypes();
+
+/** Get a comma-separated list of known filter type names. */
+const std::string& ListBlockFilterTypes();
+
 /**
  * Complete block filter struct as defined in BIP 157. Serialization matches
  * payload of "cfilter" messages.

src/index/base.cpp

@@ -40,9 +40,9 @@ bool BaseIndex::DB::ReadBestBlock(CBlockLocator& locator) const
     return success;
 }
 
-bool BaseIndex::DB::WriteBestBlock(const CBlockLocator& locator)
+void BaseIndex::DB::WriteBestBlock(CDBBatch& batch, const CBlockLocator& locator)
 {
-    return Write(DB_BEST_BLOCK, locator);
+    batch.Write(DB_BEST_BLOCK, locator);
 }
 
 BaseIndex::~BaseIndex()
@@ -94,7 +94,11 @@ void BaseIndex::ThreadSync()
         int64_t last_locator_write_time = 0;
         while (true) {
             if (m_interrupt) {
-                WriteBestBlock(pindex);
+                m_best_block_index = pindex;
+                // No need to handle errors in Commit. If it fails, the error will be already be
+                // logged. The best way to recover is to continue, as index cannot be corrupted by
+                // a missed commit to disk for an advanced index state.
+                Commit();
                 return;
             }
 
@@ -102,11 +106,17 @@ void BaseIndex::ThreadSync()
                 LOCK(cs_main);
                 const CBlockIndex* pindex_next = NextSyncBlock(pindex);
                 if (!pindex_next) {
-                    WriteBestBlock(pindex);
                     m_best_block_index = pindex;
                     m_synced = true;
+                    // No need to handle errors in Commit. See rationale above.
+                    Commit();
                     break;
                 }
+                if (pindex_next->pprev != pindex && !Rewind(pindex, pindex_next->pprev)) {
+                    FatalError("%s: Failed to rewind index %s to a previous chain tip",
+                               __func__, GetName());
+                    return;
+                }
                 pindex = pindex_next;
             }
 
@@ -118,8 +128,10 @@ void BaseIndex::ThreadSync()
             }
 
             if (last_locator_write_time + SYNC_LOCATOR_WRITE_INTERVAL < current_time) {
-                WriteBestBlock(pindex);
+                m_best_block_index = pindex;
                 last_locator_write_time = current_time;
+                // No need to handle errors in Commit. See rationale above.
+                Commit();
             }
 
             CBlock block;
@@ -143,12 +155,35 @@ void BaseIndex::ThreadSync()
     }
 }
 
-bool BaseIndex::WriteBestBlock(const CBlockIndex* block_index)
+bool BaseIndex::Commit()
+{
+    CDBBatch batch(GetDB());
+    if (!CommitInternal(batch) || !GetDB().WriteBatch(batch)) {
+        return error("%s: Failed to commit latest %s state", __func__, GetName());
+    }
+    return true;
+}
+
+bool BaseIndex::CommitInternal(CDBBatch& batch)
 {
     LOCK(cs_main);
-    if (!GetDB().WriteBestBlock(chainActive.GetLocator(block_index))) {
+    GetDB().WriteBestBlock(batch, chainActive.GetLocator(m_best_block_index));
-        return error("%s: Failed to write locator to disk", __func__);
+    return true;
+}
+
+bool BaseIndex::Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_tip)
+{
+    assert(current_tip == m_best_block_index);
+    assert(current_tip->GetAncestor(new_tip->nHeight) == new_tip);
+
+    // In the case of a reorg, ensure persisted block locator is not stale.
+    m_best_block_index = new_tip;
+    if (!Commit()) {
+        // If commit fails, revert the best block index to avoid corruption.
+        m_best_block_index = current_tip;
+        return false;
     }
+
     return true;
 }
 
@@ -179,6 +214,11 @@ void BaseIndex::BlockConnected(const std::shared_ptr<const CBlock>& block, const
                       best_block_index->GetBlockHash().ToString());
             return;
         }
+        if (best_block_index != pindex->pprev && !Rewind(best_block_index, pindex->pprev)) {
+            FatalError("%s: Failed to rewind index %s to a previous chain tip",
+                       __func__, GetName());
+            return;
+        }
     }
 
     if (WriteBlock(*block, pindex)) {
@@ -223,9 +263,10 @@ void BaseIndex::ChainStateFlushed(const CBlockLocator& locator)
         return;
     }
 
-    if (!GetDB().WriteBestBlock(locator)) {
+    // No need to handle errors in Commit. If it fails, the error will be already be logged. The
-        error("%s: Failed to write locator to disk", __func__);
+    // best way to recover is to continue, as index cannot be corrupted by a missed commit to disk
-    }
+    // for an advanced index state.
+    Commit();
 }
 
 bool BaseIndex::BlockUntilSyncedToCurrentChain()

src/index/base.h

@@ -32,7 +32,7 @@ protected:
         bool ReadBestBlock(CBlockLocator& locator) const;
 
         /// Write block locator of the chain that the txindex is in sync with.
-        bool WriteBestBlock(const CBlockLocator& locator);
+        void WriteBestBlock(CDBBatch& batch, const CBlockLocator& locator);
     };
 
 private:
@@ -54,8 +54,15 @@ private:
     /// over and the sync thread exits.
     void ThreadSync();
 
-    /// Write the current chain block locator to the DB.
+    /// Write the current index state (eg. chain block locator and subclass-specific items) to disk.
-    bool WriteBestBlock(const CBlockIndex* block_index);
+    ///
+    /// Recommendations for error handling:
+    /// If called on a successor of the previous committed best block in the index, the index can
+    /// continue processing without risk of corruption, though the index state will need to catch up
+    /// from further behind on reboot. If the new state is not a successor of the previous state (due
+    /// to a chain reorganization), the index must halt until Commit succeeds or else it could end up
+    /// getting corrupted.
+    bool Commit();
 
 protected:
     void BlockConnected(const std::shared_ptr<const CBlock>& block, const CBlockIndex* pindex,
@@ -69,6 +76,14 @@ protected:
     /// Write update index entries for a newly connected block.
     virtual bool WriteBlock(const CBlock& block, const CBlockIndex* pindex) { return true; }
 
+    /// Virtual method called internally by Commit that can be overridden to atomically
+    /// commit more index state.
+    virtual bool CommitInternal(CDBBatch& batch);
+
+    /// Rewind index to an earlier chain tip during a chain reorg. The tip must
+    /// be an ancestor of the current best block.
+    virtual bool Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_tip);
+
     virtual DB& GetDB() const = 0;
 
     /// Get the name of the index for display in logs.

src/index/blockfilterindex.cpp

@@ -0,0 +1,464 @@
+// Copyright (c) 2018 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <map>
+
+#include <dbwrapper.h>
+#include <index/blockfilterindex.h>
+#include <serialize.h>
+#include <util/system.h>
+#include <validation.h>
+
+/* The index database stores three items for each block: the disk location of the encoded filter,
+ * its dSHA256 hash, and the header. Those belonging to blocks on the active chain are indexed by
+ * height, and those belonging to blocks that have been reorganized out of the active chain are
+ * indexed by block hash. This ensures that filter data for any block that becomes part of the
+ * active chain can always be retrieved, alleviating timing concerns.
+ *
+ * The filters themselves are stored in flat files and referenced by the LevelDB entries. This
+ * minimizes the amount of data written to LevelDB and keeps the database values constant size. The
+ * disk location of the next block filter to be written (represented as a FlatFilePos) is stored
+ * under the DB_FILTER_POS key.
+ *
+ * Keys for the height index have the type [DB_BLOCK_HEIGHT, uint32 (BE)]. The height is represented
+ * as big-endian so that sequential reads of filters by height are fast.
+ * Keys for the hash index have the type [DB_BLOCK_HASH, uint256].
+ */
+constexpr char DB_BLOCK_HASH = 's';
+constexpr char DB_BLOCK_HEIGHT = 't';
+constexpr char DB_FILTER_POS = 'P';
+
+constexpr unsigned int MAX_FLTR_FILE_SIZE = 0x1000000; // 16 MiB
+/** The pre-allocation chunk size for fltr?????.dat files */
+constexpr unsigned int FLTR_FILE_CHUNK_SIZE = 0x100000; // 1 MiB
+
+namespace {
+
+struct DBVal {
+    uint256 hash;
+    uint256 header;
+    FlatFilePos pos;
+
+    SERIALIZE_METHODS(DBVal, obj)
+    {
+        READWRITE(obj.hash);
+        READWRITE(obj.header);
+        READWRITE(obj.pos);
+    }
+};
+
+struct DBHeightKey {
+    int height;
+
+    DBHeightKey() : height(0) {}
+    DBHeightKey(int height_in) : height(height_in) {}
+
+    template<typename Stream>
+    void Serialize(Stream& s) const
+    {
+        ser_writedata8(s, DB_BLOCK_HEIGHT);
+        ser_writedata32be(s, height);
+    }
+
+    template<typename Stream>
+    void Unserialize(Stream& s)
+    {
+        char prefix = ser_readdata8(s);
+        if (prefix != DB_BLOCK_HEIGHT) {
+            throw std::ios_base::failure("Invalid format for block filter index DB height key");
+        }
+        height = ser_readdata32be(s);
+    }
+};
+
+struct DBHashKey {
+    uint256 hash;
+
+    DBHashKey(const uint256& hash_in) : hash(hash_in) {}
+
+    SERIALIZE_METHODS(DBHashKey, obj)
+    {
+        char prefix = DB_BLOCK_HASH;
+        READWRITE(prefix);
+        if (prefix != DB_BLOCK_HASH) {
+            throw std::ios_base::failure("Invalid format for block filter index DB hash key");
+        }
+
+        READWRITE(obj.hash);
+    }
+};
+
+}; // namespace
+
+static std::map<BlockFilterType, BlockFilterIndex> g_filter_indexes;
+
+BlockFilterIndex::BlockFilterIndex(BlockFilterType filter_type,
+                                   size_t n_cache_size, bool f_memory, bool f_wipe)
+    : m_filter_type(filter_type)
+{
+    const std::string& filter_name = BlockFilterTypeName(filter_type);
+    if (filter_name.empty()) throw std::invalid_argument("unknown filter_type");
+
+    fs::path path = GetDataDir() / "indexes" / "blockfilter" / filter_name;
+    fs::create_directories(path);
+
+    m_name = filter_name + " block filter index";
+    m_db = MakeUnique<BaseIndex::DB>(path / "db", n_cache_size, f_memory, f_wipe);
+    m_filter_fileseq = MakeUnique<FlatFileSeq>(std::move(path), "fltr", FLTR_FILE_CHUNK_SIZE);
+}
+
+bool BlockFilterIndex::Init()
+{
+    if (!m_db->Read(DB_FILTER_POS, m_next_filter_pos)) {
+        // Check that the cause of the read failure is that the key does not exist. Any other errors
+        // indicate database corruption or a disk failure, and starting the index would cause
+        // further corruption.
+        if (m_db->Exists(DB_FILTER_POS)) {
+            return error("%s: Cannot read current %s state; index may be corrupted",
+                         __func__, GetName());
+        }
+
+        // If the DB_FILTER_POS is not set, then initialize to the first location.
+        m_next_filter_pos.nFile = 0;
+        m_next_filter_pos.nPos = 0;
+    }
+    return BaseIndex::Init();
+}
+
+bool BlockFilterIndex::CommitInternal(CDBBatch& batch)
+{
+    const FlatFilePos& pos = m_next_filter_pos;
+
+    // Flush current filter file to disk.
+    CAutoFile file(m_filter_fileseq->Open(pos), SER_DISK, CLIENT_VERSION);
+    if (file.IsNull()) {
+        return error("%s: Failed to open filter file %d", __func__, pos.nFile);
+    }
+    if (!FileCommit(file.Get())) {
+        return error("%s: Failed to commit filter file %d", __func__, pos.nFile);
+    }
+
+    batch.Write(DB_FILTER_POS, pos);
+    return BaseIndex::CommitInternal(batch);
+}
+
+bool BlockFilterIndex::ReadFilterFromDisk(const FlatFilePos& pos, BlockFilter& filter) const
+{
+    CAutoFile filein(m_filter_fileseq->Open(pos, true), SER_DISK, CLIENT_VERSION);
+    if (filein.IsNull()) {
+        return false;
+    }
+
+    uint256 block_hash;
+    std::vector<unsigned char> encoded_filter;
+    try {
+        filein >> block_hash >> encoded_filter;
+        filter = BlockFilter(GetFilterType(), block_hash, std::move(encoded_filter));
+    }
+    catch (const std::exception& e) {
+        return error("%s: Failed to deserialize block filter from disk: %s", __func__, e.what());
+    }
+
+    return true;
+}
+
+size_t BlockFilterIndex::WriteFilterToDisk(FlatFilePos& pos, const BlockFilter& filter)
+{
+    assert(filter.GetFilterType() == GetFilterType());
+
+    size_t data_size =
+        GetSerializeSize(filter.GetBlockHash(), CLIENT_VERSION) +
+        GetSerializeSize(filter.GetEncodedFilter(), CLIENT_VERSION);
+
+    // If writing the filter would overflow the file, flush and move to the next one.
+    if (pos.nPos + data_size > MAX_FLTR_FILE_SIZE) {
+        CAutoFile last_file(m_filter_fileseq->Open(pos), SER_DISK, CLIENT_VERSION);
+        if (last_file.IsNull()) {
+            LogPrintf("%s: Failed to open filter file %d\n", __func__, pos.nFile);
+            return 0;
+        }
+        if (!TruncateFile(last_file.Get(), pos.nPos)) {
+            LogPrintf("%s: Failed to truncate filter file %d\n", __func__, pos.nFile);
+            return 0;
+        }
+        if (!FileCommit(last_file.Get())) {
+            LogPrintf("%s: Failed to commit filter file %d\n", __func__, pos.nFile);
+            return 0;
+        }
+
+        pos.nFile++;
+        pos.nPos = 0;
+    }
+
+    // Pre-allocate sufficient space for filter data.
+    bool out_of_space;
+    m_filter_fileseq->Allocate(pos, data_size, out_of_space);
+    if (out_of_space) {
+        LogPrintf("%s: out of disk space\n", __func__);
+        return 0;
+    }
+
+    CAutoFile fileout(m_filter_fileseq->Open(pos), SER_DISK, CLIENT_VERSION);
+    if (fileout.IsNull()) {
+        LogPrintf("%s: Failed to open filter file %d\n", __func__, pos.nFile);
+        return 0;
+    }
+
+    fileout << filter.GetBlockHash() << filter.GetEncodedFilter();
+    return data_size;
+}
+
+bool BlockFilterIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex)
+{
+    CBlockUndo block_undo;
+    uint256 prev_header;
+
+    if (pindex->nHeight > 0) {
+        if (!UndoReadFromDisk(block_undo, pindex)) {
+            return false;
+        }
+
+        std::pair<uint256, DBVal> read_out;
+        if (!m_db->Read(DBHeightKey(pindex->nHeight - 1), read_out)) {
+            return false;
+        }
+
+        uint256 expected_block_hash = pindex->pprev->GetBlockHash();
+        if (read_out.first != expected_block_hash) {
+            return error("%s: previous block header belongs to unexpected block %s; expected %s",
+                         __func__, read_out.first.ToString(), expected_block_hash.ToString());
+        }
+
+        prev_header = read_out.second.header;
+    }
+
+    BlockFilter filter(m_filter_type, block, block_undo);
+
+    size_t bytes_written = WriteFilterToDisk(m_next_filter_pos, filter);
+    if (bytes_written == 0) return false;
+
+    std::pair<uint256, DBVal> value;
+    value.first = pindex->GetBlockHash();
+    value.second.hash = filter.GetHash();
+    value.second.header = filter.ComputeHeader(prev_header);
+    value.second.pos = m_next_filter_pos;
+
+    if (!m_db->Write(DBHeightKey(pindex->nHeight), value)) {
+        return false;
+    }
+
+    m_next_filter_pos.nPos += bytes_written;
+    return true;
+}
+
+static bool CopyHeightIndexToHashIndex(CDBIterator& db_it, CDBBatch& batch,
+                                       const std::string& index_name,
+                                       int start_height, int stop_height)
+{
+    DBHeightKey key(start_height);
+    db_it.Seek(key);
+
+    for (int height = start_height; height <= stop_height; ++height) {
+        if (!db_it.GetKey(key) || key.height != height) {
+            return error("%s: unexpected key in %s: expected (%c, %d)",
+                         __func__, index_name, DB_BLOCK_HEIGHT, height);
+        }
+
+        std::pair<uint256, DBVal> value;
+        if (!db_it.GetValue(value)) {
+            return error("%s: unable to read value in %s at key (%c, %d)",
+                         __func__, index_name, DB_BLOCK_HEIGHT, height);
+        }
+
+        batch.Write(DBHashKey(value.first), std::move(value.second));
+
+        db_it.Next();
+    }
+    return true;
+}
+
+bool BlockFilterIndex::Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_tip)
+{
+    assert(current_tip->GetAncestor(new_tip->nHeight) == new_tip);
+
+    CDBBatch batch(*m_db);
+    std::unique_ptr<CDBIterator> db_it(m_db->NewIterator());
+
+    // During a reorg, we need to copy all filters for blocks that are getting disconnected from the
+    // height index to the hash index so we can still find them when the height index entries are
+    // overwritten.
+    if (!CopyHeightIndexToHashIndex(*db_it, batch, m_name, new_tip->nHeight, current_tip->nHeight)) {
+        return false;
+    }
+
+    // The latest filter position gets written in Commit by the call to the BaseIndex::Rewind.
+    // But since this creates new references to the filter, the position should get updated here
+    // atomically as well in case Commit fails.
+    batch.Write(DB_FILTER_POS, m_next_filter_pos);
+    if (!m_db->WriteBatch(batch)) return false;
+
+    return BaseIndex::Rewind(current_tip, new_tip);
+}
+
+static bool LookupOne(const CDBWrapper& db, const CBlockIndex* block_index, DBVal& result)
+{
+    // First check if the result is stored under the height index and the value there matches the
+    // block hash. This should be the case if the block is on the active chain.
+    std::pair<uint256, DBVal> read_out;
+    if (!db.Read(DBHeightKey(block_index->nHeight), read_out)) {
+        return false;
+    }
+    if (read_out.first == block_index->GetBlockHash()) {
+        result = std::move(read_out.second);
+        return true;
+    }
+
+    // If value at the height index corresponds to an different block, the result will be stored in
+    // the hash index.
+    return db.Read(DBHashKey(block_index->GetBlockHash()), result);
+}
+
+static bool LookupRange(CDBWrapper& db, const std::string& index_name, int start_height,
+                        const CBlockIndex* stop_index, std::vector<DBVal>& results)
+{
+    if (start_height < 0) {
+        return error("%s: start height (%d) is negative", __func__, start_height);
+    }
+    if (start_height > stop_index->nHeight) {
+        return error("%s: start height (%d) is greater than stop height (%d)",
+                     __func__, start_height, stop_index->nHeight);
+    }
+
+    size_t results_size = static_cast<size_t>(stop_index->nHeight - start_height + 1);
+    std::vector<std::pair<uint256, DBVal>> values(results_size);
+
+    DBHeightKey key(start_height);
+    std::unique_ptr<CDBIterator> db_it(db.NewIterator());
+    db_it->Seek(DBHeightKey(start_height));
+    for (int height = start_height; height <= stop_index->nHeight; ++height) {
+        if (!db_it->Valid() || !db_it->GetKey(key) || key.height != height) {
+            return false;
+        }
+
+        size_t i = static_cast<size_t>(height - start_height);
+        if (!db_it->GetValue(values[i])) {
+            return error("%s: unable to read value in %s at key (%c, %d)",
+                         __func__, index_name, DB_BLOCK_HEIGHT, height);
+        }
+
+        db_it->Next();
+    }
+
+    results.resize(results_size);
+
+    // Iterate backwards through block indexes collecting results in order to access the block hash
+    // of each entry in case we need to look it up in the hash index.
+    for (const CBlockIndex* block_index = stop_index;
+         block_index && block_index->nHeight >= start_height;
+         block_index = block_index->pprev) {
+        uint256 block_hash = block_index->GetBlockHash();
+
+        size_t i = static_cast<size_t>(block_index->nHeight - start_height);
+        if (block_hash == values[i].first) {
+            results[i] = std::move(values[i].second);
+            continue;
+        }
+
+        if (!db.Read(DBHashKey(block_hash), results[i])) {
+            return error("%s: unable to read value in %s at key (%c, %s)",
+                         __func__, index_name, DB_BLOCK_HASH, block_hash.ToString());
+        }
+    }
+
+    return true;
+}
+
+bool BlockFilterIndex::LookupFilter(const CBlockIndex* block_index, BlockFilter& filter_out) const
+{
+    DBVal entry;
+    if (!LookupOne(*m_db, block_index, entry)) {
+        return false;
+    }
+
+    return ReadFilterFromDisk(entry.pos, filter_out);
+}
+
+bool BlockFilterIndex::LookupFilterHeader(const CBlockIndex* block_index, uint256& header_out) const
+{
+    DBVal entry;
+    if (!LookupOne(*m_db, block_index, entry)) {
+        return false;
+    }
+
+    header_out = entry.header;
+    return true;
+}
+
+bool BlockFilterIndex::LookupFilterRange(int start_height, const CBlockIndex* stop_index,
+                                         std::vector<BlockFilter>& filters_out) const
+{
+    std::vector<DBVal> entries;
+    if (!LookupRange(*m_db, m_name, start_height, stop_index, entries)) {
+        return false;
+    }
+
+    filters_out.resize(entries.size());
+    auto filter_pos_it = filters_out.begin();
+    for (const auto& entry : entries) {
+        if (!ReadFilterFromDisk(entry.pos, *filter_pos_it)) {
+            return false;
+        }
+        ++filter_pos_it;
+    }
+
+    return true;
+}
+
+bool BlockFilterIndex::LookupFilterHashRange(int start_height, const CBlockIndex* stop_index,
+                                             std::vector<uint256>& hashes_out) const
+
+{
+    std::vector<DBVal> entries;
+    if (!LookupRange(*m_db, m_name, start_height, stop_index, entries)) {
+        return false;
+    }
+
+    hashes_out.clear();
+    hashes_out.reserve(entries.size());
+    for (const auto& entry : entries) {
+        hashes_out.push_back(entry.hash);
+    }
+    return true;
+}
+
+BlockFilterIndex* GetBlockFilterIndex(BlockFilterType filter_type)
+{
+    auto it = g_filter_indexes.find(filter_type);
+    return it != g_filter_indexes.end() ? &it->second : nullptr;
+}
+
+void ForEachBlockFilterIndex(std::function<void (BlockFilterIndex&)> fn)
+{
+    for (auto& entry : g_filter_indexes) fn(entry.second);
+}
+
+bool InitBlockFilterIndex(BlockFilterType filter_type,
+                          size_t n_cache_size, bool f_memory, bool f_wipe)
+{
+    auto result = g_filter_indexes.emplace(std::piecewise_construct,
+                                           std::forward_as_tuple(filter_type),
+                                           std::forward_as_tuple(filter_type,
+                                                                 n_cache_size, f_memory, f_wipe));
+    return result.second;
+}
+
+bool DestroyBlockFilterIndex(BlockFilterType filter_type)
+{
+    return g_filter_indexes.erase(filter_type);
+}
+
+void DestroyAllBlockFilterIndexes()
+{
+    g_filter_indexes.clear();
+}

src/index/blockfilterindex.h

@@ -0,0 +1,94 @@
+// Copyright (c) 2018 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#ifndef BITCOIN_INDEX_BLOCKFILTERINDEX_H
+#define BITCOIN_INDEX_BLOCKFILTERINDEX_H
+
+#include <blockfilter.h>
+#include <chain.h>
+#include <flatfile.h>
+#include <index/base.h>
+
+/**
+ * BlockFilterIndex is used to store and retrieve block filters, hashes, and headers for a range of
+ * blocks by height. An index is constructed for each supported filter type with its own database
+ * (ie. filter data for different types are stored in separate databases).
+ *
+ * This index is used to serve BIP 157 net requests.
+ */
+class BlockFilterIndex final : public BaseIndex
+{
+private:
+    BlockFilterType m_filter_type;
+    std::string m_name;
+    std::unique_ptr<BaseIndex::DB> m_db;
+
+    FlatFilePos m_next_filter_pos;
+    std::unique_ptr<FlatFileSeq> m_filter_fileseq;
+
+    bool ReadFilterFromDisk(const FlatFilePos& pos, BlockFilter& filter) const;
+    size_t WriteFilterToDisk(FlatFilePos& pos, const BlockFilter& filter);
+
+protected:
+    bool Init() override;
+
+    bool CommitInternal(CDBBatch& batch) override;
+
+    bool WriteBlock(const CBlock& block, const CBlockIndex* pindex) override;
+
+    bool Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_tip) override;
+
+    BaseIndex::DB& GetDB() const override { return *m_db; }
+
+    const char* GetName() const override { return m_name.c_str(); }
+
+public:
+    /** Constructs the index, which becomes available to be queried. */
+    explicit BlockFilterIndex(BlockFilterType filter_type,
+                              size_t n_cache_size, bool f_memory = false, bool f_wipe = false);
+
+    BlockFilterType GetFilterType() const { return m_filter_type; }
+
+    /** Get a single filter by block. */
+    bool LookupFilter(const CBlockIndex* block_index, BlockFilter& filter_out) const;
+
+    /** Get a single filter header by block. */
+    bool LookupFilterHeader(const CBlockIndex* block_index, uint256& header_out) const;
+
+    /** Get a range of filters between two heights on a chain. */
+    bool LookupFilterRange(int start_height, const CBlockIndex* stop_index,
+                           std::vector<BlockFilter>& filters_out) const;
+
+    /** Get a range of filter hashes between two heights on a chain. */
+    bool LookupFilterHashRange(int start_height, const CBlockIndex* stop_index,
+                               std::vector<uint256>& hashes_out) const;
+};
+
+/**
+ * Get a block filter index by type. Returns nullptr if index has not been initialized or was
+ * already destroyed.
+ */
+BlockFilterIndex* GetBlockFilterIndex(BlockFilterType filter_type);
+
+/** Iterate over all running block filter indexes, invoking fn on each. */
+void ForEachBlockFilterIndex(std::function<void (BlockFilterIndex&)> fn);
+
+/**
+ * Initialize a block filter index for the given type if one does not already exist. Returns true if
+ * a new index is created and false if one has already been initialized.
+ */
+bool InitBlockFilterIndex(BlockFilterType filter_type,
+                          size_t n_cache_size, bool f_memory = false, bool f_wipe = false);
+
+/**
+ * Destroy the block filter index with the given type. Returns false if no such index exists. This
+ * just releases the allocated memory and closes the database connection, it does not delete the
+ * index data.
+ */
+bool DestroyBlockFilterIndex(BlockFilterType filter_type);
+
+/** Destroy all open block filter indexes. */
+void DestroyAllBlockFilterIndexes();
+
+#endif // BITCOIN_INDEX_BLOCKFILTERINDEX_H

src/init.cpp

@@ -14,6 +14,7 @@
 #include <amount.h>
 #include <banman.h>
 #include <base58.h>
+#include <blockfilter.h>
 #include <chain.h>
 #include <chainparams.h>
 #include <checkpoints.h>
@@ -24,6 +25,7 @@
 #include <hash.h>
 #include <httpserver.h>
 #include <httprpc.h>
+#include <index/blockfilterindex.h>
 #include <index/txindex.h>
 #include <key.h>
 #include <validation.h>
@@ -197,6 +199,7 @@ void Interrupt()
     if (g_txindex) {
         g_txindex->Interrupt();
     }
+    ForEachBlockFilterIndex([](BlockFilterIndex& index) { index.Interrupt(); });
 }
 
 /** Preparing steps before shutting down or restarting the wallet */
@@ -233,6 +236,7 @@ void PrepareShutdown()
     if (peerLogic) UnregisterValidationInterface(peerLogic.get());
     if (g_connman) g_connman->Stop();
     if (g_txindex) g_txindex->Stop();
+    ForEachBlockFilterIndex([](BlockFilterIndex& index) { index.Stop(); });
 
     StopTorControl();
 
@@ -266,6 +270,7 @@ void PrepareShutdown()
     g_connman.reset();
     g_banman.reset();
     g_txindex.reset();
+    DestroyAllBlockFilterIndexes();
 
     if (g_is_mempool_loaded && gArgs.GetArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) {
         DumpMempool();
@@ -499,6 +504,10 @@ void SetupServerArgs()
     gArgs.AddArg("-spentindex", strprintf("Maintain a full spent index, used to query the spending txid and input index for an outpoint (default: %u)", DEFAULT_SPENTINDEX), false, OptionsCategory::INDEXING);
     gArgs.AddArg("-timestampindex", strprintf("Maintain a timestamp index for block hashes, used to query blocks hashes by a range of timestamps (default: %u)", DEFAULT_TIMESTAMPINDEX), false, OptionsCategory::INDEXING);
     gArgs.AddArg("-txindex", strprintf("Maintain a full transaction index, used by the getrawtransaction rpc call (default: %u)", DEFAULT_TXINDEX), false, OptionsCategory::INDEXING);
+    gArgs.AddArg("-blockfilterindex=<type>",
+                 strprintf("Maintain an index of compact filters by block (default: %s, values: %s).", DEFAULT_BLOCKFILTERINDEX, ListBlockFilterTypes()) +
+                 " If <type> is not supplied or if <type> = 1, indexes for all known types are enabled.",
+                 false, OptionsCategory::OPTIONS);
 
     gArgs.AddArg("-asmap=<file>", strprintf("Specify asn mapping used for bucketing of the peers (default: %s). Relative paths will be prefixed by the net-specific datadir location.", DEFAULT_ASMAP_FILENAME), false, OptionsCategory::CONNECTION);
     gArgs.AddArg("-addnode=<ip>", "Add a node to connect to and attempt to keep the connection open (see the `addnode` RPC command help for more info). This option can be specified multiple times to add multiple nodes.", false, OptionsCategory::CONNECTION);
@@ -1147,6 +1156,7 @@ int nUserMaxConnections;
 int nFD;
 ServiceFlags nLocalServices = ServiceFlags(NODE_NETWORK | NODE_NETWORK_LIMITED);
 int64_t peer_connect_timeout;
+std::vector<BlockFilterType> g_enabled_filter_types;
 
 } // namespace
 
@@ -1224,6 +1234,22 @@ bool AppInitParameterInteraction()
         return InitError(strprintf(_("Specified blocks directory \"%s\" does not exist."), gArgs.GetArg("-blocksdir", "").c_str()));
     }
 
+    // parse and validate enabled filter types
+    std::string blockfilterindex_value = gArgs.GetArg("-blockfilterindex", DEFAULT_BLOCKFILTERINDEX);
+    if (blockfilterindex_value == "" || blockfilterindex_value == "1") {
+        g_enabled_filter_types = AllBlockFilterTypes();
+    } else if (blockfilterindex_value != "0") {
+        const std::vector<std::string> names = gArgs.GetArgs("-blockfilterindex");
+        g_enabled_filter_types.reserve(names.size());
+        for (const auto& name : names) {
+            BlockFilterType filter_type;
+            if (!BlockFilterTypeByName(name, filter_type)) {
+                return InitError(strprintf(_("Unknown -blockfilterindex value %s."), name));
+            }
+            g_enabled_filter_types.push_back(filter_type);
+        }
+    }
+
     // if using block pruning, then disallow txindex and require disabling governance validation
     if (gArgs.GetArg("-prune", 0)) {
         if (gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX))
@@ -1231,6 +1257,9 @@ bool AppInitParameterInteraction()
         if (!gArgs.GetBoolArg("-disablegovernance", false)) {
             return InitError(_("Prune mode is incompatible with -disablegovernance=false."));
         }
+        if (!g_enabled_filter_types.empty()) {
+            return InitError(_("Prune mode is incompatible with -blockfilterindex."));
+        }
     }
 
     if (gArgs.IsArgSet("-devnet")) {
@@ -2016,6 +2045,13 @@ bool AppInitMain()
     nTotalCache -= nBlockTreeDBCache;
     int64_t nTxIndexCache = std::min(nTotalCache / 8, gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX) ? nMaxTxIndexCache << 20 : 0);
     nTotalCache -= nTxIndexCache;
+    int64_t filter_index_cache = 0;
+    if (!g_enabled_filter_types.empty()) {
+        size_t n_indexes = g_enabled_filter_types.size();
+        int64_t max_cache = std::min(nTotalCache / 8, max_filter_index_cache << 20);
+        filter_index_cache = max_cache / n_indexes;
+        nTotalCache -= filter_index_cache * n_indexes;
+    }
     int64_t nCoinDBCache = std::min(nTotalCache / 2, (nTotalCache / 4) + (1 << 23)); // use 25%-50% of the remainder for disk cache
     nCoinDBCache = std::min(nCoinDBCache, nMaxCoinsDBCache << 20); // cap total coins db cache
     nTotalCache -= nCoinDBCache;
@@ -2027,6 +2063,10 @@ bool AppInitMain()
     if (gArgs.GetBoolArg("-txindex", DEFAULT_TXINDEX)) {
         LogPrintf("* Using %.1fMiB for transaction index database\n", nTxIndexCache * (1.0 / 1024 / 1024));
     }
+    for (BlockFilterType filter_type : g_enabled_filter_types) {
+        LogPrintf("* Using %.1f MiB for %s block filter index database\n",
+                  filter_index_cache * (1.0 / 1024 / 1024), BlockFilterTypeName(filter_type));
+    }
     LogPrintf("* Using %.1fMiB for chain state database\n", nCoinDBCache * (1.0 / 1024 / 1024));
     LogPrintf("* Using %.1fMiB for in-memory UTXO set (plus up to %.1fMiB of unused mempool space)\n", nCoinCacheUsage * (1.0 / 1024 / 1024), nMempoolSizeMax * (1.0 / 1024 / 1024));
 
@@ -2251,6 +2291,11 @@ bool AppInitMain()
         g_txindex->Start();
     }
 
+    for (const auto& filter_type : g_enabled_filter_types) {
+        InitBlockFilterIndex(filter_type, filter_index_cache, false, fReindex);
+        GetBlockFilterIndex(filter_type)->Start();
+    }
+
     // ********************************************************* Step 9: load wallet
     if (!g_wallet_init_interface.Open()) return false;
 

src/rpc/blockchain.cpp

@@ -8,6 +8,7 @@
 
 #include <amount.h>
 #include <base58.h>
+#include <blockfilter.h>
 #include <chain.h>
 #include <chainparams.h>
 #include <checkpoints.h>
@@ -15,6 +16,7 @@
 #include <node/coinstats.h>
 #include <core_io.h>
 #include <consensus/validation.h>
+#include <index/blockfilterindex.h>
 #include <key_io.h>
 #include <validation.h>
 #include <index/txindex.h>
@@ -2488,6 +2490,80 @@ UniValue scantxoutset(const JSONRPCRequest& request)
     return result;
 }
 
+static UniValue getblockfilter(const JSONRPCRequest& request)
+{
+    if (request.fHelp || request.params.size() < 1 || request.params.size() > 2) {
+        throw std::runtime_error(
+            "getblockfilter <blockhash> <filtertype>\n"
+            "\nRetrieve a BIP 157 content filter for a particular block.\n"
+            "\nArguments:\n"
+            "\n1. blockhash           (string, required) The hash of the block"
+            "\n2. filtertype          (string, optional) The type name of the filter (default: basic)"
+            "\nResult:\n"
+            "{\n"
+            "  \"filter\" : (string) the hex-encoded filter data\n"
+            "  \"header\" : (string) the hex-encoded filter header\n"
+            "}\n"
+            + HelpExampleCli("getblockfilter", "\"00000000c937983704a73af28acdec37b049d214adbda81d7e2a3dd146f6ed09\" \"basic\"")
+        );
+    }
+
+    uint256 block_hash = ParseHashV(request.params[0], "blockhash");
+    std::string filtertype_name = "basic";
+    if (!request.params[1].isNull()) {
+        filtertype_name = request.params[1].get_str();
+    }
+
+    BlockFilterType filtertype;
+    if (!BlockFilterTypeByName(filtertype_name, filtertype)) {
+        throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Unknown filtertype");
+    }
+
+    BlockFilterIndex* index = GetBlockFilterIndex(filtertype);
+    if (!index) {
+        throw JSONRPCError(RPC_MISC_ERROR, "Index is not enabled for filtertype " + filtertype_name);
+    }
+
+    const CBlockIndex* block_index;
+    bool block_was_connected;
+    {
+        LOCK(cs_main);
+        block_index = LookupBlockIndex(block_hash);
+        if (!block_index) {
+            throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found");
+        }
+        block_was_connected = block_index->IsValid(BLOCK_VALID_SCRIPTS);
+    }
+
+    bool index_ready = index->BlockUntilSyncedToCurrentChain();
+
+    BlockFilter filter;
+    uint256 filter_header;
+    if (!index->LookupFilter(block_index, filter) ||
+        !index->LookupFilterHeader(block_index, filter_header)) {
+        int err_code;
+        std::string errmsg = "Filter not found.";
+
+        if (!block_was_connected) {
+            err_code = RPC_INVALID_ADDRESS_OR_KEY;
+            errmsg += " Block was not connected to active chain.";
+        } else if (!index_ready) {
+            err_code = RPC_MISC_ERROR;
+            errmsg += " Block filters are still in the process of being indexed.";
+        } else {
+            err_code = RPC_INTERNAL_ERROR;
+            errmsg += " This error is unexpected and indicates index corruption.";
+        }
+
+        throw JSONRPCError(err_code, errmsg);
+    }
+
+    UniValue ret(UniValue::VOBJ);
+    ret.pushKV("filter", HexStr(filter.GetEncodedFilter()));
+    ret.pushKV("header", filter_header.GetHex());
+    return ret;
+}
+
 static const CRPCCommand commands[] =
 { //  category              name                      actor (function)         argNames
   //  --------------------- ------------------------  -----------------------  ----------
@@ -2519,6 +2595,7 @@ static const CRPCCommand commands[] =
 
     { "blockchain",         "preciousblock",          &preciousblock,          {"blockhash"} },
     { "blockchain",         "scantxoutset",           &scantxoutset,           {"action", "scanobjects"} },
+    { "blockchain",         "getblockfilter",         &getblockfilter,         {"blockhash", "filtertype"} },
 
     /* Not shown in help */
     { "hidden",             "invalidateblock",        &invalidateblock,        {"blockhash"} },

src/test/blockfilter_index_tests.cpp

@@ -0,0 +1,307 @@
+// Copyright (c) 2017-2018 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <blockfilter.h>
+#include <chainparams.h>
+#include <consensus/validation.h>
+#include <index/blockfilterindex.h>
+#include <miner.h>
+#include <pow.h>
+#include <test/test_dash.h>
+#include <script/standard.h>
+#include <validation.h>
+
+#include <boost/test/unit_test.hpp>
+
+BOOST_AUTO_TEST_SUITE(blockfilter_index_tests)
+
+static bool ComputeFilter(BlockFilterType filter_type, const CBlockIndex* block_index,
+                          BlockFilter& filter)
+{
+    CBlock block;
+    if (!ReadBlockFromDisk(block, block_index->GetBlockPos(), Params().GetConsensus())) {
+        return false;
+    }
+
+    CBlockUndo block_undo;
+    if (block_index->nHeight > 0 && !UndoReadFromDisk(block_undo, block_index)) {
+        return false;
+    }
+
+    filter = BlockFilter(filter_type, block, block_undo);
+    return true;
+}
+
+static bool CheckFilterLookups(BlockFilterIndex& filter_index, const CBlockIndex* block_index,
+                               uint256& last_header)
+{
+    BlockFilter expected_filter;
+    if (!ComputeFilter(filter_index.GetFilterType(), block_index, expected_filter)) {
+        BOOST_ERROR("ComputeFilter failed on block " << block_index->nHeight);
+        return false;
+    }
+
+    BlockFilter filter;
+    uint256 filter_header;
+    std::vector<BlockFilter> filters;
+    std::vector<uint256> filter_hashes;
+
+    BOOST_CHECK(filter_index.LookupFilter(block_index, filter));
+    BOOST_CHECK(filter_index.LookupFilterHeader(block_index, filter_header));
+    BOOST_CHECK(filter_index.LookupFilterRange(block_index->nHeight, block_index, filters));
+    BOOST_CHECK(filter_index.LookupFilterHashRange(block_index->nHeight, block_index,
+                                                   filter_hashes));
+
+    BOOST_CHECK_EQUAL(filters.size(), 1);
+    BOOST_CHECK_EQUAL(filter_hashes.size(), 1);
+
+    BOOST_CHECK_EQUAL(filter.GetHash(), expected_filter.GetHash());
+    BOOST_CHECK_EQUAL(filter_header, expected_filter.ComputeHeader(last_header));
+    BOOST_CHECK_EQUAL(filters[0].GetHash(), expected_filter.GetHash());
+    BOOST_CHECK_EQUAL(filter_hashes[0], expected_filter.GetHash());
+
+    filters.clear();
+    filter_hashes.clear();
+    last_header = filter_header;
+    return true;
+}
+
+static CBlock CreateBlock(const CBlockIndex* prev,
+                          const std::vector<CMutableTransaction>& txns,
+                          const CScript& scriptPubKey)
+{
+    const CChainParams& chainparams = Params();
+    std::unique_ptr<CBlockTemplate> pblocktemplate = BlockAssembler(chainparams).CreateNewBlock(scriptPubKey);
+    CBlock& block = pblocktemplate->block;
+    block.hashPrevBlock = prev->GetBlockHash();
+    block.nTime = prev->nTime + 1;
+
+    // Replace mempool-selected txns with just coinbase plus passed-in txns:
+    block.vtx.resize(1);
+    for (const CMutableTransaction& tx : txns) {
+        block.vtx.push_back(MakeTransactionRef(tx));
+    }
+    // IncrementExtraNonce creates a valid coinbase and merkleRoot
+    unsigned int extraNonce = 0;
+    IncrementExtraNonce(&block, prev, extraNonce);
+
+    while (!CheckProofOfWork(block.GetHash(), block.nBits, chainparams.GetConsensus())) ++block.nNonce;
+
+    return block;
+}
+
+static bool BuildChain(const CBlockIndex* pindex, const CScript& coinbase_script_pub_key,
+                       size_t length, std::vector<std::shared_ptr<CBlock>>& chain)
+{
+    std::vector<CMutableTransaction> no_txns;
+
+    chain.resize(length);
+    for (auto& block : chain) {
+        block = std::make_shared<CBlock>(CreateBlock(pindex, no_txns, coinbase_script_pub_key));
+        CBlockHeader header = block->GetBlockHeader();
+
+        CValidationState state;
+        if (!ProcessNewBlockHeaders({header}, state, Params(), &pindex, nullptr)) {
+            return false;
+        }
+    }
+
+    return true;
+}
+
+BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, TestChain100Setup)
+{
+    BlockFilterIndex filter_index(BlockFilterType::BASIC_FILTER, 1 << 20, true);
+
+    uint256 last_header;
+
+    // Filter should not be found in the index before it is started.
+    {
+        LOCK(cs_main);
+
+        BlockFilter filter;
+        uint256 filter_header;
+        std::vector<BlockFilter> filters;
+        std::vector<uint256> filter_hashes;
+
+        for (const CBlockIndex* block_index = chainActive.Genesis();
+             block_index != nullptr;
+             block_index = chainActive.Next(block_index)) {
+            BOOST_CHECK(!filter_index.LookupFilter(block_index, filter));
+            BOOST_CHECK(!filter_index.LookupFilterHeader(block_index, filter_header));
+            BOOST_CHECK(!filter_index.LookupFilterRange(block_index->nHeight, block_index, filters));
+            BOOST_CHECK(!filter_index.LookupFilterHashRange(block_index->nHeight, block_index,
+                                                            filter_hashes));
+        }
+    }
+
+    // BlockUntilSyncedToCurrentChain should return false before index is started.
+    BOOST_CHECK(!filter_index.BlockUntilSyncedToCurrentChain());
+
+    filter_index.Start();
+
+    // Allow filter index to catch up with the block index.
+    constexpr int64_t timeout_ms = 10 * 1000;
+    int64_t time_start = GetTimeMillis();
+    while (!filter_index.BlockUntilSyncedToCurrentChain()) {
+        BOOST_REQUIRE(time_start + timeout_ms > GetTimeMillis());
+        UninterruptibleSleep(std::chrono::milliseconds{100});
+    }
+
+    // Check that filter index has all blocks that were in the chain before it started.
+    {
+        LOCK(cs_main);
+        const CBlockIndex* block_index;
+        for (block_index = chainActive.Genesis();
+             block_index != nullptr;
+             block_index = chainActive.Next(block_index)) {
+            CheckFilterLookups(filter_index, block_index, last_header);
+        }
+    }
+
+    // Create two forks.
+    const CBlockIndex* tip;
+    {
+        LOCK(cs_main);
+        tip = chainActive.Tip();
+    }
+    CScript coinbase_script_pub_key = GetScriptForDestination(coinbaseKey.GetPubKey().GetID());
+    std::vector<std::shared_ptr<CBlock>> chainA, chainB;
+    BOOST_REQUIRE(BuildChain(tip, coinbase_script_pub_key, 10, chainA));
+    BOOST_REQUIRE(BuildChain(tip, coinbase_script_pub_key, 10, chainB));
+
+    // Check that new blocks on chain A get indexed.
+    uint256 chainA_last_header = last_header;
+    for (size_t i = 0; i < 2; i++) {
+        const auto& block = chainA[i];
+        BOOST_REQUIRE(ProcessNewBlock(Params(), block, true, nullptr));
+
+        const CBlockIndex* block_index;
+        {
+            LOCK(cs_main);
+            block_index = LookupBlockIndex(block->GetHash());
+        }
+
+        BOOST_CHECK(filter_index.BlockUntilSyncedToCurrentChain());
+        CheckFilterLookups(filter_index, block_index, chainA_last_header);
+    }
+
+    // Reorg to chain B.
+    uint256 chainB_last_header = last_header;
+    for (size_t i = 0; i < 3; i++) {
+        const auto& block = chainB[i];
+        BOOST_REQUIRE(ProcessNewBlock(Params(), block, true, nullptr));
+
+        const CBlockIndex* block_index;
+        {
+            LOCK(cs_main);
+            block_index = LookupBlockIndex(block->GetHash());
+        }
+
+        BOOST_CHECK(filter_index.BlockUntilSyncedToCurrentChain());
+        CheckFilterLookups(filter_index, block_index, chainB_last_header);
+    }
+
+    // Check that filters for stale blocks on A can be retrieved.
+    chainA_last_header = last_header;
+    for (size_t i = 0; i < 2; i++) {
+        const auto& block = chainA[i];
+        const CBlockIndex* block_index;
+        {
+            LOCK(cs_main);
+            block_index = LookupBlockIndex(block->GetHash());
+        }
+
+        BOOST_CHECK(filter_index.BlockUntilSyncedToCurrentChain());
+        CheckFilterLookups(filter_index, block_index, chainA_last_header);
+    }
+
+    // Reorg back to chain A.
+     for (size_t i = 2; i < 4; i++) {
+         const auto& block = chainA[i];
+        BOOST_REQUIRE(ProcessNewBlock(Params(), block, true, nullptr));
+     }
+
+     // Check that chain A and B blocks can be retrieved.
+     chainA_last_header = last_header;
+     chainB_last_header = last_header;
+     for (size_t i = 0; i < 3; i++) {
+         const CBlockIndex* block_index;
+
+         {
+             LOCK(cs_main);
+             block_index = LookupBlockIndex(chainA[i]->GetHash());
+         }
+         BOOST_CHECK(filter_index.BlockUntilSyncedToCurrentChain());
+         CheckFilterLookups(filter_index, block_index, chainA_last_header);
+
+         {
+             LOCK(cs_main);
+             block_index = LookupBlockIndex(chainB[i]->GetHash());
+         }
+         BOOST_CHECK(filter_index.BlockUntilSyncedToCurrentChain());
+         CheckFilterLookups(filter_index, block_index, chainB_last_header);
+     }
+
+    // Test lookups for a range of filters/hashes.
+    std::vector<BlockFilter> filters;
+    std::vector<uint256> filter_hashes;
+
+    {
+        LOCK(cs_main);
+        tip = chainActive.Tip();
+    }
+    BOOST_CHECK(filter_index.LookupFilterRange(0, tip, filters));
+    BOOST_CHECK(filter_index.LookupFilterHashRange(0, tip, filter_hashes));
+
+    BOOST_CHECK_EQUAL(filters.size(), tip->nHeight + 1);
+    BOOST_CHECK_EQUAL(filter_hashes.size(), tip->nHeight + 1);
+
+    filters.clear();
+    filter_hashes.clear();
+
+    filter_index.Interrupt();
+    filter_index.Stop();
+}
+
+BOOST_FIXTURE_TEST_CASE(blockfilter_index_init_destroy, BasicTestingSetup)
+{
+    SetDataDir("tempdir");
+
+    BlockFilterIndex* filter_index;
+
+    filter_index = GetBlockFilterIndex(BlockFilterType::BASIC_FILTER);
+    BOOST_CHECK(filter_index == nullptr);
+
+    BOOST_CHECK(InitBlockFilterIndex(BlockFilterType::BASIC_FILTER, 1 << 20, true, false));
+
+    filter_index = GetBlockFilterIndex(BlockFilterType::BASIC_FILTER);
+    BOOST_CHECK(filter_index != nullptr);
+    BOOST_CHECK(filter_index->GetFilterType() == BlockFilterType::BASIC_FILTER);
+
+    // Initialize returns false if index already exists.
+    BOOST_CHECK(!InitBlockFilterIndex(BlockFilterType::BASIC_FILTER, 1 << 20, true, false));
+
+    int iter_count = 0;
+    ForEachBlockFilterIndex([&iter_count](BlockFilterIndex& _index) { iter_count++; });
+    BOOST_CHECK_EQUAL(iter_count, 1);
+
+    BOOST_CHECK(DestroyBlockFilterIndex(BlockFilterType::BASIC_FILTER));
+
+    // Destroy returns false because index was already destroyed.
+    BOOST_CHECK(!DestroyBlockFilterIndex(BlockFilterType::BASIC_FILTER));
+
+    filter_index = GetBlockFilterIndex(BlockFilterType::BASIC_FILTER);
+    BOOST_CHECK(filter_index == nullptr);
+
+    // Reinitialize index.
+    BOOST_CHECK(InitBlockFilterIndex(BlockFilterType::BASIC_FILTER, 1 << 20, true, false));
+
+    DestroyAllBlockFilterIndexes();
+
+    filter_index = GetBlockFilterIndex(BlockFilterType::BASIC_FILTER);
+    BOOST_CHECK(filter_index == nullptr);
+}
+
+BOOST_AUTO_TEST_SUITE_END()

src/test/blockfilter_tests.cpp

@@ -54,7 +54,7 @@ BOOST_AUTO_TEST_CASE(gcsfilter_default_constructor)
 
 BOOST_AUTO_TEST_CASE(blockfilter_basic_test)
 {
-    CScript included_scripts[5], excluded_scripts[3];
+    CScript included_scripts[5], excluded_scripts[4];
 
     // First two are outputs on a single transaction.
     included_scripts[0] << std::vector<unsigned char>(0, 65) << OP_CHECKSIG;
@@ -73,14 +73,19 @@ BOOST_AUTO_TEST_CASE(blockfilter_basic_test)
     // This script is not related to the block at all.
     excluded_scripts[1] << std::vector<unsigned char>(5, 33) << OP_CHECKSIG;
 
+    // OP_RETURN is non-standard since it's not followed by a data push, but is still excluded from
+    // filter.
+    excluded_scripts[2] << OP_RETURN << OP_4 << OP_ADD << OP_8 << OP_EQUAL;
+
     CMutableTransaction tx_1;
     tx_1.vout.emplace_back(100, included_scripts[0]);
     tx_1.vout.emplace_back(200, included_scripts[1]);
+    tx_1.vout.emplace_back(0, excluded_scripts[0]);
 
     CMutableTransaction tx_2;
     tx_2.vout.emplace_back(300, included_scripts[2]);
-    tx_2.vout.emplace_back(0, excluded_scripts[0]);
+    tx_2.vout.emplace_back(0, excluded_scripts[2]);
-    tx_2.vout.emplace_back(400, excluded_scripts[2]); // Script is empty
+    tx_2.vout.emplace_back(400, excluded_scripts[3]); // Script is empty
 
     CBlock block;
     block.vtx.push_back(MakeTransactionRef(tx_1));
@@ -90,7 +95,7 @@ BOOST_AUTO_TEST_CASE(blockfilter_basic_test)
     block_undo.vtxundo.emplace_back();
     block_undo.vtxundo.back().vprevout.emplace_back(CTxOut(500, included_scripts[3]), 1000, true);
     block_undo.vtxundo.back().vprevout.emplace_back(CTxOut(600, included_scripts[4]), 10000, false);
-    block_undo.vtxundo.back().vprevout.emplace_back(CTxOut(700, excluded_scripts[2]), 100000, false);
+    block_undo.vtxundo.back().vprevout.emplace_back(CTxOut(700, excluded_scripts[3]), 100000, false);
 
     BlockFilter block_filter(BlockFilterType::BASIC_FILTER, block, block_undo);
     const GCSFilter& filter = block_filter.GetFilter();
@@ -168,4 +173,16 @@ BOOST_AUTO_TEST_CASE(blockfilters_json_test)
     }
 }
 
+BOOST_AUTO_TEST_CASE(blockfilter_type_names)
+{
+    BOOST_CHECK_EQUAL(BlockFilterTypeName(BlockFilterType::BASIC_FILTER), "basic");
+    BOOST_CHECK_EQUAL(BlockFilterTypeName(static_cast<BlockFilterType>(255)), "");
+
+    BlockFilterType filter_type;
+    BOOST_CHECK(BlockFilterTypeByName("basic", filter_type));
+    BOOST_CHECK_EQUAL(filter_type, BlockFilterType::BASIC_FILTER);
+
+    BOOST_CHECK(!BlockFilterTypeByName("unknown", filter_type));
+}
+
 BOOST_AUTO_TEST_SUITE_END()

src/txdb.h

@@ -40,6 +40,8 @@ static const int64_t nMaxBlockDBCache = 2;
 // Unlike for the UTXO database, for the txindex scenario the leveldb cache make
 // a meaningful difference: https://github.com/bitcoin/bitcoin/pull/8273#issuecomment-229601991
 static const int64_t nMaxTxIndexCache = 1024;
+//! Max memory allocated to all block filter index caches combined in MiB.
+static const int64_t max_filter_index_cache = 1024;
 //! Max memory allocated to coin DB specific cache (MiB)
 static const int64_t nMaxCoinsDBCache = 8;
 

src/validation.h

@@ -88,6 +88,7 @@ static const bool DEFAULT_TXINDEX = true;
 static const bool DEFAULT_ADDRESSINDEX = false;
 static const bool DEFAULT_TIMESTAMPINDEX = false;
 static const bool DEFAULT_SPENTINDEX = false;
+static const char* const DEFAULT_BLOCKFILTERINDEX = "0";
 static const unsigned int DEFAULT_BANSCORE_THRESHOLD = 100;
 /** Default for -persistmempool */
 static const bool DEFAULT_PERSIST_MEMPOOL = true;

test/functional/rpc_getblockfilter.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+# Copyright (c) 2018 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+"""Test the getblockfilter RPC."""
+
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.util import (
+    assert_equal, assert_is_hex_string, assert_raises_rpc_error,
+    connect_nodes, disconnect_nodes, sync_blocks
+    )
+
+FILTER_TYPES = ["basic"]
+
+class GetBlockFilterTest(BitcoinTestFramework):
+    def set_test_params(self):
+        self.setup_clean_chain = True
+        self.num_nodes = 2
+        self.extra_args = [["-blockfilterindex"], []]
+
+    def run_test(self):
+        # Create two chains by disconnecting nodes 0 & 1, mining, then reconnecting
+        disconnect_nodes(self.nodes[0], 1)
+
+        self.nodes[0].generate(3)
+        self.nodes[1].generate(4)
+
+        assert_equal(self.nodes[0].getblockcount(), 3)
+        chain0_hashes = [self.nodes[0].getblockhash(block_height) for block_height in range(4)]
+
+        # Reorg node 0 to a new chain
+        connect_nodes(self.nodes[0], 1)
+        sync_blocks(self.nodes)
+
+        assert_equal(self.nodes[0].getblockcount(), 4)
+        chain1_hashes = [self.nodes[0].getblockhash(block_height) for block_height in range(4)]
+
+        # Test getblockfilter returns a filter for all blocks and filter types on active chain
+        for block_hash in chain1_hashes:
+            for filter_type in FILTER_TYPES:
+                result = self.nodes[0].getblockfilter(block_hash, filter_type)
+                assert_is_hex_string(result['filter'])
+
+        # Test getblockfilter returns a filter for all blocks and filter types on stale chain
+        for block_hash in chain0_hashes:
+            for filter_type in FILTER_TYPES:
+                result = self.nodes[0].getblockfilter(block_hash, filter_type)
+                assert_is_hex_string(result['filter'])
+
+        # Test getblockfilter with unknown block
+        bad_block_hash = "0123456789abcdef" * 4
+        assert_raises_rpc_error(-5, "Block not found", self.nodes[0].getblockfilter, bad_block_hash, "basic")
+
+        # Test getblockfilter with undefined filter type
+        genesis_hash = self.nodes[0].getblockhash(0)
+        assert_raises_rpc_error(-5, "Unknown filtertype", self.nodes[0].getblockfilter, genesis_hash, "unknown")
+
+if __name__ == '__main__':
+    GetBlockFilterTest().main()

test/functional/test_runner.py

@@ -146,6 +146,7 @@ BASE_SCRIPTS = [
     'wallet_txn_doublespend.py',
     'wallet_txn_clone.py --mineblock',
     'feature_notifications.py',
+    'rpc_getblockfilter.py',
     'rpc_invalidateblock.py',
     'feature_txindex.py',
     'mempool_packages.py',