From 935ca8f832cd3b7bddf762ce236225161ea6f1aa Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 22 Mar 2016 18:11:04 -0400 Subject: [PATCH] main: add block timestamp index --- qa/pull-tester/rpc-tests.py | 1 + qa/rpc-tests/timestampindex.py | 51 +++++++++++++++++++++++++++ src/main.cpp | 28 +++++++++++++-- src/main.h | 63 ++++++++++++++++++++++++++++++++++ src/rpcblockchain.cpp | 32 +++++++++++++++++ src/rpcserver.cpp | 1 + src/rpcserver.h | 1 + src/txdb.cpp | 27 +++++++++++++++ src/txdb.h | 4 +++ 9 files changed, 206 insertions(+), 2 deletions(-) create mode 100755 qa/rpc-tests/timestampindex.py diff --git a/qa/pull-tester/rpc-tests.py b/qa/pull-tester/rpc-tests.py index 804914e29..73bfe1e28 100755 --- a/qa/pull-tester/rpc-tests.py +++ b/qa/pull-tester/rpc-tests.py @@ -98,6 +98,7 @@ testScripts = [ 'nodehandling.py', 'reindex.py', 'addressindex.py', + 'timestampindex.py', 'decodescript.py', 'p2p-fullblocktest.py', 'blockchain.py', diff --git a/qa/rpc-tests/timestampindex.py b/qa/rpc-tests/timestampindex.py new file mode 100755 index 000000000..46ff710bd --- /dev/null +++ b/qa/rpc-tests/timestampindex.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python2 +# Copyright (c) 2014-2015 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +# +# Test timestampindex generation and fetching +# + +import time + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import * + + +class TimestampIndexTest(BitcoinTestFramework): + + def setup_chain(self): + print("Initializing test directory "+self.options.tmpdir) + initialize_chain_clean(self.options.tmpdir, 4) + + def setup_network(self): + self.nodes = [] + # Nodes 0/1 are "wallet" nodes + self.nodes.append(start_node(0, self.options.tmpdir, ["-debug"])) + self.nodes.append(start_node(1, self.options.tmpdir, ["-debug", "-timestampindex"])) + # Nodes 2/3 are used for testing + self.nodes.append(start_node(2, self.options.tmpdir, ["-debug"])) + self.nodes.append(start_node(3, self.options.tmpdir, ["-debug", "-timestampindex"])) + connect_nodes(self.nodes[0], 1) + connect_nodes(self.nodes[0], 2) + connect_nodes(self.nodes[0], 3) + + self.is_network_split = False + self.sync_all() + + def run_test(self): + print "Mining 5 blocks..." + blockhashes = self.nodes[0].generate(5) + low = self.nodes[0].getblock(blockhashes[0])["time"] + high = self.nodes[0].getblock(blockhashes[4])["time"] + self.sync_all() + print "Checking timestamp index..." + hashes = self.nodes[1].getblockhashes(high, low) + assert_equal(len(hashes), 5) + assert_equal(sorted(blockhashes), sorted(hashes)) + print "Passed\n" + + +if __name__ == '__main__': + TimestampIndexTest().main() diff --git a/src/main.cpp b/src/main.cpp index 0ad52abbb..de65db40a 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -67,6 +67,7 @@ bool fImporting = false; bool fReindex = false; bool fTxIndex = false; bool fAddressIndex = false; +bool fTimestampIndex = false; bool fHavePruned = false; bool fPruneMode = false; bool fIsBareMultisigStd = DEFAULT_PERMIT_BAREMULTISIG; @@ -1439,6 +1440,17 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa return res; } +bool GetTimestampIndex(const unsigned int &high, const unsigned int &low, std::vector &hashes) +{ + if (!fTimestampIndex) + return error("Timestamp index not enabled"); + + if (!pblocktree->ReadTimestampIndex(high, low, hashes)) + return error("Unable to get hashes for timestamps"); + + return true; +} + bool GetAddressIndex(uint160 addressHash, int type, std::vector > &addressIndex) { if (!fAddressIndex) @@ -2471,8 +2483,12 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin return AbortNode(state, "Failed to write transaction index"); if (fAddressIndex) - if (!pblocktree->WriteAddressIndex(addressIndex)) - return AbortNode(state, "Failed to write address index"); + if (!pblocktree->WriteAddressIndex(addressIndex)) + return AbortNode(state, "Failed to write address index"); + + if (fTimestampIndex) + if (!pblocktree->WriteTimestampIndex(CTimestampIndexKey(pindex->nTime, pindex->GetBlockHash()))) + return AbortNode(state, "Failed to write timestamp index"); // add this block to the view's block chain view.SetBestBlock(pindex->GetBlockHash()); @@ -3869,6 +3885,10 @@ bool static LoadBlockIndexDB() pblocktree->ReadFlag("addressindex", fAddressIndex); LogPrintf("%s: address index %s\n", __func__, fAddressIndex ? "enabled" : "disabled"); + // Check whether we have a timestamp index + pblocktree->ReadFlag("timestampindex", fTimestampIndex); + LogPrintf("%s: timestamp index %s\n", __func__, fTimestampIndex ? "enabled" : "disabled"); + // Load pointer to end of best chain BlockMap::iterator it = mapBlockIndex.find(pcoinsTip->GetBestBlock()); if (it == mapBlockIndex.end()) @@ -4034,6 +4054,10 @@ bool InitBlockIndex(const CChainParams& chainparams) fAddressIndex = GetBoolArg("-addressindex", DEFAULT_ADDRESSINDEX); pblocktree->WriteFlag("addressindex", fAddressIndex); + // Use the provided setting for -timestampindex in the new database + fTimestampIndex = GetBoolArg("-timestampindex", DEFAULT_TIMESTAMPINDEX); + pblocktree->WriteFlag("timestampindex", fTimestampIndex); + LogPrintf("Initializing databases...\n"); // Only add the genesis block if not reindexing (in which case we reuse the one already on disk) diff --git a/src/main.h b/src/main.h index 137915253..fe0a3dd10 100644 --- a/src/main.h +++ b/src/main.h @@ -112,6 +112,7 @@ static const unsigned int DEFAULT_BYTES_PER_SIGOP = 20; static const bool DEFAULT_CHECKPOINTS_ENABLED = true; static const bool DEFAULT_TXINDEX = false; static const bool DEFAULT_ADDRESSINDEX = false; +static const bool DEFAULT_TIMESTAMPINDEX = false; static const unsigned int DEFAULT_BANSCORE_THRESHOLD = 100; static const bool DEFAULT_TESTSAFEMODE = false; @@ -291,6 +292,67 @@ struct CNodeStateStats { std::vector vHeightInFlight; }; +struct CTimestampIndexIteratorKey { + unsigned int timestamp; + + size_t GetSerializeSize(int nType, int nVersion) const { + return 4; + } + template + void Serialize(Stream& s, int nType, int nVersion) const { + ser_writedata32be(s, timestamp); + } + template + void Unserialize(Stream& s, int nType, int nVersion) { + timestamp = ser_readdata32be(s); + } + + CTimestampIndexIteratorKey(unsigned int time) { + timestamp = time; + } + + CTimestampIndexIteratorKey() { + SetNull(); + } + + void SetNull() { + timestamp = 0; + } +}; + +struct CTimestampIndexKey { + unsigned int timestamp; + uint256 blockHash; + + size_t GetSerializeSize(int nType, int nVersion) const { + return 36; + } + template + void Serialize(Stream& s, int nType, int nVersion) const { + ser_writedata32be(s, timestamp); + blockHash.Serialize(s, nType, nVersion); + } + template + void Unserialize(Stream& s, int nType, int nVersion) { + timestamp = ser_readdata32be(s); + blockHash.Unserialize(s, nType, nVersion); + } + + CTimestampIndexKey(unsigned int time, uint256 hash) { + timestamp = time; + blockHash = hash; + } + + CTimestampIndexKey() { + SetNull(); + } + + void SetNull() { + timestamp = 0; + blockHash.SetNull(); + } +}; + struct CAddressIndexKey { unsigned int type; uint160 hashBytes; @@ -510,6 +572,7 @@ public: ScriptError GetScriptError() const { return error; } }; +bool GetTimestampIndex(const unsigned int &high, const unsigned int &low, std::vector &hashes); bool GetAddressIndex(uint160 addressHash, int type, std::vector > &addressIndex); /** Functions for disk access for blocks */ diff --git a/src/rpcblockchain.cpp b/src/rpcblockchain.cpp index f0bcafafe..276e99400 100644 --- a/src/rpcblockchain.cpp +++ b/src/rpcblockchain.cpp @@ -275,6 +275,38 @@ UniValue getrawmempool(const UniValue& params, bool fHelp) return mempoolToJSON(fVerbose); } +UniValue getblockhashes(const UniValue& params, bool fHelp) +{ + if (fHelp || params.size() != 2) + throw runtime_error( + "getblockhashes timestamp\n" + "\nReturns array of hashes of blocks within the timestamp range provided.\n" + "\nArguments:\n" + "1. high (numeric, required) The newer block timestamp\n" + "2. low (numeric, required) The older block timestamp\n" + "\nResult:\n" + "[" + " \"hash\" (string) The block hash\n" + "]" + "\nExamples:\n" + ); + + unsigned int high = params[0].get_int(); + unsigned int low = params[1].get_int(); + std::vector blockHashes; + + if (!GetTimestampIndex(high, low, blockHashes)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for block hashes"); + } + + UniValue result(UniValue::VARR); + for (std::vector::const_iterator it=blockHashes.begin(); it!=blockHashes.end(); it++) { + result.push_back(it->GetHex()); + } + + return result; +} + UniValue getblockhash(const UniValue& params, bool fHelp) { if (fHelp || params.size() != 1) diff --git a/src/rpcserver.cpp b/src/rpcserver.cpp index da66b1c0a..51235c184 100644 --- a/src/rpcserver.cpp +++ b/src/rpcserver.cpp @@ -278,6 +278,7 @@ static const CRPCCommand vRPCCommands[] = { "blockchain", "getbestblockhash", &getbestblockhash, true }, { "blockchain", "getblockcount", &getblockcount, true }, { "blockchain", "getblock", &getblock, true }, + { "blockchain", "getblockhashes", &getblockhashes, true }, { "blockchain", "getblockhash", &getblockhash, true }, { "blockchain", "getblockheader", &getblockheader, true }, { "blockchain", "getchaintips", &getchaintips, true }, diff --git a/src/rpcserver.h b/src/rpcserver.h index 8a80fc5ee..0be818093 100644 --- a/src/rpcserver.h +++ b/src/rpcserver.h @@ -258,6 +258,7 @@ extern UniValue getdifficulty(const UniValue& params, bool fHelp); extern UniValue settxfee(const UniValue& params, bool fHelp); extern UniValue getmempoolinfo(const UniValue& params, bool fHelp); extern UniValue getrawmempool(const UniValue& params, bool fHelp); +extern UniValue getblockhashes(const UniValue& params, bool fHelp); extern UniValue getblockhash(const UniValue& params, bool fHelp); extern UniValue getblockheader(const UniValue& params, bool fHelp); extern UniValue getblock(const UniValue& params, bool fHelp); diff --git a/src/txdb.cpp b/src/txdb.cpp index e93594b9e..3aa686058 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -22,6 +22,7 @@ static const char DB_COINS = 'c'; static const char DB_BLOCK_FILES = 'f'; static const char DB_TXINDEX = 't'; static const char DB_ADDRESSINDEX = 'a'; +static const char DB_TIMESTAMPINDEX = 's'; static const char DB_BLOCK_INDEX = 'b'; static const char DB_BEST_BLOCK = 'B'; @@ -196,6 +197,32 @@ bool CBlockTreeDB::ReadAddressIndex(uint160 addressHash, int type, std::vector &hashes) { + + boost::scoped_ptr pcursor(NewIterator()); + + pcursor->Seek(make_pair(DB_TIMESTAMPINDEX, CTimestampIndexIteratorKey(low))); + + while (pcursor->Valid()) { + boost::this_thread::interruption_point(); + std::pair key; + if (pcursor->GetKey(key) && key.first == DB_TIMESTAMPINDEX && key.second.timestamp <= high) { + hashes.push_back(key.second.blockHash); + pcursor->Next(); + } else { + break; + } + } + + return true; +} + bool CBlockTreeDB::WriteFlag(const std::string &name, bool fValue) { return Write(std::make_pair(DB_FLAG, name), fValue ? '1' : '0'); } diff --git a/src/txdb.h b/src/txdb.h index ca32869cc..7fefcaafa 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -19,6 +19,8 @@ class CBlockIndex; struct CDiskTxPos; struct CAddressIndexKey; struct CAddressIndexIteratorKey; +struct CTimestampIndexKey; +struct CTimestampIndexIteratorKey; class uint256; //! -dbcache default (MiB) @@ -61,6 +63,8 @@ public: bool WriteTxIndex(const std::vector > &list); bool WriteAddressIndex(const std::vector > &vect); bool ReadAddressIndex(uint160 addressHash, int type, std::vector > &addressIndex); + bool WriteTimestampIndex(const CTimestampIndexKey ×tampIndex); + bool ReadTimestampIndex(const unsigned int &high, const unsigned int &low, std::vector &vect); bool WriteFlag(const std::string &name, bool fValue); bool ReadFlag(const std::string &name, bool &fValue); bool LoadBlockIndexGuts();