From 075b416f560463c818f8cf2c5821c5a9dd7c2aca Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 2 Jun 2016 11:52:24 -0400 Subject: [PATCH 01/59] --- bitcore start --- From 9babc7ff9fa19b93cf7b3ef2f73ec4e66f8c132f Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Sat, 5 Mar 2016 16:31:10 -0500 Subject: [PATCH 02/59] main: start of address index Adds a configuration option for addressindex to search for txids by address. Includes an additional rpc method for getting the txids for an address. --- src/init.cpp | 2 ++ src/main.cpp | 62 +++++++++++++++++++++++++++++++++++++++++++ src/main.h | 38 ++++++++++++++++++++++++++ src/rpcmisc.cpp | 36 +++++++++++++++++++++++++ src/rpcserver.cpp | 3 +++ src/rpcserver.h | 2 ++ src/script/script.cpp | 11 ++++++++ src/script/script.h | 2 ++ src/txdb.cpp | 33 +++++++++++++++++++++++ src/txdb.h | 3 +++ 10 files changed, 192 insertions(+) diff --git a/src/init.cpp b/src/init.cpp index 0b3234566..16598ada4 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -350,6 +350,8 @@ std::string HelpMessage(HelpMessageMode mode) #endif strUsage += HelpMessageOpt("-txindex", strprintf(_("Maintain a full transaction index, used by the getrawtransaction rpc call (default: %u)"), DEFAULT_TXINDEX)); + strUsage += HelpMessageOpt("-addressindex", strprintf(_("Maintain a full address index, used to query for the balance, txids and unspent outputs for addresses (default: %u)"), DEFAULT_ADDRESSINDEX)); + strUsage += HelpMessageGroup(_("Connection options:")); strUsage += HelpMessageOpt("-addnode=", _("Add a node to connect to and attempt to keep the connection open")); strUsage += HelpMessageOpt("-banscore=", strprintf(_("Threshold for disconnecting misbehaving peers (default: %u)"), DEFAULT_BANSCORE_THRESHOLD)); diff --git a/src/main.cpp b/src/main.cpp index f85ac3317..adad65640 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -66,6 +66,7 @@ int nScriptCheckThreads = 0; bool fImporting = false; bool fReindex = false; bool fTxIndex = false; +bool fAddressIndex = false; bool fHavePruned = false; bool fPruneMode = false; bool fIsBareMultisigStd = DEFAULT_PERMIT_BAREMULTISIG; @@ -1438,6 +1439,17 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa return res; } +bool GetAddressIndex(uint160 addressHash, int type, std::vector > &addressIndex) +{ + if (!fAddressIndex) + return error("%s: address index not enabled"); + + if (!pblocktree->ReadAddressIndex(addressHash, type, addressIndex)) + return error("%s: unable to get txids for address"); + + return true; +} + /** Return transaction in tx, and if it was found inside a block, its hash is placed in hashBlock */ bool GetTransaction(const uint256 &hash, CTransaction &txOut, const Consensus::Params& consensusParams, uint256 &hashBlock, bool fAllowSlow) { @@ -2322,9 +2334,12 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin std::vector > vPos; vPos.reserve(block.vtx.size()); blockundo.vtxundo.reserve(block.vtx.size() - 1); + std::vector > addressIndex; + for (unsigned int i = 0; i < block.vtx.size(); i++) { const CTransaction &tx = block.vtx[i]; + const uint256 txhash = tx.GetHash(); nInputs += tx.vin.size(); nSigOps += GetLegacySigOpCount(tx); @@ -2351,6 +2366,22 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin REJECT_INVALID, "bad-txns-nonfinal"); } + if (fAddressIndex) + { + for (size_t j = 0; j < tx.vin.size(); j++) { + const CTxOut &prevout = view.GetOutputFor(tx.vin[j]); + if (prevout.scriptPubKey.IsPayToScriptHash()) { + vector hashBytes(prevout.scriptPubKey.begin()+2, prevout.scriptPubKey.begin()+22); + addressIndex.push_back(make_pair(CAddressIndexKey(uint160(hashBytes), 2, txhash, j), prevout.nValue * -1)); + } else if (prevout.scriptPubKey.IsPayToPublicKeyHash()) { + vector hashBytes(prevout.scriptPubKey.begin()+3, prevout.scriptPubKey.begin()+23); + addressIndex.push_back(make_pair(CAddressIndexKey(uint160(hashBytes), 1, txhash, j), prevout.nValue * -1)); + } else { + continue; + } + } + } + if (fStrictPayToScriptHash) { // Add in sigops done by pay-to-script-hash inputs; @@ -2372,6 +2403,24 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin control.Add(vChecks); } + if (fAddressIndex) { + for (unsigned int k = 0; k < tx.vout.size(); k++) { + const CTxOut &out = tx.vout[k]; + + if (out.scriptPubKey.IsPayToScriptHash()) { + vector hashBytes(out.scriptPubKey.begin()+2, out.scriptPubKey.begin()+22); + addressIndex.push_back(make_pair(CAddressIndexKey(uint160(hashBytes), 2, txhash, k), out.nValue)); + } else if (out.scriptPubKey.IsPayToPublicKeyHash()) { + vector hashBytes(out.scriptPubKey.begin()+3, out.scriptPubKey.begin()+23); + addressIndex.push_back(make_pair(CAddressIndexKey(uint160(hashBytes), 1, txhash, k), out.nValue)); + } else { + continue; + } + + } + } + + CTxUndo undoDummy; if (i > 0) { blockundo.vtxundo.push_back(CTxUndo()); @@ -2422,6 +2471,10 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin if (!pblocktree->WriteTxIndex(vPos)) return AbortNode(state, "Failed to write transaction index"); + if (fAddressIndex) + if (!pblocktree->WriteAddressIndex(addressIndex)) + return AbortNode(state, "Failed to write address index"); + // add this block to the view's block chain view.SetBestBlock(pindex->GetBlockHash()); @@ -3813,6 +3866,10 @@ bool static LoadBlockIndexDB() pblocktree->ReadFlag("txindex", fTxIndex); LogPrintf("%s: transaction index %s\n", __func__, fTxIndex ? "enabled" : "disabled"); + // Check whether we have an address index + pblocktree->ReadFlag("addressindex", fAddressIndex); + LogPrintf("%s: address index %s\n", __func__, fAddressIndex ? "enabled" : "disabled"); + // Load pointer to end of best chain BlockMap::iterator it = mapBlockIndex.find(pcoinsTip->GetBestBlock()); if (it == mapBlockIndex.end()) @@ -3973,6 +4030,11 @@ bool InitBlockIndex(const CChainParams& chainparams) // Use the provided setting for -txindex in the new database fTxIndex = GetBoolArg("-txindex", DEFAULT_TXINDEX); pblocktree->WriteFlag("txindex", fTxIndex); + + // Use the provided setting for -addressindex in the new database + fAddressIndex = GetBoolArg("-addressindex", DEFAULT_ADDRESSINDEX); + pblocktree->WriteFlag("addressindex", fAddressIndex); + LogPrintf("Initializing databases...\n"); // Only add the genesis block if not reindexing (in which case we reuse the one already on disk) diff --git a/src/main.h b/src/main.h index 1a696dcd9..159099036 100644 --- a/src/main.h +++ b/src/main.h @@ -111,6 +111,7 @@ static const bool DEFAULT_PERMIT_BAREMULTISIG = true; static const unsigned int DEFAULT_BYTES_PER_SIGOP = 20; static const bool DEFAULT_CHECKPOINTS_ENABLED = true; static const bool DEFAULT_TXINDEX = false; +static const bool DEFAULT_ADDRESSINDEX = false; static const unsigned int DEFAULT_BANSCORE_THRESHOLD = 100; static const bool DEFAULT_TESTSAFEMODE = false; @@ -290,6 +291,42 @@ struct CNodeStateStats { std::vector vHeightInFlight; }; +struct CAddressIndexKey { + uint160 hashBytes; + unsigned int type; + uint256 txhash; + size_t index; + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + READWRITE(hashBytes); + READWRITE(type); + READWRITE(txhash); + READWRITE(index); + } + + CAddressIndexKey(uint160 addressHash, unsigned int addressType, uint256 txid, size_t txindex) { + hashBytes = addressHash; + type = addressType; + txhash = txid; + index = txindex; + } + + CAddressIndexKey() { + SetNull(); + } + + void SetNull() { + hashBytes.SetNull(); + type = 0; + txhash.SetNull(); + index = 0; + } + +}; + struct CDiskTxPos : public CDiskBlockPos { unsigned int nTxOffset; // after header @@ -420,6 +457,7 @@ public: ScriptError GetScriptError() const { return error; } }; +bool GetAddressIndex(uint160 addressHash, int type, std::vector > &addressIndex); /** Functions for disk access for blocks */ bool WriteBlockToDisk(const CBlock& block, CDiskBlockPos& pos, const CMessageHeader::MessageStartChars& messageStart); diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index 9871c3fcc..aa762af88 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -396,3 +396,39 @@ UniValue setmocktime(const UniValue& params, bool fHelp) return NullUniValue; } + +UniValue getaddresstxids(const UniValue& params, bool fHelp) +{ + if (fHelp || params.size() != 1) + throw runtime_error( + "getaddresstxids\n" + "\nReturns the txids for an address (requires addressindex to be enabled).\n" + "\nResult\n" + "[\n" + " \"transactionid\" (string) The transaction id\n" + " ,...\n" + "]\n" + ); + + CBitcoinAddress address(params[0].get_str()); + if (!address.IsValid()) + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); + + CKeyID keyID; + address.GetKeyID(keyID); + + int type = 1; // TODO + std::vector > addressIndex; + + LOCK(cs_main); + + if (!GetAddressIndex(keyID, type, addressIndex)) + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); + + UniValue result(UniValue::VARR); + for (std::vector >::const_iterator it=addressIndex.begin(); it!=addressIndex.end(); it++) + result.push_back(it->first.txhash.GetHex()); + + return result; + +} diff --git a/src/rpcserver.cpp b/src/rpcserver.cpp index b3abeec4a..67da51ceb 100644 --- a/src/rpcserver.cpp +++ b/src/rpcserver.cpp @@ -313,6 +313,9 @@ static const CRPCCommand vRPCCommands[] = { "rawtransactions", "fundrawtransaction", &fundrawtransaction, false }, #endif + /* Address index */ + { "addressindex", "getaddresstxids", &getaddresstxids, false }, + /* Utility functions */ { "util", "createmultisig", &createmultisig, true }, { "util", "validateaddress", &validateaddress, true }, /* uses wallet if enabled */ diff --git a/src/rpcserver.h b/src/rpcserver.h index babf7c8d2..11f760051 100644 --- a/src/rpcserver.h +++ b/src/rpcserver.h @@ -166,6 +166,8 @@ extern std::string HelpExampleRpc(const std::string& methodname, const std::stri extern void EnsureWalletIsUnlocked(); extern UniValue getconnectioncount(const UniValue& params, bool fHelp); // in rpcnet.cpp +extern UniValue getaddresstxids(const UniValue& params, bool fHelp); + extern UniValue getpeerinfo(const UniValue& params, bool fHelp); extern UniValue ping(const UniValue& params, bool fHelp); extern UniValue addnode(const UniValue& params, bool fHelp); diff --git a/src/script/script.cpp b/src/script/script.cpp index 9f2809e59..3e4b72f5d 100644 --- a/src/script/script.cpp +++ b/src/script/script.cpp @@ -201,6 +201,17 @@ unsigned int CScript::GetSigOpCount(const CScript& scriptSig) const return subscript.GetSigOpCount(true); } +bool CScript::IsPayToPublicKeyHash() const +{ + // Extra-fast test for pay-to-pubkey-hash CScripts: + return (this->size() == 25 && + (*this)[0] == OP_DUP && + (*this)[1] == OP_HASH160 && + (*this)[2] == 0x14 && + (*this)[23] == OP_EQUALVERIFY && + (*this)[24] == OP_CHECKSIG); +} + bool CScript::IsPayToScriptHash() const { // Extra-fast test for pay-to-script-hash CScripts: diff --git a/src/script/script.h b/src/script/script.h index d2a68a07b..9dd75a558 100644 --- a/src/script/script.h +++ b/src/script/script.h @@ -608,6 +608,8 @@ public: */ unsigned int GetSigOpCount(const CScript& scriptSig) const; + bool IsPayToPublicKeyHash() const; + bool IsPayToScriptHash() const; /** Called by IsStandardTx and P2SH/BIP62 VerifyScript (which makes it consensus-critical). */ diff --git a/src/txdb.cpp b/src/txdb.cpp index f99e11f26..153d7b84c 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -21,6 +21,7 @@ using namespace std; static const char DB_COINS = 'c'; static const char DB_BLOCK_FILES = 'f'; static const char DB_TXINDEX = 't'; +static const char DB_ADDRESSINDEX = 'a'; static const char DB_BLOCK_INDEX = 'b'; static const char DB_BEST_BLOCK = 'B'; @@ -163,6 +164,38 @@ bool CBlockTreeDB::WriteTxIndex(const std::vector return WriteBatch(batch); } +bool CBlockTreeDB::WriteAddressIndex(const std::vector >&vect) { + CDBBatch batch(&GetObfuscateKey()); + for (std::vector >::const_iterator it=vect.begin(); it!=vect.end(); it++) + batch.Write(make_pair(DB_ADDRESSINDEX, it->first), it->second); + return WriteBatch(batch); +} + +bool CBlockTreeDB::ReadAddressIndex(uint160 addressHash, int type, std::vector > &addressIndex) { + + boost::scoped_ptr pcursor(NewIterator()); + + pcursor->Seek(make_pair(DB_ADDRESSINDEX, addressHash)); //TODO include type + + while (pcursor->Valid()) { + boost::this_thread::interruption_point(); + std::pair key; + if (pcursor->GetKey(key) && key.first == DB_ADDRESSINDEX && key.second.hashBytes == addressHash) { + CAmount nValue; + if (pcursor->GetValue(nValue)) { + addressIndex.push_back(make_pair(key.second, nValue)); + pcursor->Next(); + } else { + return error("failed to get address index value"); + } + } else { + break; + } + } + + return true; +} + bool CBlockTreeDB::WriteFlag(const std::string &name, bool fValue) { return Write(std::make_pair(DB_FLAG, name), fValue ? '1' : '0'); } diff --git a/src/txdb.h b/src/txdb.h index 22e0c5704..3d2ace581 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -17,6 +17,7 @@ class CBlockFileInfo; class CBlockIndex; struct CDiskTxPos; +struct CAddressIndexKey; class uint256; //! -dbcache default (MiB) @@ -57,6 +58,8 @@ public: bool ReadReindexing(bool &fReindex); bool ReadTxIndex(const uint256 &txid, CDiskTxPos &pos); bool WriteTxIndex(const std::vector > &list); + bool WriteAddressIndex(const std::vector > &vect); + bool ReadAddressIndex(uint160 addressHash, int type, std::vector > &addressIndex); bool WriteFlag(const std::string &name, bool fValue); bool ReadFlag(const std::string &name, bool &fValue); bool LoadBlockIndexGuts(); From 73b2d0851b5256c8abfe92763d96b60662bd5ed8 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 8 Mar 2016 14:47:12 -0500 Subject: [PATCH 03/59] test: added unit tests for CScript.IsPayToPublicKeyHash --- src/Makefile.test.include | 1 + src/test/script_P2PKH_tests.cpp | 59 +++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+) create mode 100644 src/test/script_P2PKH_tests.cpp diff --git a/src/Makefile.test.include b/src/Makefile.test.include index e96e7bec3..4ea09116c 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -70,6 +70,7 @@ BITCOIN_TESTS =\ test/sanity_tests.cpp \ test/scheduler_tests.cpp \ test/script_P2SH_tests.cpp \ + test/script_P2PKH_tests.cpp \ test/script_tests.cpp \ test/scriptnum_tests.cpp \ test/serialize_tests.cpp \ diff --git a/src/test/script_P2PKH_tests.cpp b/src/test/script_P2PKH_tests.cpp new file mode 100644 index 000000000..3a7dc1660 --- /dev/null +++ b/src/test/script_P2PKH_tests.cpp @@ -0,0 +1,59 @@ +// Copyright (c) 2012-2015 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include "script/script.h" +#include "test/test_bitcoin.h" + +#include + +using namespace std; + +BOOST_FIXTURE_TEST_SUITE(script_P2PKH_tests, BasicTestingSetup) + +BOOST_AUTO_TEST_CASE(IsPayToPublicKeyHash) +{ + // Test CScript::IsPayToPublicKeyHash() + uint160 dummy; + CScript p2pkh; + p2pkh << OP_DUP << OP_HASH160 << ToByteVector(dummy) << OP_EQUALVERIFY << OP_CHECKSIG; + BOOST_CHECK(p2pkh.IsPayToPublicKeyHash()); + + static const unsigned char direct[] = { + OP_DUP, OP_HASH160, 20, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, OP_EQUALVERIFY, OP_CHECKSIG + }; + BOOST_CHECK(CScript(direct, direct+sizeof(direct)).IsPayToPublicKeyHash()); + + static const unsigned char notp2pkh1[] = { + OP_DUP, OP_HASH160, 20, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, OP_EQUALVERIFY, OP_CHECKSIG, OP_CHECKSIG + }; + BOOST_CHECK(!CScript(notp2pkh1, notp2pkh1+sizeof(notp2pkh1)).IsPayToPublicKeyHash()); + + static const unsigned char p2sh[] = { + OP_HASH160, 20, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, OP_EQUAL + }; + BOOST_CHECK(!CScript(p2sh, p2sh+sizeof(p2sh)).IsPayToPublicKeyHash()); + + static const unsigned char extra[] = { + OP_DUP, OP_HASH160, 20, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, OP_EQUALVERIFY, OP_CHECKSIG, OP_CHECKSIG + }; + BOOST_CHECK(!CScript(extra, extra+sizeof(extra)).IsPayToPublicKeyHash()); + + static const unsigned char missing[] = { + OP_HASH160, 20, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, OP_EQUALVERIFY, OP_CHECKSIG, OP_RETURN + }; + BOOST_CHECK(!CScript(missing, missing+sizeof(missing)).IsPayToPublicKeyHash()); + + static const unsigned char missing2[] = { + OP_DUP, OP_HASH160, 20, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 + }; + BOOST_CHECK(!CScript(missing2, missing2+sizeof(missing)).IsPayToPublicKeyHash()); + + static const unsigned char tooshort[] = { + OP_DUP, OP_HASH160, 2, 0,0, OP_EQUALVERIFY, OP_CHECKSIG + }; + BOOST_CHECK(!CScript(tooshort, tooshort+sizeof(direct)).IsPayToPublicKeyHash()); + +} + +BOOST_AUTO_TEST_SUITE_END() From 4d461956349551c67567791bcb18e58a22df96b6 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 8 Mar 2016 16:15:49 -0500 Subject: [PATCH 04/59] qa: started test for addressindex rpc getaddresstxids --- qa/pull-tester/rpc-tests.py | 1 + qa/rpc-tests/addressindex.py | 54 ++++++++++++++++++++++++++++++++++++ 2 files changed, 55 insertions(+) create mode 100755 qa/rpc-tests/addressindex.py diff --git a/qa/pull-tester/rpc-tests.py b/qa/pull-tester/rpc-tests.py index 37014cf76..804914e29 100755 --- a/qa/pull-tester/rpc-tests.py +++ b/qa/pull-tester/rpc-tests.py @@ -97,6 +97,7 @@ testScripts = [ 'walletbackup.py', 'nodehandling.py', 'reindex.py', + 'addressindex.py', 'decodescript.py', 'p2p-fullblocktest.py', 'blockchain.py', diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py new file mode 100755 index 000000000..c4353d92a --- /dev/null +++ b/qa/rpc-tests/addressindex.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python2 +# Copyright (c) 2014-2015 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +# +# Test addressindex generation and fetching +# + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import * + +class AddressIndexTest(BitcoinTestFramework): + + def setup_chain(self): + print("Initializing test directory "+self.options.tmpdir) + initialize_chain_clean(self.options.tmpdir, 4) + + def setup_network(self): + self.nodes = [] + # Nodes 0/1 are "wallet" nodes + self.nodes.append(start_node(0, self.options.tmpdir, ["-debug"])) + self.nodes.append(start_node(1, self.options.tmpdir, ["-debug", "-addressindex"])) + # Nodes 2/3 are used for testing + self.nodes.append(start_node(2, self.options.tmpdir, ["-debug"])) + self.nodes.append(start_node(3, self.options.tmpdir, ["-debug", "-addressindex"])) + connect_nodes(self.nodes[0], 1) + connect_nodes(self.nodes[0], 2) + connect_nodes(self.nodes[0], 3) + + self.is_network_split = False + self.sync_all() + + def run_test(self): + print "Mining blocks..." + self.nodes[0].generate(105) + self.sync_all() + + chain_height = self.nodes[1].getblockcount() + assert_equal(chain_height, 105) + assert_equal(self.nodes[1].getbalance(), 0) + assert_equal(self.nodes[2].getbalance(), 0) + + txid1 = self.nodes[0].sendtoaddress("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs", 10) + txid2 = self.nodes[0].sendtoaddress("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs", 15) + txid3 = self.nodes[0].sendtoaddress("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs", 20) + self.nodes[0].generate(1) + self.sync_all() + + txids = self.nodes[1].getaddresstxids("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs"); + assert_equal(len(txids), 3); + +if __name__ == '__main__': + AddressIndexTest().main() From 18ea599a71614faf78d973df42ec0468a5bd86f3 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 9 Mar 2016 11:27:30 -0500 Subject: [PATCH 05/59] main: index address index sorted by height --- qa/rpc-tests/addressindex.py | 11 ++++++-- src/main.cpp | 9 +++--- src/main.h | 54 ++++++++++++++++++++++++++++++------ src/txdb.cpp | 4 +-- src/txdb.h | 1 + 5 files changed, 60 insertions(+), 19 deletions(-) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index c4353d92a..f3cbc2ef8 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -41,14 +41,19 @@ class AddressIndexTest(BitcoinTestFramework): assert_equal(self.nodes[1].getbalance(), 0) assert_equal(self.nodes[2].getbalance(), 0) - txid1 = self.nodes[0].sendtoaddress("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs", 10) - txid2 = self.nodes[0].sendtoaddress("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs", 15) - txid3 = self.nodes[0].sendtoaddress("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs", 20) + txid0 = self.nodes[0].sendtoaddress("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs", 10) + self.nodes[0].generate(1) + txid1 = self.nodes[0].sendtoaddress("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs", 15) + self.nodes[0].generate(1) + txid2 = self.nodes[0].sendtoaddress("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs", 20) self.nodes[0].generate(1) self.sync_all() txids = self.nodes[1].getaddresstxids("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs"); assert_equal(len(txids), 3); + assert_equal(txids[0], txid0); + assert_equal(txids[1], txid1); + assert_equal(txids[2], txid2); if __name__ == '__main__': AddressIndexTest().main() diff --git a/src/main.cpp b/src/main.cpp index adad65640..0ad52abbb 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -2372,10 +2372,10 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin const CTxOut &prevout = view.GetOutputFor(tx.vin[j]); if (prevout.scriptPubKey.IsPayToScriptHash()) { vector hashBytes(prevout.scriptPubKey.begin()+2, prevout.scriptPubKey.begin()+22); - addressIndex.push_back(make_pair(CAddressIndexKey(uint160(hashBytes), 2, txhash, j), prevout.nValue * -1)); + addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, j), prevout.nValue * -1)); } else if (prevout.scriptPubKey.IsPayToPublicKeyHash()) { vector hashBytes(prevout.scriptPubKey.begin()+3, prevout.scriptPubKey.begin()+23); - addressIndex.push_back(make_pair(CAddressIndexKey(uint160(hashBytes), 1, txhash, j), prevout.nValue * -1)); + addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, j), prevout.nValue * -1)); } else { continue; } @@ -2409,10 +2409,10 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin if (out.scriptPubKey.IsPayToScriptHash()) { vector hashBytes(out.scriptPubKey.begin()+2, out.scriptPubKey.begin()+22); - addressIndex.push_back(make_pair(CAddressIndexKey(uint160(hashBytes), 2, txhash, k), out.nValue)); + addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, k), out.nValue)); } else if (out.scriptPubKey.IsPayToPublicKeyHash()) { vector hashBytes(out.scriptPubKey.begin()+3, out.scriptPubKey.begin()+23); - addressIndex.push_back(make_pair(CAddressIndexKey(uint160(hashBytes), 1, txhash, k), out.nValue)); + addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, k), out.nValue)); } else { continue; } @@ -2420,7 +2420,6 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin } } - CTxUndo undoDummy; if (i > 0) { blockundo.vtxundo.push_back(CTxUndo()); diff --git a/src/main.h b/src/main.h index 159099036..c2196fcbd 100644 --- a/src/main.h +++ b/src/main.h @@ -292,26 +292,33 @@ struct CNodeStateStats { }; struct CAddressIndexKey { - uint160 hashBytes; unsigned int type; + uint160 hashBytes; + int blockHeight; + unsigned int txindex; uint256 txhash; - size_t index; + size_t outindex; ADD_SERIALIZE_METHODS; template inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { - READWRITE(hashBytes); READWRITE(type); + READWRITE(hashBytes); + READWRITE(blockHeight); + READWRITE(txindex); READWRITE(txhash); - READWRITE(index); + READWRITE(outindex); } - CAddressIndexKey(uint160 addressHash, unsigned int addressType, uint256 txid, size_t txindex) { - hashBytes = addressHash; + CAddressIndexKey(unsigned int addressType, uint160 addressHash, int height, int blockindex, + uint256 txid, size_t outputIndex) { type = addressType; + hashBytes = addressHash; + blockHeight = height; + txindex = blockindex; txhash = txid; - index = txindex; + outindex = outputIndex; } CAddressIndexKey() { @@ -319,14 +326,43 @@ struct CAddressIndexKey { } void SetNull() { - hashBytes.SetNull(); type = 0; + hashBytes.SetNull(); + blockHeight = 0; + txindex = 0; txhash.SetNull(); - index = 0; + outindex = 0; } }; +struct CAddressIndexIteratorKey { + unsigned int type; + uint160 hashBytes; + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + READWRITE(type); + READWRITE(hashBytes); + } + + CAddressIndexIteratorKey(unsigned int addressType, uint160 addressHash) { + type = addressType; + hashBytes = addressHash; + } + + CAddressIndexIteratorKey() { + SetNull(); + } + + void SetNull() { + type = 0; + hashBytes.SetNull(); + } +}; + struct CDiskTxPos : public CDiskBlockPos { unsigned int nTxOffset; // after header diff --git a/src/txdb.cpp b/src/txdb.cpp index 153d7b84c..e93594b9e 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -167,7 +167,7 @@ bool CBlockTreeDB::WriteTxIndex(const std::vector bool CBlockTreeDB::WriteAddressIndex(const std::vector >&vect) { CDBBatch batch(&GetObfuscateKey()); for (std::vector >::const_iterator it=vect.begin(); it!=vect.end(); it++) - batch.Write(make_pair(DB_ADDRESSINDEX, it->first), it->second); + batch.Write(make_pair(DB_ADDRESSINDEX, it->first), it->second); return WriteBatch(batch); } @@ -175,7 +175,7 @@ bool CBlockTreeDB::ReadAddressIndex(uint160 addressHash, int type, std::vector pcursor(NewIterator()); - pcursor->Seek(make_pair(DB_ADDRESSINDEX, addressHash)); //TODO include type + pcursor->Seek(make_pair(DB_ADDRESSINDEX, CAddressIndexIteratorKey(type, addressHash))); while (pcursor->Valid()) { boost::this_thread::interruption_point(); diff --git a/src/txdb.h b/src/txdb.h index 3d2ace581..ca32869cc 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -18,6 +18,7 @@ class CBlockFileInfo; class CBlockIndex; struct CDiskTxPos; struct CAddressIndexKey; +struct CAddressIndexIteratorKey; class uint256; //! -dbcache default (MiB) From fcac6bcdc82008376fae530a07ae002e64b44c40 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 9 Mar 2016 17:40:40 -0500 Subject: [PATCH 06/59] rpc: fix issue for querying txids for p2sh addresses --- qa/rpc-tests/addressindex.py | 13 +++++++++++++ src/base58.cpp | 17 +++++++++++++++++ src/base58.h | 1 + src/rpcmisc.cpp | 11 +++++------ 4 files changed, 36 insertions(+), 6 deletions(-) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index f3cbc2ef8..88533d028 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -42,11 +42,17 @@ class AddressIndexTest(BitcoinTestFramework): assert_equal(self.nodes[2].getbalance(), 0) txid0 = self.nodes[0].sendtoaddress("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs", 10) + txidb0 = self.nodes[0].sendtoaddress("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br", 10) self.nodes[0].generate(1) + txid1 = self.nodes[0].sendtoaddress("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs", 15) + txidb1 = self.nodes[0].sendtoaddress("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br", 15) self.nodes[0].generate(1) + txid2 = self.nodes[0].sendtoaddress("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs", 20) + txidb2 = self.nodes[0].sendtoaddress("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br", 20) self.nodes[0].generate(1) + self.sync_all() txids = self.nodes[1].getaddresstxids("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs"); @@ -55,5 +61,12 @@ class AddressIndexTest(BitcoinTestFramework): assert_equal(txids[1], txid1); assert_equal(txids[2], txid2); + txidsb = self.nodes[1].getaddresstxids("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br"); + assert_equal(len(txidsb), 3); + assert_equal(txidsb[0], txidb0); + assert_equal(txidsb[1], txidb1); + assert_equal(txidsb[2], txidb2); + + if __name__ == '__main__': AddressIndexTest().main() diff --git a/src/base58.cpp b/src/base58.cpp index 5e26cf8d4..80fa99c2a 100644 --- a/src/base58.cpp +++ b/src/base58.cpp @@ -262,6 +262,23 @@ CTxDestination CBitcoinAddress::Get() const return CNoDestination(); } +bool CBitcoinAddress::GetIndexKey(uint160& hashBytes, int& type) const +{ + if (!IsValid()) { + return false; + } else if (vchVersion == Params().Base58Prefix(CChainParams::PUBKEY_ADDRESS)) { + memcpy(&hashBytes, &vchData[0], 20); + type = 1; + return true; + } else if (vchVersion == Params().Base58Prefix(CChainParams::SCRIPT_ADDRESS)) { + memcpy(&hashBytes, &vchData[0], 20); + type = 2; + return true; + } + + return false; +} + bool CBitcoinAddress::GetKeyID(CKeyID& keyID) const { if (!IsValid() || vchVersion != Params().Base58Prefix(CChainParams::PUBKEY_ADDRESS)) diff --git a/src/base58.h b/src/base58.h index a3980118a..b8280b1dd 100644 --- a/src/base58.h +++ b/src/base58.h @@ -116,6 +116,7 @@ public: CTxDestination Get() const; bool GetKeyID(CKeyID &keyID) const; + bool GetIndexKey(uint160& hashBytes, int& type) const; bool IsScript() const; }; diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index aa762af88..0a39d2fbc 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -411,18 +411,17 @@ UniValue getaddresstxids(const UniValue& params, bool fHelp) ); CBitcoinAddress address(params[0].get_str()); - if (!address.IsValid()) + uint160 hashBytes; + int type = 0; + if (!address.GetIndexKey(hashBytes, type)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); + } - CKeyID keyID; - address.GetKeyID(keyID); - - int type = 1; // TODO std::vector > addressIndex; LOCK(cs_main); - if (!GetAddressIndex(keyID, type, addressIndex)) + if (!GetAddressIndex(hashBytes, type, addressIndex)) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); UniValue result(UniValue::VARR); From 2500d1d1159b2021e2bdcdde3c1e7d4f04917d90 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 9 Mar 2016 19:45:08 -0500 Subject: [PATCH 07/59] rpc: update getaddresstxids for uniqueness --- qa/rpc-tests/addressindex.py | 28 ++++++++++++++++++++++++++++ src/rpcmisc.cpp | 10 ++++++++-- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index 88533d028..fd750403f 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -9,6 +9,9 @@ from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * +from test_framework.script import * +from test_framework.mininode import * +import binascii class AddressIndexTest(BitcoinTestFramework): @@ -41,6 +44,9 @@ class AddressIndexTest(BitcoinTestFramework): assert_equal(self.nodes[1].getbalance(), 0) assert_equal(self.nodes[2].getbalance(), 0) + # Check p2pkh and p2sh address indexes + print "Testing p2pkh and p2sh address index..." + txid0 = self.nodes[0].sendtoaddress("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs", 10) txidb0 = self.nodes[0].sendtoaddress("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br", 10) self.nodes[0].generate(1) @@ -67,6 +73,28 @@ class AddressIndexTest(BitcoinTestFramework): assert_equal(txidsb[1], txidb1); assert_equal(txidsb[2], txidb2); + # Check that outputs with the same address will only return one txid + print "Testing for txid uniqueness..." + addressHash = "6349a418fc4578d10a372b54b45c280cc8c4382f".decode("hex") + scriptPubKey = CScript([OP_HASH160, addressHash, OP_EQUAL]) + unspent = self.nodes[0].listunspent() + tx = CTransaction() + tx.vin = [CTxIn(COutPoint(int(unspent[0]["txid"], 16), unspent[0]["vout"]))] + tx.vout = [CTxOut(10, scriptPubKey), CTxOut(11, scriptPubKey)] + tx.rehash() + + signed_tx = self.nodes[0].signrawtransaction(binascii.hexlify(tx.serialize()).decode('utf-8')) + sent_txid = self.nodes[0].sendrawtransaction(signed_tx['hex'], True) + + self.nodes[0].generate(1) + self.sync_all() + + txidsmany = self.nodes[1].getaddresstxids("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br"); + assert_equal(len(txidsmany), 4); + assert_equal(txidsmany[3], sent_txid); + + print "Passed\n" + if __name__ == '__main__': AddressIndexTest().main() diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index 0a39d2fbc..cca8a9896 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -424,9 +424,15 @@ UniValue getaddresstxids(const UniValue& params, bool fHelp) if (!GetAddressIndex(hashBytes, type, addressIndex)) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); + std::set txids; + UniValue result(UniValue::VARR); - for (std::vector >::const_iterator it=addressIndex.begin(); it!=addressIndex.end(); it++) - result.push_back(it->first.txhash.GetHex()); + for (std::vector >::const_iterator it=addressIndex.begin(); it!=addressIndex.end(); it++) { + std::string txid = it->first.txhash.GetHex(); + if (txids.insert(txid).second) { + result.push_back(txid); + } + } return result; From f4d11ffc7c3fd875ea8d464e25c7c23494c6e7e9 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 15 Mar 2016 15:25:01 -0400 Subject: [PATCH 08/59] rpc: query for multiple addresses txids --- qa/rpc-tests/addressindex.py | 4 ++++ src/rpcmisc.cpp | 46 +++++++++++++++++++++++++++++------- 2 files changed, 42 insertions(+), 8 deletions(-) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index fd750403f..a985735f8 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -73,6 +73,10 @@ class AddressIndexTest(BitcoinTestFramework): assert_equal(txidsb[1], txidb1); assert_equal(txidsb[2], txidb2); + # Check that multiple addresses works + multitxids = self.nodes[1].getaddresstxids({"addresses": ["2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br", "mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs"]}); + assert_equal(len(multitxids), 6); + # Check that outputs with the same address will only return one txid print "Testing for txid uniqueness..." addressHash = "6349a418fc4578d10a372b54b45c280cc8c4382f".decode("hex") diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index cca8a9896..a7bf7aad2 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -410,20 +410,50 @@ UniValue getaddresstxids(const UniValue& params, bool fHelp) "]\n" ); - CBitcoinAddress address(params[0].get_str()); - uint160 hashBytes; - int type = 0; - if (!address.GetIndexKey(hashBytes, type)) { + std::vector > addresses; + + if (params[0].isStr()) { + + CBitcoinAddress address(params[0].get_str()); + uint160 hashBytes; + int type = 0; + if (!address.GetIndexKey(hashBytes, type)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); + } + addresses.push_back(std::make_pair(hashBytes, type)); + + } else if (params[0].isObject()) { + + UniValue addressValues = find_value(params[0].get_obj(), "addresses"); + if (!addressValues.isArray()) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Addresses is expected to be an array"); + } + + std::vector values = addressValues.getValues(); + + for (std::vector::iterator it = values.begin(); it != values.end(); ++it) { + + CBitcoinAddress address(it->get_str()); + uint160 hashBytes; + int type = 0; + if (!address.GetIndexKey(hashBytes, type)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); + } + addresses.push_back(std::make_pair(hashBytes, type)); + } + } else { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); } std::vector > addressIndex; - LOCK(cs_main); - - if (!GetAddressIndex(hashBytes, type, addressIndex)) - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); + for (std::vector >::iterator it = addresses.begin(); it != addresses.end(); ++it) { + if (!GetAddressIndex((*it).first, (*it).second, addressIndex)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); + } + } + // TODO sort by height std::set txids; UniValue result(UniValue::VARR); From 5b5f3f7d00465200315a06cf40f4ce31a01d0d50 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 15 Mar 2016 16:24:55 -0400 Subject: [PATCH 09/59] rpc: sort txids by height for multiple addresses --- qa/rpc-tests/addressindex.py | 12 ++++++++++++ src/rpcmisc.cpp | 13 ++++++++++--- 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index a985735f8..2e15348d6 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -48,14 +48,20 @@ class AddressIndexTest(BitcoinTestFramework): print "Testing p2pkh and p2sh address index..." txid0 = self.nodes[0].sendtoaddress("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs", 10) + self.nodes[0].generate(1) + txidb0 = self.nodes[0].sendtoaddress("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br", 10) self.nodes[0].generate(1) txid1 = self.nodes[0].sendtoaddress("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs", 15) + self.nodes[0].generate(1) + txidb1 = self.nodes[0].sendtoaddress("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br", 15) self.nodes[0].generate(1) txid2 = self.nodes[0].sendtoaddress("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs", 20) + self.nodes[0].generate(1) + txidb2 = self.nodes[0].sendtoaddress("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br", 20) self.nodes[0].generate(1) @@ -76,6 +82,12 @@ class AddressIndexTest(BitcoinTestFramework): # Check that multiple addresses works multitxids = self.nodes[1].getaddresstxids({"addresses": ["2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br", "mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs"]}); assert_equal(len(multitxids), 6); + assert_equal(multitxids[0], txid0); + assert_equal(multitxids[1], txidb0); + assert_equal(multitxids[2], txid1); + assert_equal(multitxids[3], txidb1); + assert_equal(multitxids[4], txid2); + assert_equal(multitxids[5], txidb2); # Check that outputs with the same address will only return one txid print "Testing for txid uniqueness..." diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index a7bf7aad2..e04b77d75 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -453,17 +453,24 @@ UniValue getaddresstxids(const UniValue& params, bool fHelp) } } - // TODO sort by height std::set txids; + std::vector > vtxids; - UniValue result(UniValue::VARR); for (std::vector >::const_iterator it=addressIndex.begin(); it!=addressIndex.end(); it++) { + int height = it->first.blockHeight; std::string txid = it->first.txhash.GetHex(); if (txids.insert(txid).second) { - result.push_back(txid); + vtxids.push_back(std::make_pair(height, txid)); } } + std::sort(vtxids.begin(), vtxids.end()); + + UniValue result(UniValue::VARR); + for (std::vector >::const_iterator it=vtxids.begin(); it!=vtxids.end(); it++) { + result.push_back(it->second); + } + return result; } From 7959a190850db293fa3faf6251e270738581a764 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 16 Mar 2016 14:00:50 -0400 Subject: [PATCH 10/59] main: serialize height in BE for address index key fixes a sorting issue when iterating over keys --- src/main.h | 49 +++++++++++++++++++++++++++++++++---------------- src/serialize.h | 11 +++++++++++ 2 files changed, 44 insertions(+), 16 deletions(-) diff --git a/src/main.h b/src/main.h index c2196fcbd..137915253 100644 --- a/src/main.h +++ b/src/main.h @@ -299,16 +299,27 @@ struct CAddressIndexKey { uint256 txhash; size_t outindex; - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { - READWRITE(type); - READWRITE(hashBytes); - READWRITE(blockHeight); - READWRITE(txindex); - READWRITE(txhash); - READWRITE(outindex); + size_t GetSerializeSize(int nType, int nVersion) const { + return 65; + } + template + void Serialize(Stream& s, int nType, int nVersion) const { + ser_writedata8(s, type); + hashBytes.Serialize(s, nType, nVersion); + // Heights are stored big-endian for key sorting in LevelDB + ser_writedata32be(s, blockHeight); + ser_writedata32be(s, txindex); + txhash.Serialize(s, nType, nVersion); + ser_writedata32(s, outindex); + } + template + void Unserialize(Stream& s, int nType, int nVersion) { + type = ser_readdata8(s); + hashBytes.Unserialize(s, nType, nVersion); + blockHeight = ser_readdata32be(s); + txindex = ser_readdata32be(s); + txhash.Unserialize(s, nType, nVersion); + outindex = ser_readdata32(s); } CAddressIndexKey(unsigned int addressType, uint160 addressHash, int height, int blockindex, @@ -340,12 +351,18 @@ struct CAddressIndexIteratorKey { unsigned int type; uint160 hashBytes; - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { - READWRITE(type); - READWRITE(hashBytes); + size_t GetSerializeSize(int nType, int nVersion) const { + return 21; + } + template + void Serialize(Stream& s, int nType, int nVersion) const { + ser_writedata8(s, type); + hashBytes.Serialize(s, nType, nVersion); + } + template + void Unserialize(Stream& s, int nType, int nVersion) { + type = ser_readdata8(s); + hashBytes.Unserialize(s, nType, nVersion); } CAddressIndexIteratorKey(unsigned int addressType, uint160 addressHash) { diff --git a/src/serialize.h b/src/serialize.h index 5c2db9d33..41a51d237 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -91,6 +91,11 @@ template inline void ser_writedata32(Stream &s, uint32_t obj) obj = htole32(obj); s.write((char*)&obj, 4); } +template inline void ser_writedata32be(Stream &s, uint32_t obj) +{ + obj = htobe32(obj); + s.write((char*)&obj, 4); +} template inline void ser_writedata64(Stream &s, uint64_t obj) { obj = htole64(obj); @@ -114,6 +119,12 @@ template inline uint32_t ser_readdata32(Stream &s) s.read((char*)&obj, 4); return le32toh(obj); } +template inline uint32_t ser_readdata32be(Stream &s) +{ + uint32_t obj; + s.read((char*)&obj, 4); + return be32toh(obj); +} template inline uint64_t ser_readdata64(Stream &s) { uint64_t obj; From 7dbbb79cecb5f9f06c9406eaa680e047f7024af6 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 16 Mar 2016 14:50:19 -0400 Subject: [PATCH 11/59] rpc: only sort when combining multiple results It's only necessary to sort when combining results for several addresses as the results are already in order from the database. --- src/rpcmisc.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index e04b77d75..1a86353e2 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -464,7 +464,9 @@ UniValue getaddresstxids(const UniValue& params, bool fHelp) } } - std::sort(vtxids.begin(), vtxids.end()); + if (addresses.size() > 1) { + std::sort(vtxids.begin(), vtxids.end()); + } UniValue result(UniValue::VARR); for (std::vector >::const_iterator it=vtxids.begin(); it!=vtxids.end(); it++) { From 5bb6d69ff849732f5695e221a06d50547a08db0a Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 17 Mar 2016 16:06:08 -0400 Subject: [PATCH 12/59] rpc: added getaddressbalance method using addressindex --- qa/rpc-tests/addressindex.py | 12 ++++++ src/rpcmisc.cpp | 83 ++++++++++++++++++++++++++++-------- src/rpcserver.cpp | 1 + src/rpcserver.h | 1 + 4 files changed, 80 insertions(+), 17 deletions(-) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index 2e15348d6..8f123eaa0 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -44,6 +44,10 @@ class AddressIndexTest(BitcoinTestFramework): assert_equal(self.nodes[1].getbalance(), 0) assert_equal(self.nodes[2].getbalance(), 0) + # Check that balances are correct + balance0 = self.nodes[1].getaddressbalance("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br") + assert_equal(balance0['balance'], 0); + # Check p2pkh and p2sh address indexes print "Testing p2pkh and p2sh address index..." @@ -89,6 +93,10 @@ class AddressIndexTest(BitcoinTestFramework): assert_equal(multitxids[4], txid2); assert_equal(multitxids[5], txidb2); + # Check that balances are correct + balance0 = self.nodes[1].getaddressbalance("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br") + assert_equal(balance0['balance'], 45 * 100000000); + # Check that outputs with the same address will only return one txid print "Testing for txid uniqueness..." addressHash = "6349a418fc4578d10a372b54b45c280cc8c4382f".decode("hex") @@ -109,6 +117,10 @@ class AddressIndexTest(BitcoinTestFramework): assert_equal(len(txidsmany), 4); assert_equal(txidsmany[3], sent_txid); + # Check that balances are correct + balance0 = self.nodes[1].getaddressbalance("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br") + assert_equal(balance0['balance'], 45 * 100000000 + 21); + print "Passed\n" diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index 1a86353e2..d4ad29e65 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -397,23 +397,9 @@ UniValue setmocktime(const UniValue& params, bool fHelp) return NullUniValue; } -UniValue getaddresstxids(const UniValue& params, bool fHelp) +bool getAddressesFromParams(const UniValue& params, std::vector > &addresses) { - if (fHelp || params.size() != 1) - throw runtime_error( - "getaddresstxids\n" - "\nReturns the txids for an address (requires addressindex to be enabled).\n" - "\nResult\n" - "[\n" - " \"transactionid\" (string) The transaction id\n" - " ,...\n" - "]\n" - ); - - std::vector > addresses; - if (params[0].isStr()) { - CBitcoinAddress address(params[0].get_str()); uint160 hashBytes; int type = 0; @@ -421,7 +407,6 @@ UniValue getaddresstxids(const UniValue& params, bool fHelp) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); } addresses.push_back(std::make_pair(hashBytes, type)); - } else if (params[0].isObject()) { UniValue addressValues = find_value(params[0].get_obj(), "addresses"); @@ -445,9 +430,73 @@ UniValue getaddresstxids(const UniValue& params, bool fHelp) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); } + return true; + + +} + +UniValue getaddressbalance(const UniValue& params, bool fHelp) +{ + if (fHelp || params.size() != 1) + throw runtime_error( + "getaddressbalance\n" + "\nReturns the balance for an address (requires addressindex to be enabled).\n" + "\nResult\n" + "{\n" + " \"balance\" (string) The current balance\n" + " ,...\n" + "}\n" + ); + + std::vector > addresses; + + if (!getAddressesFromParams(params, addresses)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); + } + std::vector > addressIndex; - for (std::vector >::iterator it = addresses.begin(); it != addresses.end(); ++it) { + for (std::vector >::iterator it = addresses.begin(); it != addresses.end(); it++) { + if (!GetAddressIndex((*it).first, (*it).second, addressIndex)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); + } + } + + CAmount balance = 0; + + for (std::vector >::const_iterator it=addressIndex.begin(); it!=addressIndex.end(); it++) { + balance += it->second; + } + + UniValue result(UniValue::VOBJ); + result.push_back(Pair("balance", balance)); + + return result; + +} + +UniValue getaddresstxids(const UniValue& params, bool fHelp) +{ + if (fHelp || params.size() != 1) + throw runtime_error( + "getaddresstxids\n" + "\nReturns the txids for an address (requires addressindex to be enabled).\n" + "\nResult\n" + "[\n" + " \"transactionid\" (string) The transaction id\n" + " ,...\n" + "]\n" + ); + + std::vector > addresses; + + if (!getAddressesFromParams(params, addresses)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); + } + + std::vector > addressIndex; + + for (std::vector >::iterator it = addresses.begin(); it != addresses.end(); it++) { if (!GetAddressIndex((*it).first, (*it).second, addressIndex)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); } diff --git a/src/rpcserver.cpp b/src/rpcserver.cpp index 67da51ceb..da66b1c0a 100644 --- a/src/rpcserver.cpp +++ b/src/rpcserver.cpp @@ -315,6 +315,7 @@ static const CRPCCommand vRPCCommands[] = /* Address index */ { "addressindex", "getaddresstxids", &getaddresstxids, false }, + { "addressindex", "getaddressbalance", &getaddressbalance, false }, /* Utility functions */ { "util", "createmultisig", &createmultisig, true }, diff --git a/src/rpcserver.h b/src/rpcserver.h index 11f760051..8a80fc5ee 100644 --- a/src/rpcserver.h +++ b/src/rpcserver.h @@ -167,6 +167,7 @@ extern void EnsureWalletIsUnlocked(); extern UniValue getconnectioncount(const UniValue& params, bool fHelp); // in rpcnet.cpp extern UniValue getaddresstxids(const UniValue& params, bool fHelp); +extern UniValue getaddressbalance(const UniValue& params, bool fHelp); extern UniValue getpeerinfo(const UniValue& params, bool fHelp); extern UniValue ping(const UniValue& params, bool fHelp); From 2e8a4c00fa03f8efe2b59ef9f32e9683f1f6472e Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 17 Mar 2016 16:40:16 -0400 Subject: [PATCH 13/59] rpc: add receieved to balance --- src/rpcmisc.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index d4ad29e65..549e6c7a5 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -463,13 +463,18 @@ UniValue getaddressbalance(const UniValue& params, bool fHelp) } CAmount balance = 0; + CAmount received = 0; for (std::vector >::const_iterator it=addressIndex.begin(); it!=addressIndex.end(); it++) { + if (it->second > 0) { + received += it->second; + } balance += it->second; } UniValue result(UniValue::VOBJ); result.push_back(Pair("balance", balance)); + result.push_back(Pair("received", received)); return result; From 24deb4efc146848269c4fecb2ae6db62930307f2 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 22 Mar 2016 13:55:19 -0400 Subject: [PATCH 14/59] rpc: include height in getrawtransaction results --- src/rpcrawtransaction.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/rpcrawtransaction.cpp b/src/rpcrawtransaction.cpp index bd51aa0ab..6ab1807d4 100644 --- a/src/rpcrawtransaction.cpp +++ b/src/rpcrawtransaction.cpp @@ -101,12 +101,14 @@ void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry) if (mi != mapBlockIndex.end() && (*mi).second) { CBlockIndex* pindex = (*mi).second; if (chainActive.Contains(pindex)) { + entry.push_back(Pair("height", pindex->nHeight)); entry.push_back(Pair("confirmations", 1 + chainActive.Height() - pindex->nHeight)); entry.push_back(Pair("time", pindex->GetBlockTime())); entry.push_back(Pair("blocktime", pindex->GetBlockTime())); - } - else + } else { + entry.push_back(Pair("height", -1)); entry.push_back(Pair("confirmations", 0)); + } } } } From 935ca8f832cd3b7bddf762ce236225161ea6f1aa Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 22 Mar 2016 18:11:04 -0400 Subject: [PATCH 15/59] main: add block timestamp index --- qa/pull-tester/rpc-tests.py | 1 + qa/rpc-tests/timestampindex.py | 51 +++++++++++++++++++++++++++ src/main.cpp | 28 +++++++++++++-- src/main.h | 63 ++++++++++++++++++++++++++++++++++ src/rpcblockchain.cpp | 32 +++++++++++++++++ src/rpcserver.cpp | 1 + src/rpcserver.h | 1 + src/txdb.cpp | 27 +++++++++++++++ src/txdb.h | 4 +++ 9 files changed, 206 insertions(+), 2 deletions(-) create mode 100755 qa/rpc-tests/timestampindex.py diff --git a/qa/pull-tester/rpc-tests.py b/qa/pull-tester/rpc-tests.py index 804914e29..73bfe1e28 100755 --- a/qa/pull-tester/rpc-tests.py +++ b/qa/pull-tester/rpc-tests.py @@ -98,6 +98,7 @@ testScripts = [ 'nodehandling.py', 'reindex.py', 'addressindex.py', + 'timestampindex.py', 'decodescript.py', 'p2p-fullblocktest.py', 'blockchain.py', diff --git a/qa/rpc-tests/timestampindex.py b/qa/rpc-tests/timestampindex.py new file mode 100755 index 000000000..46ff710bd --- /dev/null +++ b/qa/rpc-tests/timestampindex.py @@ -0,0 +1,51 @@ +#!/usr/bin/env python2 +# Copyright (c) 2014-2015 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +# +# Test timestampindex generation and fetching +# + +import time + +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import * + + +class TimestampIndexTest(BitcoinTestFramework): + + def setup_chain(self): + print("Initializing test directory "+self.options.tmpdir) + initialize_chain_clean(self.options.tmpdir, 4) + + def setup_network(self): + self.nodes = [] + # Nodes 0/1 are "wallet" nodes + self.nodes.append(start_node(0, self.options.tmpdir, ["-debug"])) + self.nodes.append(start_node(1, self.options.tmpdir, ["-debug", "-timestampindex"])) + # Nodes 2/3 are used for testing + self.nodes.append(start_node(2, self.options.tmpdir, ["-debug"])) + self.nodes.append(start_node(3, self.options.tmpdir, ["-debug", "-timestampindex"])) + connect_nodes(self.nodes[0], 1) + connect_nodes(self.nodes[0], 2) + connect_nodes(self.nodes[0], 3) + + self.is_network_split = False + self.sync_all() + + def run_test(self): + print "Mining 5 blocks..." + blockhashes = self.nodes[0].generate(5) + low = self.nodes[0].getblock(blockhashes[0])["time"] + high = self.nodes[0].getblock(blockhashes[4])["time"] + self.sync_all() + print "Checking timestamp index..." + hashes = self.nodes[1].getblockhashes(high, low) + assert_equal(len(hashes), 5) + assert_equal(sorted(blockhashes), sorted(hashes)) + print "Passed\n" + + +if __name__ == '__main__': + TimestampIndexTest().main() diff --git a/src/main.cpp b/src/main.cpp index 0ad52abbb..de65db40a 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -67,6 +67,7 @@ bool fImporting = false; bool fReindex = false; bool fTxIndex = false; bool fAddressIndex = false; +bool fTimestampIndex = false; bool fHavePruned = false; bool fPruneMode = false; bool fIsBareMultisigStd = DEFAULT_PERMIT_BAREMULTISIG; @@ -1439,6 +1440,17 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa return res; } +bool GetTimestampIndex(const unsigned int &high, const unsigned int &low, std::vector &hashes) +{ + if (!fTimestampIndex) + return error("Timestamp index not enabled"); + + if (!pblocktree->ReadTimestampIndex(high, low, hashes)) + return error("Unable to get hashes for timestamps"); + + return true; +} + bool GetAddressIndex(uint160 addressHash, int type, std::vector > &addressIndex) { if (!fAddressIndex) @@ -2471,8 +2483,12 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin return AbortNode(state, "Failed to write transaction index"); if (fAddressIndex) - if (!pblocktree->WriteAddressIndex(addressIndex)) - return AbortNode(state, "Failed to write address index"); + if (!pblocktree->WriteAddressIndex(addressIndex)) + return AbortNode(state, "Failed to write address index"); + + if (fTimestampIndex) + if (!pblocktree->WriteTimestampIndex(CTimestampIndexKey(pindex->nTime, pindex->GetBlockHash()))) + return AbortNode(state, "Failed to write timestamp index"); // add this block to the view's block chain view.SetBestBlock(pindex->GetBlockHash()); @@ -3869,6 +3885,10 @@ bool static LoadBlockIndexDB() pblocktree->ReadFlag("addressindex", fAddressIndex); LogPrintf("%s: address index %s\n", __func__, fAddressIndex ? "enabled" : "disabled"); + // Check whether we have a timestamp index + pblocktree->ReadFlag("timestampindex", fTimestampIndex); + LogPrintf("%s: timestamp index %s\n", __func__, fTimestampIndex ? "enabled" : "disabled"); + // Load pointer to end of best chain BlockMap::iterator it = mapBlockIndex.find(pcoinsTip->GetBestBlock()); if (it == mapBlockIndex.end()) @@ -4034,6 +4054,10 @@ bool InitBlockIndex(const CChainParams& chainparams) fAddressIndex = GetBoolArg("-addressindex", DEFAULT_ADDRESSINDEX); pblocktree->WriteFlag("addressindex", fAddressIndex); + // Use the provided setting for -timestampindex in the new database + fTimestampIndex = GetBoolArg("-timestampindex", DEFAULT_TIMESTAMPINDEX); + pblocktree->WriteFlag("timestampindex", fTimestampIndex); + LogPrintf("Initializing databases...\n"); // Only add the genesis block if not reindexing (in which case we reuse the one already on disk) diff --git a/src/main.h b/src/main.h index 137915253..fe0a3dd10 100644 --- a/src/main.h +++ b/src/main.h @@ -112,6 +112,7 @@ static const unsigned int DEFAULT_BYTES_PER_SIGOP = 20; static const bool DEFAULT_CHECKPOINTS_ENABLED = true; static const bool DEFAULT_TXINDEX = false; static const bool DEFAULT_ADDRESSINDEX = false; +static const bool DEFAULT_TIMESTAMPINDEX = false; static const unsigned int DEFAULT_BANSCORE_THRESHOLD = 100; static const bool DEFAULT_TESTSAFEMODE = false; @@ -291,6 +292,67 @@ struct CNodeStateStats { std::vector vHeightInFlight; }; +struct CTimestampIndexIteratorKey { + unsigned int timestamp; + + size_t GetSerializeSize(int nType, int nVersion) const { + return 4; + } + template + void Serialize(Stream& s, int nType, int nVersion) const { + ser_writedata32be(s, timestamp); + } + template + void Unserialize(Stream& s, int nType, int nVersion) { + timestamp = ser_readdata32be(s); + } + + CTimestampIndexIteratorKey(unsigned int time) { + timestamp = time; + } + + CTimestampIndexIteratorKey() { + SetNull(); + } + + void SetNull() { + timestamp = 0; + } +}; + +struct CTimestampIndexKey { + unsigned int timestamp; + uint256 blockHash; + + size_t GetSerializeSize(int nType, int nVersion) const { + return 36; + } + template + void Serialize(Stream& s, int nType, int nVersion) const { + ser_writedata32be(s, timestamp); + blockHash.Serialize(s, nType, nVersion); + } + template + void Unserialize(Stream& s, int nType, int nVersion) { + timestamp = ser_readdata32be(s); + blockHash.Unserialize(s, nType, nVersion); + } + + CTimestampIndexKey(unsigned int time, uint256 hash) { + timestamp = time; + blockHash = hash; + } + + CTimestampIndexKey() { + SetNull(); + } + + void SetNull() { + timestamp = 0; + blockHash.SetNull(); + } +}; + struct CAddressIndexKey { unsigned int type; uint160 hashBytes; @@ -510,6 +572,7 @@ public: ScriptError GetScriptError() const { return error; } }; +bool GetTimestampIndex(const unsigned int &high, const unsigned int &low, std::vector &hashes); bool GetAddressIndex(uint160 addressHash, int type, std::vector > &addressIndex); /** Functions for disk access for blocks */ diff --git a/src/rpcblockchain.cpp b/src/rpcblockchain.cpp index f0bcafafe..276e99400 100644 --- a/src/rpcblockchain.cpp +++ b/src/rpcblockchain.cpp @@ -275,6 +275,38 @@ UniValue getrawmempool(const UniValue& params, bool fHelp) return mempoolToJSON(fVerbose); } +UniValue getblockhashes(const UniValue& params, bool fHelp) +{ + if (fHelp || params.size() != 2) + throw runtime_error( + "getblockhashes timestamp\n" + "\nReturns array of hashes of blocks within the timestamp range provided.\n" + "\nArguments:\n" + "1. high (numeric, required) The newer block timestamp\n" + "2. low (numeric, required) The older block timestamp\n" + "\nResult:\n" + "[" + " \"hash\" (string) The block hash\n" + "]" + "\nExamples:\n" + ); + + unsigned int high = params[0].get_int(); + unsigned int low = params[1].get_int(); + std::vector blockHashes; + + if (!GetTimestampIndex(high, low, blockHashes)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for block hashes"); + } + + UniValue result(UniValue::VARR); + for (std::vector::const_iterator it=blockHashes.begin(); it!=blockHashes.end(); it++) { + result.push_back(it->GetHex()); + } + + return result; +} + UniValue getblockhash(const UniValue& params, bool fHelp) { if (fHelp || params.size() != 1) diff --git a/src/rpcserver.cpp b/src/rpcserver.cpp index da66b1c0a..51235c184 100644 --- a/src/rpcserver.cpp +++ b/src/rpcserver.cpp @@ -278,6 +278,7 @@ static const CRPCCommand vRPCCommands[] = { "blockchain", "getbestblockhash", &getbestblockhash, true }, { "blockchain", "getblockcount", &getblockcount, true }, { "blockchain", "getblock", &getblock, true }, + { "blockchain", "getblockhashes", &getblockhashes, true }, { "blockchain", "getblockhash", &getblockhash, true }, { "blockchain", "getblockheader", &getblockheader, true }, { "blockchain", "getchaintips", &getchaintips, true }, diff --git a/src/rpcserver.h b/src/rpcserver.h index 8a80fc5ee..0be818093 100644 --- a/src/rpcserver.h +++ b/src/rpcserver.h @@ -258,6 +258,7 @@ extern UniValue getdifficulty(const UniValue& params, bool fHelp); extern UniValue settxfee(const UniValue& params, bool fHelp); extern UniValue getmempoolinfo(const UniValue& params, bool fHelp); extern UniValue getrawmempool(const UniValue& params, bool fHelp); +extern UniValue getblockhashes(const UniValue& params, bool fHelp); extern UniValue getblockhash(const UniValue& params, bool fHelp); extern UniValue getblockheader(const UniValue& params, bool fHelp); extern UniValue getblock(const UniValue& params, bool fHelp); diff --git a/src/txdb.cpp b/src/txdb.cpp index e93594b9e..3aa686058 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -22,6 +22,7 @@ static const char DB_COINS = 'c'; static const char DB_BLOCK_FILES = 'f'; static const char DB_TXINDEX = 't'; static const char DB_ADDRESSINDEX = 'a'; +static const char DB_TIMESTAMPINDEX = 's'; static const char DB_BLOCK_INDEX = 'b'; static const char DB_BEST_BLOCK = 'B'; @@ -196,6 +197,32 @@ bool CBlockTreeDB::ReadAddressIndex(uint160 addressHash, int type, std::vector &hashes) { + + boost::scoped_ptr pcursor(NewIterator()); + + pcursor->Seek(make_pair(DB_TIMESTAMPINDEX, CTimestampIndexIteratorKey(low))); + + while (pcursor->Valid()) { + boost::this_thread::interruption_point(); + std::pair key; + if (pcursor->GetKey(key) && key.first == DB_TIMESTAMPINDEX && key.second.timestamp <= high) { + hashes.push_back(key.second.blockHash); + pcursor->Next(); + } else { + break; + } + } + + return true; +} + bool CBlockTreeDB::WriteFlag(const std::string &name, bool fValue) { return Write(std::make_pair(DB_FLAG, name), fValue ? '1' : '0'); } diff --git a/src/txdb.h b/src/txdb.h index ca32869cc..7fefcaafa 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -19,6 +19,8 @@ class CBlockIndex; struct CDiskTxPos; struct CAddressIndexKey; struct CAddressIndexIteratorKey; +struct CTimestampIndexKey; +struct CTimestampIndexIteratorKey; class uint256; //! -dbcache default (MiB) @@ -61,6 +63,8 @@ public: bool WriteTxIndex(const std::vector > &list); bool WriteAddressIndex(const std::vector > &vect); bool ReadAddressIndex(uint160 addressHash, int type, std::vector > &addressIndex); + bool WriteTimestampIndex(const CTimestampIndexKey ×tampIndex); + bool ReadTimestampIndex(const unsigned int &high, const unsigned int &low, std::vector &vect); bool WriteFlag(const std::string &name, bool fValue); bool ReadFlag(const std::string &name, bool &fValue); bool LoadBlockIndexGuts(); From f76c2585f079b3e2e9c504db343ccd3c3897b3aa Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 23 Mar 2016 14:14:36 -0400 Subject: [PATCH 16/59] test: added to for balance after spending --- qa/rpc-tests/addressindex.py | 87 +++++++++++++++++++++++++----------- 1 file changed, 61 insertions(+), 26 deletions(-) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index 8f123eaa0..59837f783 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -46,7 +46,7 @@ class AddressIndexTest(BitcoinTestFramework): # Check that balances are correct balance0 = self.nodes[1].getaddressbalance("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br") - assert_equal(balance0['balance'], 0); + assert_equal(balance0["balance"], 0) # Check p2pkh and p2sh address indexes print "Testing p2pkh and p2sh address index..." @@ -71,31 +71,31 @@ class AddressIndexTest(BitcoinTestFramework): self.sync_all() - txids = self.nodes[1].getaddresstxids("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs"); - assert_equal(len(txids), 3); - assert_equal(txids[0], txid0); - assert_equal(txids[1], txid1); - assert_equal(txids[2], txid2); + txids = self.nodes[1].getaddresstxids("mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs") + assert_equal(len(txids), 3) + assert_equal(txids[0], txid0) + assert_equal(txids[1], txid1) + assert_equal(txids[2], txid2) - txidsb = self.nodes[1].getaddresstxids("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br"); - assert_equal(len(txidsb), 3); - assert_equal(txidsb[0], txidb0); - assert_equal(txidsb[1], txidb1); - assert_equal(txidsb[2], txidb2); + txidsb = self.nodes[1].getaddresstxids("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br") + assert_equal(len(txidsb), 3) + assert_equal(txidsb[0], txidb0) + assert_equal(txidsb[1], txidb1) + assert_equal(txidsb[2], txidb2) # Check that multiple addresses works - multitxids = self.nodes[1].getaddresstxids({"addresses": ["2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br", "mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs"]}); - assert_equal(len(multitxids), 6); - assert_equal(multitxids[0], txid0); - assert_equal(multitxids[1], txidb0); - assert_equal(multitxids[2], txid1); - assert_equal(multitxids[3], txidb1); - assert_equal(multitxids[4], txid2); - assert_equal(multitxids[5], txidb2); + multitxids = self.nodes[1].getaddresstxids({"addresses": ["2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br", "mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs"]}) + assert_equal(len(multitxids), 6) + assert_equal(multitxids[0], txid0) + assert_equal(multitxids[1], txidb0) + assert_equal(multitxids[2], txid1) + assert_equal(multitxids[3], txidb1) + assert_equal(multitxids[4], txid2) + assert_equal(multitxids[5], txidb2) # Check that balances are correct balance0 = self.nodes[1].getaddressbalance("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br") - assert_equal(balance0['balance'], 45 * 100000000); + assert_equal(balance0["balance"], 45 * 100000000) # Check that outputs with the same address will only return one txid print "Testing for txid uniqueness..." @@ -107,19 +107,54 @@ class AddressIndexTest(BitcoinTestFramework): tx.vout = [CTxOut(10, scriptPubKey), CTxOut(11, scriptPubKey)] tx.rehash() - signed_tx = self.nodes[0].signrawtransaction(binascii.hexlify(tx.serialize()).decode('utf-8')) - sent_txid = self.nodes[0].sendrawtransaction(signed_tx['hex'], True) + signed_tx = self.nodes[0].signrawtransaction(binascii.hexlify(tx.serialize()).decode("utf-8")) + sent_txid = self.nodes[0].sendrawtransaction(signed_tx["hex"], True) self.nodes[0].generate(1) self.sync_all() - txidsmany = self.nodes[1].getaddresstxids("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br"); - assert_equal(len(txidsmany), 4); - assert_equal(txidsmany[3], sent_txid); + txidsmany = self.nodes[1].getaddresstxids("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br") + assert_equal(len(txidsmany), 4) + assert_equal(txidsmany[3], sent_txid) # Check that balances are correct balance0 = self.nodes[1].getaddressbalance("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br") - assert_equal(balance0['balance'], 45 * 100000000 + 21); + assert_equal(balance0["balance"], 45 * 100000000 + 21) + + # Check that balances are correct after spending + privkey2 = "cSdkPxkAjA4HDr5VHgsebAPDEh9Gyub4HK8UJr2DFGGqKKy4K5sG" + address2 = "mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW" + addressHash2 = "0b2f0a0c31bfe0406b0ccc1381fdbe311946dadc".decode("hex") + scriptPubKey2 = CScript([OP_DUP, OP_HASH160, addressHash2, OP_EQUALVERIFY, OP_CHECKSIG]) + self.nodes[0].importprivkey(privkey2) + + unspent = self.nodes[0].listunspent() + tx = CTransaction() + tx.vin = [CTxIn(COutPoint(int(unspent[0]["txid"], 16), unspent[0]["vout"]))] + amount = unspent[0]["amount"] * 100000000 + tx.vout = [CTxOut(amount, scriptPubKey2)] + tx.rehash() + signed_tx = self.nodes[0].signrawtransaction(binascii.hexlify(tx.serialize()).decode("utf-8")) + spending_txid = self.nodes[0].sendrawtransaction(signed_tx["hex"], True) + self.nodes[0].generate(1) + self.sync_all() + balance1 = self.nodes[1].getaddressbalance(address2) + assert_equal(balance1["balance"], amount) + + tx = CTransaction() + tx.vin = [CTxIn(COutPoint(int(spending_txid, 16), 0))] + send_amount = 1 * 100000000 + 12840 + change_amount = amount - send_amount - 10000 + tx.vout = [CTxOut(send_amount, scriptPubKey), CTxOut(change_amount, scriptPubKey2)] + tx.rehash() + + signed_tx = self.nodes[0].signrawtransaction(binascii.hexlify(tx.serialize()).decode("utf-8")) + sent_txid = self.nodes[0].sendrawtransaction(signed_tx["hex"], True) + self.nodes[0].generate(1) + self.sync_all() + + balance2 = self.nodes[1].getaddressbalance(address2) + assert_equal(balance2["balance"], change_amount) print "Passed\n" From 206882cd4b7a93c08579134986814b76c49564e2 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 23 Mar 2016 14:41:08 -0400 Subject: [PATCH 17/59] main: fixed bug with overlapping address index keys There was a bug where the spending address index could have the same key as the receiving address index if the input and output indexes matched. This lead to the output always overwriting the input index leading to incorrect balances with missing spent amounts. This patch separates the two so that they have unique keys so balances will be correctly calculated. --- qa/rpc-tests/addressindex.py | 2 +- src/main.cpp | 8 ++++---- src/main.h | 9 ++++++++- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index 59837f783..0ffbcffd7 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -145,7 +145,7 @@ class AddressIndexTest(BitcoinTestFramework): tx.vin = [CTxIn(COutPoint(int(spending_txid, 16), 0))] send_amount = 1 * 100000000 + 12840 change_amount = amount - send_amount - 10000 - tx.vout = [CTxOut(send_amount, scriptPubKey), CTxOut(change_amount, scriptPubKey2)] + tx.vout = [CTxOut(change_amount, scriptPubKey2), CTxOut(send_amount, scriptPubKey)] tx.rehash() signed_tx = self.nodes[0].signrawtransaction(binascii.hexlify(tx.serialize()).decode("utf-8")) diff --git a/src/main.cpp b/src/main.cpp index de65db40a..3826bc198 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -2384,10 +2384,10 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin const CTxOut &prevout = view.GetOutputFor(tx.vin[j]); if (prevout.scriptPubKey.IsPayToScriptHash()) { vector hashBytes(prevout.scriptPubKey.begin()+2, prevout.scriptPubKey.begin()+22); - addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, j), prevout.nValue * -1)); + addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, j, true), prevout.nValue * -1)); } else if (prevout.scriptPubKey.IsPayToPublicKeyHash()) { vector hashBytes(prevout.scriptPubKey.begin()+3, prevout.scriptPubKey.begin()+23); - addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, j), prevout.nValue * -1)); + addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, j, true), prevout.nValue * -1)); } else { continue; } @@ -2421,10 +2421,10 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin if (out.scriptPubKey.IsPayToScriptHash()) { vector hashBytes(out.scriptPubKey.begin()+2, out.scriptPubKey.begin()+22); - addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, k), out.nValue)); + addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, k, false), out.nValue)); } else if (out.scriptPubKey.IsPayToPublicKeyHash()) { vector hashBytes(out.scriptPubKey.begin()+3, out.scriptPubKey.begin()+23); - addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, k), out.nValue)); + addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, k, false), out.nValue)); } else { continue; } diff --git a/src/main.h b/src/main.h index fe0a3dd10..3378b24df 100644 --- a/src/main.h +++ b/src/main.h @@ -360,6 +360,7 @@ struct CAddressIndexKey { unsigned int txindex; uint256 txhash; size_t outindex; + bool spending; size_t GetSerializeSize(int nType, int nVersion) const { return 65; @@ -373,6 +374,8 @@ struct CAddressIndexKey { ser_writedata32be(s, txindex); txhash.Serialize(s, nType, nVersion); ser_writedata32(s, outindex); + char f = spending; + ser_writedata8(s, f); } template void Unserialize(Stream& s, int nType, int nVersion) { @@ -382,16 +385,19 @@ struct CAddressIndexKey { txindex = ser_readdata32be(s); txhash.Unserialize(s, nType, nVersion); outindex = ser_readdata32(s); + char f = ser_readdata8(s); + spending = f; } CAddressIndexKey(unsigned int addressType, uint160 addressHash, int height, int blockindex, - uint256 txid, size_t outputIndex) { + uint256 txid, size_t outputIndex, bool isSpending) { type = addressType; hashBytes = addressHash; blockHeight = height; txindex = blockindex; txhash = txid; outindex = outputIndex; + spending = isSpending; } CAddressIndexKey() { @@ -405,6 +411,7 @@ struct CAddressIndexKey { txindex = 0; txhash.SetNull(); outindex = 0; + spending = false; } }; From cad092aebb03fcb7884c947c0ecac26474ba4931 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 24 Mar 2016 15:44:23 -0400 Subject: [PATCH 18/59] main: get address deltas between range of block heights --- qa/rpc-tests/addressindex.py | 11 +++++++ src/main.cpp | 9 +++--- src/main.h | 36 +++++++++++++++++----- src/rpcmisc.cpp | 60 ++++++++++++++++++++++++++++++++++++ src/rpcserver.cpp | 1 + src/rpcserver.h | 1 + src/txdb.cpp | 13 ++++++-- src/txdb.h | 4 ++- 8 files changed, 120 insertions(+), 15 deletions(-) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index 0ffbcffd7..4fc4cc557 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -156,6 +156,17 @@ class AddressIndexTest(BitcoinTestFramework): balance2 = self.nodes[1].getaddressbalance(address2) assert_equal(balance2["balance"], change_amount) + # Check that deltas are returned correctly + deltas = self.nodes[1].getaddressdeltas({"addresses": [address2], "start": 0, "end": 200}) + balance3 = 0; + for delta in deltas: + balance3 += delta["satoshis"] + assert_equal(balance3, change_amount) + + # Check that deltas can be returned from range of block heights + deltas = self.nodes[1].getaddressdeltas({"addresses": [address2], "start": 113, "end": 113}) + assert_equal(len(deltas), 1); + print "Passed\n" diff --git a/src/main.cpp b/src/main.cpp index 3826bc198..4b7c95afd 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1451,13 +1451,14 @@ bool GetTimestampIndex(const unsigned int &high, const unsigned int &low, std::v return true; } -bool GetAddressIndex(uint160 addressHash, int type, std::vector > &addressIndex) +bool GetAddressIndex(uint160 addressHash, int type, + std::vector > &addressIndex, int start, int end) { if (!fAddressIndex) - return error("%s: address index not enabled"); + return error("address index not enabled"); - if (!pblocktree->ReadAddressIndex(addressHash, type, addressIndex)) - return error("%s: unable to get txids for address"); + if (!pblocktree->ReadAddressIndex(addressHash, type, addressIndex, start, end)) + return error("unable to get txids for address"); return true; } diff --git a/src/main.h b/src/main.h index 3378b24df..181f88993 100644 --- a/src/main.h +++ b/src/main.h @@ -359,7 +359,7 @@ struct CAddressIndexKey { int blockHeight; unsigned int txindex; uint256 txhash; - size_t outindex; + size_t index; bool spending; size_t GetSerializeSize(int nType, int nVersion) const { @@ -373,7 +373,7 @@ struct CAddressIndexKey { ser_writedata32be(s, blockHeight); ser_writedata32be(s, txindex); txhash.Serialize(s, nType, nVersion); - ser_writedata32(s, outindex); + ser_writedata32(s, index); char f = spending; ser_writedata8(s, f); } @@ -384,19 +384,19 @@ struct CAddressIndexKey { blockHeight = ser_readdata32be(s); txindex = ser_readdata32be(s); txhash.Unserialize(s, nType, nVersion); - outindex = ser_readdata32(s); + index = ser_readdata32(s); char f = ser_readdata8(s); spending = f; } CAddressIndexKey(unsigned int addressType, uint160 addressHash, int height, int blockindex, - uint256 txid, size_t outputIndex, bool isSpending) { + uint256 txid, size_t indexValue, bool isSpending) { type = addressType; hashBytes = addressHash; blockHeight = height; txindex = blockindex; txhash = txid; - outindex = outputIndex; + index = indexValue; spending = isSpending; } @@ -410,7 +410,7 @@ struct CAddressIndexKey { blockHeight = 0; txindex = 0; txhash.SetNull(); - outindex = 0; + index = 0; spending = false; } @@ -419,14 +419,23 @@ struct CAddressIndexKey { struct CAddressIndexIteratorKey { unsigned int type; uint160 hashBytes; + bool includeHeight; + int blockHeight; size_t GetSerializeSize(int nType, int nVersion) const { - return 21; + if (includeHeight) { + return 25; + } else { + return 21; + } } template void Serialize(Stream& s, int nType, int nVersion) const { ser_writedata8(s, type); hashBytes.Serialize(s, nType, nVersion); + if (includeHeight) { + ser_writedata32be(s, blockHeight); + } } template void Unserialize(Stream& s, int nType, int nVersion) { @@ -437,6 +446,14 @@ struct CAddressIndexIteratorKey { CAddressIndexIteratorKey(unsigned int addressType, uint160 addressHash) { type = addressType; hashBytes = addressHash; + includeHeight = false; + } + + CAddressIndexIteratorKey(unsigned int addressType, uint160 addressHash, int height) { + type = addressType; + hashBytes = addressHash; + blockHeight = height; + includeHeight = true; } CAddressIndexIteratorKey() { @@ -446,6 +463,7 @@ struct CAddressIndexIteratorKey { void SetNull() { type = 0; hashBytes.SetNull(); + includeHeight = false; } }; @@ -580,7 +598,9 @@ public: }; bool GetTimestampIndex(const unsigned int &high, const unsigned int &low, std::vector &hashes); -bool GetAddressIndex(uint160 addressHash, int type, std::vector > &addressIndex); +bool GetAddressIndex(uint160 addressHash, int type, + std::vector > &addressIndex, + int start = 0, int end = 0); /** Functions for disk access for blocks */ bool WriteBlockToDisk(const CBlock& block, CDiskBlockPos& pos, const CMessageHeader::MessageStartChars& messageStart); diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index 549e6c7a5..8685a327a 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -435,6 +435,66 @@ bool getAddressesFromParams(const UniValue& params, std::vector > addresses; + + if (!getAddressesFromParams(params, addresses)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); + } + + std::vector > addressIndex; + + for (std::vector >::iterator it = addresses.begin(); it != addresses.end(); it++) { + if (!GetAddressIndex((*it).first, (*it).second, addressIndex, start, end)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); + } + } + + UniValue result(UniValue::VARR); + + for (std::vector >::const_iterator it=addressIndex.begin(); it!=addressIndex.end(); it++) { + UniValue delta(UniValue::VOBJ); + delta.push_back(Pair("satoshis", it->second)); + delta.push_back(Pair("txid", it->first.txhash.GetHex())); + delta.push_back(Pair("index", it->first.index)); + delta.push_back(Pair("height", it->first.blockHeight)); + delta.push_back(Pair("hash", it->first.hashBytes.GetHex())); + delta.push_back(Pair("type", (int)it->first.type)); + result.push_back(delta); + } + + return result; +} + UniValue getaddressbalance(const UniValue& params, bool fHelp) { if (fHelp || params.size() != 1) diff --git a/src/rpcserver.cpp b/src/rpcserver.cpp index 51235c184..c95548462 100644 --- a/src/rpcserver.cpp +++ b/src/rpcserver.cpp @@ -315,6 +315,7 @@ static const CRPCCommand vRPCCommands[] = #endif /* Address index */ + { "addressindex", "getaddressdeltas", &getaddressdeltas, false }, { "addressindex", "getaddresstxids", &getaddresstxids, false }, { "addressindex", "getaddressbalance", &getaddressbalance, false }, diff --git a/src/rpcserver.h b/src/rpcserver.h index 0be818093..fc57aa16b 100644 --- a/src/rpcserver.h +++ b/src/rpcserver.h @@ -166,6 +166,7 @@ extern std::string HelpExampleRpc(const std::string& methodname, const std::stri extern void EnsureWalletIsUnlocked(); extern UniValue getconnectioncount(const UniValue& params, bool fHelp); // in rpcnet.cpp +extern UniValue getaddressdeltas(const UniValue& params, bool fHelp); extern UniValue getaddresstxids(const UniValue& params, bool fHelp); extern UniValue getaddressbalance(const UniValue& params, bool fHelp); diff --git a/src/txdb.cpp b/src/txdb.cpp index 3aa686058..eb03b30e4 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -172,16 +172,25 @@ bool CBlockTreeDB::WriteAddressIndex(const std::vector > &addressIndex) { +bool CBlockTreeDB::ReadAddressIndex(uint160 addressHash, int type, + std::vector > &addressIndex, + int start, int end) { boost::scoped_ptr pcursor(NewIterator()); - pcursor->Seek(make_pair(DB_ADDRESSINDEX, CAddressIndexIteratorKey(type, addressHash))); + if (start > 0 && end > 0) { + pcursor->Seek(make_pair(DB_ADDRESSINDEX, CAddressIndexIteratorKey(type, addressHash, start))); + } else { + pcursor->Seek(make_pair(DB_ADDRESSINDEX, CAddressIndexIteratorKey(type, addressHash))); + } while (pcursor->Valid()) { boost::this_thread::interruption_point(); std::pair key; if (pcursor->GetKey(key) && key.first == DB_ADDRESSINDEX && key.second.hashBytes == addressHash) { + if (end > 0 && key.second.blockHeight > end) { + break; + } CAmount nValue; if (pcursor->GetValue(nValue)) { addressIndex.push_back(make_pair(key.second, nValue)); diff --git a/src/txdb.h b/src/txdb.h index 7fefcaafa..4586d51f8 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -62,7 +62,9 @@ public: bool ReadTxIndex(const uint256 &txid, CDiskTxPos &pos); bool WriteTxIndex(const std::vector > &list); bool WriteAddressIndex(const std::vector > &vect); - bool ReadAddressIndex(uint160 addressHash, int type, std::vector > &addressIndex); + bool ReadAddressIndex(uint160 addressHash, int type, + std::vector > &addressIndex, + int start = 0, int end = 0); bool WriteTimestampIndex(const CTimestampIndexKey ×tampIndex); bool ReadTimestampIndex(const unsigned int &high, const unsigned int &low, std::vector &vect); bool WriteFlag(const std::string &name, bool fValue); From 38a7d6d3234723d88be2b4a317e304ec9a8ada5e Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Fri, 25 Mar 2016 14:52:45 -0400 Subject: [PATCH 19/59] rpc: optimize address txid queries --- src/rpcmisc.cpp | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index 8685a327a..8d939ffea 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -567,24 +567,26 @@ UniValue getaddresstxids(const UniValue& params, bool fHelp) } } - std::set txids; - std::vector > vtxids; + std::set > txids; + UniValue result(UniValue::VARR); for (std::vector >::const_iterator it=addressIndex.begin(); it!=addressIndex.end(); it++) { int height = it->first.blockHeight; std::string txid = it->first.txhash.GetHex(); - if (txids.insert(txid).second) { - vtxids.push_back(std::make_pair(height, txid)); + + if (addresses.size() > 1) { + txids.insert(std::make_pair(height, txid)); + } else { + if (txids.insert(std::make_pair(height, txid)).second) { + result.push_back(txid); + } } } if (addresses.size() > 1) { - std::sort(vtxids.begin(), vtxids.end()); - } - - UniValue result(UniValue::VARR); - for (std::vector >::const_iterator it=vtxids.begin(); it!=vtxids.end(); it++) { - result.push_back(it->second); + for (std::set >::const_iterator it=txids.begin(); it!=txids.end(); it++) { + result.push_back(it->second); + } } return result; From 186e11fde7eac7765686b7e52655da7666866fb7 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 28 Mar 2016 16:47:20 -0400 Subject: [PATCH 20/59] main: update address index during reorgs --- qa/rpc-tests/addressindex.py | 19 +++++++++++++-- src/main.cpp | 46 ++++++++++++++++++++++++++++++++++++ src/txdb.cpp | 7 ++++++ src/txdb.h | 1 + 4 files changed, 71 insertions(+), 2 deletions(-) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index 4fc4cc557..a60c845b3 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -118,10 +118,12 @@ class AddressIndexTest(BitcoinTestFramework): assert_equal(txidsmany[3], sent_txid) # Check that balances are correct + print "Testing balances..." balance0 = self.nodes[1].getaddressbalance("2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br") assert_equal(balance0["balance"], 45 * 100000000 + 21) # Check that balances are correct after spending + print "Testing balances after spending..." privkey2 = "cSdkPxkAjA4HDr5VHgsebAPDEh9Gyub4HK8UJr2DFGGqKKy4K5sG" address2 = "mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW" addressHash2 = "0b2f0a0c31bfe0406b0ccc1381fdbe311946dadc".decode("hex") @@ -158,14 +160,27 @@ class AddressIndexTest(BitcoinTestFramework): # Check that deltas are returned correctly deltas = self.nodes[1].getaddressdeltas({"addresses": [address2], "start": 0, "end": 200}) - balance3 = 0; + balance3 = 0 for delta in deltas: balance3 += delta["satoshis"] assert_equal(balance3, change_amount) # Check that deltas can be returned from range of block heights deltas = self.nodes[1].getaddressdeltas({"addresses": [address2], "start": 113, "end": 113}) - assert_equal(len(deltas), 1); + assert_equal(len(deltas), 1) + + # Check that indexes will be updated with a reorg + print "Testing reorg..." + + best_hash = self.nodes[0].getbestblockhash() + self.nodes[0].invalidateblock(best_hash) + self.nodes[1].invalidateblock(best_hash) + self.nodes[2].invalidateblock(best_hash) + self.nodes[3].invalidateblock(best_hash) + self.sync_all() + + balance4 = self.nodes[1].getaddressbalance(address2) + assert_equal(balance4, balance1) print "Passed\n" diff --git a/src/main.cpp b/src/main.cpp index 4b7c95afd..d4555b0ce 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -2079,6 +2079,52 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI } } + // undo address indexes + if (fAddressIndex) { + std::vector > addressIndex; + + for (unsigned int i = 0; i < block.vtx.size(); i++) { + const CTransaction &tx = block.vtx[i]; + const uint256 txhash = tx.GetHash(); + + if (!tx.IsCoinBase()) { + for (size_t j = 0; j < tx.vin.size(); j++) { + const CTxOut &prevout = view.GetOutputFor(tx.vin[j]); + if (prevout.scriptPubKey.IsPayToScriptHash()) { + vector hashBytes(prevout.scriptPubKey.begin()+2, prevout.scriptPubKey.begin()+22); + addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, j, true), prevout.nValue * -1)); + } else if (prevout.scriptPubKey.IsPayToPublicKeyHash()) { + vector hashBytes(prevout.scriptPubKey.begin()+3, prevout.scriptPubKey.begin()+23); + addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, j, true), prevout.nValue * -1)); + } else { + continue; + } + } + } + + for (unsigned int k = 0; k < tx.vout.size(); k++) { + const CTxOut &out = tx.vout[k]; + + if (out.scriptPubKey.IsPayToScriptHash()) { + vector hashBytes(out.scriptPubKey.begin()+2, out.scriptPubKey.begin()+22); + addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, k, false), out.nValue)); + } else if (out.scriptPubKey.IsPayToPublicKeyHash()) { + vector hashBytes(out.scriptPubKey.begin()+3, out.scriptPubKey.begin()+23); + addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, k, false), out.nValue)); + } else { + continue; + } + + } + + } + + if (!pblocktree->EraseAddressIndex(addressIndex)) { + return AbortNode(state, "Failed to delete address index"); + } + } + + // move best block pointer to prevout block view.SetBestBlock(pindex->pprev->GetBlockHash()); diff --git a/src/txdb.cpp b/src/txdb.cpp index eb03b30e4..2128c181f 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -172,6 +172,13 @@ bool CBlockTreeDB::WriteAddressIndex(const std::vector >&vect) { + CDBBatch batch(&GetObfuscateKey()); + for (std::vector >::const_iterator it=vect.begin(); it!=vect.end(); it++) + batch.Erase(make_pair(DB_ADDRESSINDEX, it->first)); + return WriteBatch(batch); +} + bool CBlockTreeDB::ReadAddressIndex(uint160 addressHash, int type, std::vector > &addressIndex, int start, int end) { diff --git a/src/txdb.h b/src/txdb.h index 4586d51f8..93b5c0f5c 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -62,6 +62,7 @@ public: bool ReadTxIndex(const uint256 &txid, CDiskTxPos &pos); bool WriteTxIndex(const std::vector > &list); bool WriteAddressIndex(const std::vector > &vect); + bool EraseAddressIndex(const std::vector > &vect); bool ReadAddressIndex(uint160 addressHash, int type, std::vector > &addressIndex, int start = 0, int end = 0); From 8597289d8b1cef8ebc2895f383f0ce9ac9666b00 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 29 Mar 2016 09:13:31 -0400 Subject: [PATCH 21/59] main: fix order of address index when disconnecting block --- src/main.cpp | 59 ++++++++++++++++++++++++---------------------------- 1 file changed, 27 insertions(+), 32 deletions(-) diff --git a/src/main.cpp b/src/main.cpp index d4555b0ce..6616c290e 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -2041,11 +2041,32 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI if (blockUndo.vtxundo.size() + 1 != block.vtx.size()) return error("DisconnectBlock(): block and undo data inconsistent"); + std::vector > addressIndex; + // undo transactions in reverse order for (int i = block.vtx.size() - 1; i >= 0; i--) { const CTransaction &tx = block.vtx[i]; uint256 hash = tx.GetHash(); + if (fAddressIndex) { + + for (unsigned int k = tx.vout.size(); k-- > 0;) { + const CTxOut &out = tx.vout[k]; + + if (out.scriptPubKey.IsPayToScriptHash()) { + vector hashBytes(out.scriptPubKey.begin()+2, out.scriptPubKey.begin()+22); + addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, hash, k, false), out.nValue)); + } else if (out.scriptPubKey.IsPayToPublicKeyHash()) { + vector hashBytes(out.scriptPubKey.begin()+3, out.scriptPubKey.begin()+23); + addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, hash, k, false), out.nValue)); + } else { + continue; + } + + } + + } + // Check that all outputs are available and match the outputs in the block itself // exactly. { @@ -2075,56 +2096,30 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI const CTxInUndo &undo = txundo.vprevout[j]; if (!ApplyTxInUndo(undo, view, out)) fClean = false; - } - } - } - // undo address indexes - if (fAddressIndex) { - std::vector > addressIndex; - - for (unsigned int i = 0; i < block.vtx.size(); i++) { - const CTransaction &tx = block.vtx[i]; - const uint256 txhash = tx.GetHash(); - - if (!tx.IsCoinBase()) { - for (size_t j = 0; j < tx.vin.size(); j++) { + if (fAddressIndex) { const CTxOut &prevout = view.GetOutputFor(tx.vin[j]); if (prevout.scriptPubKey.IsPayToScriptHash()) { vector hashBytes(prevout.scriptPubKey.begin()+2, prevout.scriptPubKey.begin()+22); - addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, j, true), prevout.nValue * -1)); + addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, hash, j, true), prevout.nValue * -1)); } else if (prevout.scriptPubKey.IsPayToPublicKeyHash()) { vector hashBytes(prevout.scriptPubKey.begin()+3, prevout.scriptPubKey.begin()+23); - addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, j, true), prevout.nValue * -1)); + addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, hash, j, true), prevout.nValue * -1)); } else { continue; } } } - - for (unsigned int k = 0; k < tx.vout.size(); k++) { - const CTxOut &out = tx.vout[k]; - - if (out.scriptPubKey.IsPayToScriptHash()) { - vector hashBytes(out.scriptPubKey.begin()+2, out.scriptPubKey.begin()+22); - addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, k, false), out.nValue)); - } else if (out.scriptPubKey.IsPayToPublicKeyHash()) { - vector hashBytes(out.scriptPubKey.begin()+3, out.scriptPubKey.begin()+23); - addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, k, false), out.nValue)); - } else { - continue; - } - - } - } + } + + if (fAddressIndex) { if (!pblocktree->EraseAddressIndex(addressIndex)) { return AbortNode(state, "Failed to delete address index"); } } - // move best block pointer to prevout block view.SetBestBlock(pindex->pprev->GetBlockHash()); From 0b42ba227af7fe537e7bad095c0015f9201d9cca Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 29 Mar 2016 15:17:30 -0400 Subject: [PATCH 22/59] main: index unspent outputs by address --- qa/rpc-tests/addressindex.py | 6 +++ src/main.cpp | 46 +++++++++++++++++- src/main.h | 91 +++++++++++++++++++++++++++++++++++- src/rpcmisc.cpp | 50 ++++++++++++++++++++ src/rpcserver.cpp | 1 + src/rpcserver.h | 1 + src/txdb.cpp | 39 ++++++++++++++++ src/txdb.h | 5 ++ 8 files changed, 236 insertions(+), 3 deletions(-) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index a60c845b3..5448af6c6 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -169,6 +169,12 @@ class AddressIndexTest(BitcoinTestFramework): deltas = self.nodes[1].getaddressdeltas({"addresses": [address2], "start": 113, "end": 113}) assert_equal(len(deltas), 1) + # Check that unspent outputs can be queried + print "Testing utxos..." + utxos = self.nodes[1].getaddressutxos({"addresses": [address2]}) + assert_equal(len(utxos), 2) + assert_equal(utxos[0]["satoshis"], 5000000000) + # Check that indexes will be updated with a reorg print "Testing reorg..." diff --git a/src/main.cpp b/src/main.cpp index 6616c290e..dbbeef8f9 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1463,6 +1463,18 @@ bool GetAddressIndex(uint160 addressHash, int type, return true; } +bool GetAddressUnspent(uint160 addressHash, int type, + std::vector > &unspentOutputs) +{ + if (!fAddressIndex) + return error("address index not enabled"); + + if (!pblocktree->ReadAddressUnspentIndex(addressHash, type, unspentOutputs)) + return error("unable to get txids for address"); + + return true; +} + /** Return transaction in tx, and if it was found inside a block, its hash is placed in hashBlock */ bool GetTransaction(const uint256 &hash, CTransaction &txOut, const Consensus::Params& consensusParams, uint256 &hashBlock, bool fAllowSlow) { @@ -2389,6 +2401,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin vPos.reserve(block.vtx.size()); blockundo.vtxundo.reserve(block.vtx.size() - 1); std::vector > addressIndex; + std::vector > addressUnspentIndex; for (unsigned int i = 0; i < block.vtx.size(); i++) { @@ -2426,10 +2439,21 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin const CTxOut &prevout = view.GetOutputFor(tx.vin[j]); if (prevout.scriptPubKey.IsPayToScriptHash()) { vector hashBytes(prevout.scriptPubKey.begin()+2, prevout.scriptPubKey.begin()+22); + + // record spending activity addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, j, true), prevout.nValue * -1)); + + // remove address from unspent index + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, j), CAddressUnspentValue())); } else if (prevout.scriptPubKey.IsPayToPublicKeyHash()) { vector hashBytes(prevout.scriptPubKey.begin()+3, prevout.scriptPubKey.begin()+23); + + // record spending activity addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, j, true), prevout.nValue * -1)); + + // remove address from unspent index + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, j), CAddressUnspentValue())); + } else { continue; } @@ -2463,10 +2487,22 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin if (out.scriptPubKey.IsPayToScriptHash()) { vector hashBytes(out.scriptPubKey.begin()+2, out.scriptPubKey.begin()+22); + + // record receiving activity addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, k, false), out.nValue)); + + // record unspent output + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, k), CAddressUnspentValue(out.nValue, out.scriptPubKey))); + } else if (out.scriptPubKey.IsPayToPublicKeyHash()) { vector hashBytes(out.scriptPubKey.begin()+3, out.scriptPubKey.begin()+23); + + // record receiving activity addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, k, false), out.nValue)); + + // record unspent output + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, k), CAddressUnspentValue(out.nValue, out.scriptPubKey))); + } else { continue; } @@ -2524,9 +2560,15 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin if (!pblocktree->WriteTxIndex(vPos)) return AbortNode(state, "Failed to write transaction index"); - if (fAddressIndex) - if (!pblocktree->WriteAddressIndex(addressIndex)) + if (fAddressIndex) { + if (!pblocktree->WriteAddressIndex(addressIndex)) { return AbortNode(state, "Failed to write address index"); + } + + if (!pblocktree->UpdateAddressUnspentIndex(addressUnspentIndex)) { + return AbortNode(state, "Failed to write address unspent index"); + } + } if (fTimestampIndex) if (!pblocktree->WriteTimestampIndex(CTimestampIndexKey(pindex->nTime, pindex->GetBlockHash()))) diff --git a/src/main.h b/src/main.h index 181f88993..0964e6dc3 100644 --- a/src/main.h +++ b/src/main.h @@ -353,6 +353,93 @@ struct CTimestampIndexKey { } }; +struct CAddressUnspentKey { + unsigned int type; + uint160 hashBytes; + int blockHeight; + unsigned int txindex; + uint256 txhash; + size_t index; + + size_t GetSerializeSize(int nType, int nVersion) const { + return 65; + } + template + void Serialize(Stream& s, int nType, int nVersion) const { + ser_writedata8(s, type); + hashBytes.Serialize(s, nType, nVersion); + // Heights are stored big-endian for key sorting in LevelDB + ser_writedata32be(s, blockHeight); + ser_writedata32be(s, txindex); + txhash.Serialize(s, nType, nVersion); + ser_writedata32(s, index); + } + template + void Unserialize(Stream& s, int nType, int nVersion) { + type = ser_readdata8(s); + hashBytes.Unserialize(s, nType, nVersion); + blockHeight = ser_readdata32be(s); + txindex = ser_readdata32be(s); + txhash.Unserialize(s, nType, nVersion); + index = ser_readdata32(s); + } + + CAddressUnspentKey(unsigned int addressType, uint160 addressHash, int height, int blockindex, + uint256 txid, size_t indexValue) { + type = addressType; + hashBytes = addressHash; + blockHeight = height; + txindex = blockindex; + txhash = txid; + index = indexValue; + } + + CAddressUnspentKey() { + SetNull(); + } + + void SetNull() { + type = 0; + hashBytes.SetNull(); + blockHeight = 0; + txindex = 0; + txhash.SetNull(); + index = 0; + } +}; + +struct CAddressUnspentValue { + CAmount satoshis; + CScript script; + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + READWRITE(satoshis); + READWRITE(*(CScriptBase*)(&script)); + } + + CAddressUnspentValue(CAmount sats, CScript scriptPubKey) { + satoshis = sats; + script = scriptPubKey; + } + + CAddressUnspentValue() { + SetNull(); + } + + void SetNull() { + satoshis = 0; + script = CScript(); + } + + bool IsNull() const { + return (satoshis == 0); + } + +}; + struct CAddressIndexKey { unsigned int type; uint160 hashBytes; @@ -363,7 +450,7 @@ struct CAddressIndexKey { bool spending; size_t GetSerializeSize(int nType, int nVersion) const { - return 65; + return 66; } template void Serialize(Stream& s, int nType, int nVersion) const { @@ -601,6 +688,8 @@ bool GetTimestampIndex(const unsigned int &high, const unsigned int &low, std::v bool GetAddressIndex(uint160 addressHash, int type, std::vector > &addressIndex, int start = 0, int end = 0); +bool GetAddressUnspent(uint160 addressHash, int type, + std::vector > &unspentOutputs); /** Functions for disk access for blocks */ bool WriteBlockToDisk(const CBlock& block, CDiskBlockPos& pos, const CMessageHeader::MessageStartChars& messageStart); diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index 8d939ffea..1eeded608 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -435,6 +435,56 @@ bool getAddressesFromParams(const UniValue& params, std::vector > addresses; + + if (!getAddressesFromParams(params, addresses)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); + } + + std::vector > unspentOutputs; + + for (std::vector >::iterator it = addresses.begin(); it != addresses.end(); it++) { + if (!GetAddressUnspent((*it).first, (*it).second, unspentOutputs)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); + } + } + + UniValue result(UniValue::VARR); + + for (std::vector >::const_iterator it=unspentOutputs.begin(); it!=unspentOutputs.end(); it++) { + UniValue output(UniValue::VOBJ); + output.push_back(Pair("addressType", (int)it->first.type)); + output.push_back(Pair("addressHash", it->first.hashBytes.GetHex())); + output.push_back(Pair("txid", it->first.txhash.GetHex())); + output.push_back(Pair("height", it->first.blockHeight)); + output.push_back(Pair("outputIndex", it->first.index)); + output.push_back(Pair("script", HexStr(it->second.script.begin(), it->second.script.end()))); + output.push_back(Pair("satoshis", it->second.satoshis)); + result.push_back(output); + } + + return result; +} + UniValue getaddressdeltas(const UniValue& params, bool fHelp) { if (fHelp || params.size() != 1 || !params[0].isObject()) diff --git a/src/rpcserver.cpp b/src/rpcserver.cpp index c95548462..39af0513b 100644 --- a/src/rpcserver.cpp +++ b/src/rpcserver.cpp @@ -315,6 +315,7 @@ static const CRPCCommand vRPCCommands[] = #endif /* Address index */ + { "addressindex", "getaddressutxos", &getaddressutxos, false }, { "addressindex", "getaddressdeltas", &getaddressdeltas, false }, { "addressindex", "getaddresstxids", &getaddresstxids, false }, { "addressindex", "getaddressbalance", &getaddressbalance, false }, diff --git a/src/rpcserver.h b/src/rpcserver.h index fc57aa16b..5ad147dec 100644 --- a/src/rpcserver.h +++ b/src/rpcserver.h @@ -166,6 +166,7 @@ extern std::string HelpExampleRpc(const std::string& methodname, const std::stri extern void EnsureWalletIsUnlocked(); extern UniValue getconnectioncount(const UniValue& params, bool fHelp); // in rpcnet.cpp +extern UniValue getaddressutxos(const UniValue& params, bool fHelp); extern UniValue getaddressdeltas(const UniValue& params, bool fHelp); extern UniValue getaddresstxids(const UniValue& params, bool fHelp); extern UniValue getaddressbalance(const UniValue& params, bool fHelp); diff --git a/src/txdb.cpp b/src/txdb.cpp index 2128c181f..8e485e9eb 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -22,6 +22,7 @@ static const char DB_COINS = 'c'; static const char DB_BLOCK_FILES = 'f'; static const char DB_TXINDEX = 't'; static const char DB_ADDRESSINDEX = 'a'; +static const char DB_ADDRESSUNSPENTINDEX = 'u'; static const char DB_TIMESTAMPINDEX = 's'; static const char DB_BLOCK_INDEX = 'b'; @@ -165,6 +166,44 @@ bool CBlockTreeDB::WriteTxIndex(const std::vector return WriteBatch(batch); } +bool CBlockTreeDB::UpdateAddressUnspentIndex(const std::vector >&vect) { + CDBBatch batch(&GetObfuscateKey()); + for (std::vector >::const_iterator it=vect.begin(); it!=vect.end(); it++) { + if (it->second.IsNull()) { + batch.Erase(make_pair(DB_ADDRESSUNSPENTINDEX, it->first)); + } else { + batch.Write(make_pair(DB_ADDRESSUNSPENTINDEX, it->first), it->second); + } + } + return WriteBatch(batch); +} + +bool CBlockTreeDB::ReadAddressUnspentIndex(uint160 addressHash, int type, + std::vector > &unspentOutputs) { + + boost::scoped_ptr pcursor(NewIterator()); + + pcursor->Seek(make_pair(DB_ADDRESSUNSPENTINDEX, CAddressIndexIteratorKey(type, addressHash))); + + while (pcursor->Valid()) { + boost::this_thread::interruption_point(); + std::pair key; + if (pcursor->GetKey(key) && key.first == DB_ADDRESSUNSPENTINDEX && key.second.hashBytes == addressHash) { + CAddressUnspentValue nValue; + if (pcursor->GetValue(nValue)) { + unspentOutputs.push_back(make_pair(key.second, nValue)); + pcursor->Next(); + } else { + return error("failed to get address unspent value"); + } + } else { + break; + } + } + + return true; +} + bool CBlockTreeDB::WriteAddressIndex(const std::vector >&vect) { CDBBatch batch(&GetObfuscateKey()); for (std::vector >::const_iterator it=vect.begin(); it!=vect.end(); it++) diff --git a/src/txdb.h b/src/txdb.h index 93b5c0f5c..547a48fa8 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -17,6 +17,8 @@ class CBlockFileInfo; class CBlockIndex; struct CDiskTxPos; +struct CAddressUnspentKey; +struct CAddressUnspentValue; struct CAddressIndexKey; struct CAddressIndexIteratorKey; struct CTimestampIndexKey; @@ -61,6 +63,9 @@ public: bool ReadReindexing(bool &fReindex); bool ReadTxIndex(const uint256 &txid, CDiskTxPos &pos); bool WriteTxIndex(const std::vector > &list); + bool UpdateAddressUnspentIndex(const std::vector >&vect); + bool ReadAddressUnspentIndex(uint160 addressHash, int type, + std::vector > &vect); bool WriteAddressIndex(const std::vector > &vect); bool EraseAddressIndex(const std::vector > &vect); bool ReadAddressIndex(uint160 addressHash, int type, From 08836972c093eb137e1c11eb9596e7d12d600332 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 29 Mar 2016 17:46:59 -0400 Subject: [PATCH 23/59] rpc: fix argument check for getaddressutxos --- src/rpcmisc.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index 1eeded608..2c3b7e6d0 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -437,7 +437,7 @@ bool getAddressesFromParams(const UniValue& params, std::vector Date: Tue, 29 Mar 2016 17:56:57 -0400 Subject: [PATCH 24/59] main: update unspent address index during reorgs --- qa/rpc-tests/addressindex.py | 6 ++++++ src/main.cpp | 29 +++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index 5448af6c6..d051d170a 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -174,6 +174,7 @@ class AddressIndexTest(BitcoinTestFramework): utxos = self.nodes[1].getaddressutxos({"addresses": [address2]}) assert_equal(len(utxos), 2) assert_equal(utxos[0]["satoshis"], 5000000000) + assert_equal(utxos[1]["satoshis"], 4899977160) # Check that indexes will be updated with a reorg print "Testing reorg..." @@ -188,6 +189,11 @@ class AddressIndexTest(BitcoinTestFramework): balance4 = self.nodes[1].getaddressbalance(address2) assert_equal(balance4, balance1) + utxos2 = self.nodes[1].getaddressutxos({"addresses": [address2]}) + assert_equal(len(utxos2), 2) + assert_equal(utxos2[0]["satoshis"], 5000000000) + assert_equal(utxos2[1]["satoshis"], 5000000000) + print "Passed\n" diff --git a/src/main.cpp b/src/main.cpp index dbbeef8f9..4d6872d2d 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -2054,6 +2054,7 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI return error("DisconnectBlock(): block and undo data inconsistent"); std::vector > addressIndex; + std::vector > addressUnspentIndex; // undo transactions in reverse order for (int i = block.vtx.size() - 1; i >= 0; i--) { @@ -2067,10 +2068,22 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI if (out.scriptPubKey.IsPayToScriptHash()) { vector hashBytes(out.scriptPubKey.begin()+2, out.scriptPubKey.begin()+22); + + // undo receiving activity addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, hash, k, false), out.nValue)); + + // undo unspent index + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), pindex->nHeight, i, hash, k), CAddressUnspentValue())); + } else if (out.scriptPubKey.IsPayToPublicKeyHash()) { vector hashBytes(out.scriptPubKey.begin()+3, out.scriptPubKey.begin()+23); + + // undo receiving activity addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, hash, k, false), out.nValue)); + + // undo unspent index + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), pindex->nHeight, i, hash, k), CAddressUnspentValue())); + } else { continue; } @@ -2113,10 +2126,23 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI const CTxOut &prevout = view.GetOutputFor(tx.vin[j]); if (prevout.scriptPubKey.IsPayToScriptHash()) { vector hashBytes(prevout.scriptPubKey.begin()+2, prevout.scriptPubKey.begin()+22); + + // undo spending activity addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, hash, j, true), prevout.nValue * -1)); + + // restore unspent index + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), pindex->nHeight, i, hash, j), CAddressUnspentValue(prevout.nValue, prevout.scriptPubKey))); + + } else if (prevout.scriptPubKey.IsPayToPublicKeyHash()) { vector hashBytes(prevout.scriptPubKey.begin()+3, prevout.scriptPubKey.begin()+23); + + // undo spending activity addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, hash, j, true), prevout.nValue * -1)); + + // restore unspent index + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), pindex->nHeight, i, hash, j), CAddressUnspentValue(prevout.nValue, prevout.scriptPubKey))); + } else { continue; } @@ -2130,6 +2156,9 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI if (!pblocktree->EraseAddressIndex(addressIndex)) { return AbortNode(state, "Failed to delete address index"); } + if (!pblocktree->UpdateAddressUnspentIndex(addressUnspentIndex)) { + return AbortNode(state, "Failed to write address unspent index"); + } } // move best block pointer to prevout block From 21c675855fa241c13771e03d9098939355c10b0e Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 30 Mar 2016 12:50:10 -0400 Subject: [PATCH 25/59] main: don't undo indexes when verifying blocks at startup --- src/main.cpp | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/main.cpp b/src/main.cpp index 4d6872d2d..b061383ef 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -2152,6 +2152,14 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI } + // move best block pointer to prevout block + view.SetBestBlock(pindex->pprev->GetBlockHash()); + + if (pfClean) { + *pfClean = fClean; + return true; + } + if (fAddressIndex) { if (!pblocktree->EraseAddressIndex(addressIndex)) { return AbortNode(state, "Failed to delete address index"); @@ -2161,14 +2169,6 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI } } - // move best block pointer to prevout block - view.SetBestBlock(pindex->pprev->GetBlockHash()); - - if (pfClean) { - *pfClean = fClean; - return true; - } - return fClean; } From d0483c9aa09a45f9e87902b9e023f977a3006830 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 30 Mar 2016 15:12:19 -0400 Subject: [PATCH 26/59] main: remove spent address utxo indexes --- qa/rpc-tests/addressindex.py | 8 +++----- src/main.cpp | 18 ++++++++++-------- src/main.h | 23 +++++------------------ src/rpcmisc.cpp | 1 - 4 files changed, 18 insertions(+), 32 deletions(-) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index d051d170a..703f3029a 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -172,9 +172,8 @@ class AddressIndexTest(BitcoinTestFramework): # Check that unspent outputs can be queried print "Testing utxos..." utxos = self.nodes[1].getaddressutxos({"addresses": [address2]}) - assert_equal(len(utxos), 2) - assert_equal(utxos[0]["satoshis"], 5000000000) - assert_equal(utxos[1]["satoshis"], 4899977160) + assert_equal(len(utxos), 1) + assert_equal(utxos[0]["satoshis"], change_amount) # Check that indexes will be updated with a reorg print "Testing reorg..." @@ -190,9 +189,8 @@ class AddressIndexTest(BitcoinTestFramework): assert_equal(balance4, balance1) utxos2 = self.nodes[1].getaddressutxos({"addresses": [address2]}) - assert_equal(len(utxos2), 2) + assert_equal(len(utxos2), 1) assert_equal(utxos2[0]["satoshis"], 5000000000) - assert_equal(utxos2[1]["satoshis"], 5000000000) print "Passed\n" diff --git a/src/main.cpp b/src/main.cpp index b061383ef..8a8189c97 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -2073,7 +2073,7 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, hash, k, false), out.nValue)); // undo unspent index - addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), pindex->nHeight, i, hash, k), CAddressUnspentValue())); + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), hash, k), CAddressUnspentValue())); } else if (out.scriptPubKey.IsPayToPublicKeyHash()) { vector hashBytes(out.scriptPubKey.begin()+3, out.scriptPubKey.begin()+23); @@ -2082,7 +2082,7 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, hash, k, false), out.nValue)); // undo unspent index - addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), pindex->nHeight, i, hash, k), CAddressUnspentValue())); + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), hash, k), CAddressUnspentValue())); } else { continue; @@ -2123,6 +2123,7 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI fClean = false; if (fAddressIndex) { + const CTxIn input = tx.vin[j]; const CTxOut &prevout = view.GetOutputFor(tx.vin[j]); if (prevout.scriptPubKey.IsPayToScriptHash()) { vector hashBytes(prevout.scriptPubKey.begin()+2, prevout.scriptPubKey.begin()+22); @@ -2131,7 +2132,7 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, hash, j, true), prevout.nValue * -1)); // restore unspent index - addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), pindex->nHeight, i, hash, j), CAddressUnspentValue(prevout.nValue, prevout.scriptPubKey))); + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), input.prevout.hash, input.prevout.n), CAddressUnspentValue(prevout.nValue, prevout.scriptPubKey))); } else if (prevout.scriptPubKey.IsPayToPublicKeyHash()) { @@ -2141,7 +2142,7 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, hash, j, true), prevout.nValue * -1)); // restore unspent index - addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), pindex->nHeight, i, hash, j), CAddressUnspentValue(prevout.nValue, prevout.scriptPubKey))); + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), input.prevout.hash, input.prevout.n), CAddressUnspentValue(prevout.nValue, prevout.scriptPubKey))); } else { continue; @@ -2465,6 +2466,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin if (fAddressIndex) { for (size_t j = 0; j < tx.vin.size(); j++) { + const CTxIn input = tx.vin[j]; const CTxOut &prevout = view.GetOutputFor(tx.vin[j]); if (prevout.scriptPubKey.IsPayToScriptHash()) { vector hashBytes(prevout.scriptPubKey.begin()+2, prevout.scriptPubKey.begin()+22); @@ -2473,7 +2475,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, j, true), prevout.nValue * -1)); // remove address from unspent index - addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, j), CAddressUnspentValue())); + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), input.prevout.hash, input.prevout.n), CAddressUnspentValue())); } else if (prevout.scriptPubKey.IsPayToPublicKeyHash()) { vector hashBytes(prevout.scriptPubKey.begin()+3, prevout.scriptPubKey.begin()+23); @@ -2481,7 +2483,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, j, true), prevout.nValue * -1)); // remove address from unspent index - addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, j), CAddressUnspentValue())); + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), input.prevout.hash, input.prevout.n), CAddressUnspentValue())); } else { continue; @@ -2521,7 +2523,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, k, false), out.nValue)); // record unspent output - addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, k), CAddressUnspentValue(out.nValue, out.scriptPubKey))); + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), txhash, k), CAddressUnspentValue(out.nValue, out.scriptPubKey))); } else if (out.scriptPubKey.IsPayToPublicKeyHash()) { vector hashBytes(out.scriptPubKey.begin()+3, out.scriptPubKey.begin()+23); @@ -2530,7 +2532,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, k, false), out.nValue)); // record unspent output - addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, k), CAddressUnspentValue(out.nValue, out.scriptPubKey))); + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), txhash, k), CAddressUnspentValue(out.nValue, out.scriptPubKey))); } else { continue; diff --git a/src/main.h b/src/main.h index 0964e6dc3..15ade067a 100644 --- a/src/main.h +++ b/src/main.h @@ -356,21 +356,16 @@ struct CTimestampIndexKey { struct CAddressUnspentKey { unsigned int type; uint160 hashBytes; - int blockHeight; - unsigned int txindex; uint256 txhash; size_t index; size_t GetSerializeSize(int nType, int nVersion) const { - return 65; + return 57; } template void Serialize(Stream& s, int nType, int nVersion) const { ser_writedata8(s, type); hashBytes.Serialize(s, nType, nVersion); - // Heights are stored big-endian for key sorting in LevelDB - ser_writedata32be(s, blockHeight); - ser_writedata32be(s, txindex); txhash.Serialize(s, nType, nVersion); ser_writedata32(s, index); } @@ -378,18 +373,13 @@ struct CAddressUnspentKey { void Unserialize(Stream& s, int nType, int nVersion) { type = ser_readdata8(s); hashBytes.Unserialize(s, nType, nVersion); - blockHeight = ser_readdata32be(s); - txindex = ser_readdata32be(s); txhash.Unserialize(s, nType, nVersion); index = ser_readdata32(s); } - CAddressUnspentKey(unsigned int addressType, uint160 addressHash, int height, int blockindex, - uint256 txid, size_t indexValue) { + CAddressUnspentKey(unsigned int addressType, uint160 addressHash, uint256 txid, size_t indexValue) { type = addressType; hashBytes = addressHash; - blockHeight = height; - txindex = blockindex; txhash = txid; index = indexValue; } @@ -401,8 +391,6 @@ struct CAddressUnspentKey { void SetNull() { type = 0; hashBytes.SetNull(); - blockHeight = 0; - txindex = 0; txhash.SetNull(); index = 0; } @@ -430,14 +418,13 @@ struct CAddressUnspentValue { } void SetNull() { - satoshis = 0; - script = CScript(); + satoshis = -1; + script.clear(); } bool IsNull() const { - return (satoshis == 0); + return (satoshis == -1); } - }; struct CAddressIndexKey { diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index 2c3b7e6d0..832d55f1a 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -475,7 +475,6 @@ UniValue getaddressutxos(const UniValue& params, bool fHelp) output.push_back(Pair("addressType", (int)it->first.type)); output.push_back(Pair("addressHash", it->first.hashBytes.GetHex())); output.push_back(Pair("txid", it->first.txhash.GetHex())); - output.push_back(Pair("height", it->first.blockHeight)); output.push_back(Pair("outputIndex", it->first.index)); output.push_back(Pair("script", HexStr(it->second.script.begin(), it->second.script.end()))); output.push_back(Pair("satoshis", it->second.satoshis)); From 1bd65a5c4bfebcc9a87a0f69b10b3711f8c79ec7 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 30 Mar 2016 16:42:37 -0400 Subject: [PATCH 27/59] main: sort address index utxos by height --- qa/rpc-tests/addressindex.py | 15 +++++++++++++++ src/main.cpp | 8 ++++---- src/main.h | 6 +++++- src/rpcmisc.cpp | 8 +++++++- 4 files changed, 31 insertions(+), 6 deletions(-) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index 703f3029a..ad79c7416 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -192,6 +192,21 @@ class AddressIndexTest(BitcoinTestFramework): assert_equal(len(utxos2), 1) assert_equal(utxos2[0]["satoshis"], 5000000000) + # Check sorting of utxos + self.nodes[2].generate(150) + + txidsort1 = self.nodes[2].sendtoaddress(address2, 50) + self.nodes[2].generate(1) + txidsort2 = self.nodes[2].sendtoaddress(address2, 50) + self.nodes[2].generate(1) + self.sync_all() + + utxos3 = self.nodes[1].getaddressutxos({"addresses": [address2]}) + assert_equal(len(utxos3), 3) + assert_equal(utxos3[0]["height"], 114) + assert_equal(utxos3[1]["height"], 264) + assert_equal(utxos3[2]["height"], 265) + print "Passed\n" diff --git a/src/main.cpp b/src/main.cpp index 8a8189c97..b4c7d8e24 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -2132,7 +2132,7 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, hash, j, true), prevout.nValue * -1)); // restore unspent index - addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), input.prevout.hash, input.prevout.n), CAddressUnspentValue(prevout.nValue, prevout.scriptPubKey))); + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), input.prevout.hash, input.prevout.n), CAddressUnspentValue(prevout.nValue, prevout.scriptPubKey, undo.nHeight))); } else if (prevout.scriptPubKey.IsPayToPublicKeyHash()) { @@ -2142,7 +2142,7 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, hash, j, true), prevout.nValue * -1)); // restore unspent index - addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), input.prevout.hash, input.prevout.n), CAddressUnspentValue(prevout.nValue, prevout.scriptPubKey))); + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), input.prevout.hash, input.prevout.n), CAddressUnspentValue(prevout.nValue, prevout.scriptPubKey, undo.nHeight))); } else { continue; @@ -2523,7 +2523,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, k, false), out.nValue)); // record unspent output - addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), txhash, k), CAddressUnspentValue(out.nValue, out.scriptPubKey))); + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), txhash, k), CAddressUnspentValue(out.nValue, out.scriptPubKey, pindex->nHeight))); } else if (out.scriptPubKey.IsPayToPublicKeyHash()) { vector hashBytes(out.scriptPubKey.begin()+3, out.scriptPubKey.begin()+23); @@ -2532,7 +2532,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, k, false), out.nValue)); // record unspent output - addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), txhash, k), CAddressUnspentValue(out.nValue, out.scriptPubKey))); + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), txhash, k), CAddressUnspentValue(out.nValue, out.scriptPubKey, pindex->nHeight))); } else { continue; diff --git a/src/main.h b/src/main.h index 15ade067a..90ed55e92 100644 --- a/src/main.h +++ b/src/main.h @@ -399,6 +399,7 @@ struct CAddressUnspentKey { struct CAddressUnspentValue { CAmount satoshis; CScript script; + int blockHeight; ADD_SERIALIZE_METHODS; @@ -406,11 +407,13 @@ struct CAddressUnspentValue { inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { READWRITE(satoshis); READWRITE(*(CScriptBase*)(&script)); + READWRITE(blockHeight); } - CAddressUnspentValue(CAmount sats, CScript scriptPubKey) { + CAddressUnspentValue(CAmount sats, CScript scriptPubKey, int height) { satoshis = sats; script = scriptPubKey; + blockHeight = height; } CAddressUnspentValue() { @@ -420,6 +423,7 @@ struct CAddressUnspentValue { void SetNull() { satoshis = -1; script.clear(); + blockHeight = 0; } bool IsNull() const { diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index 832d55f1a..ec0774660 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -431,8 +431,11 @@ bool getAddressesFromParams(const UniValue& params, std::vector a, + std::pair b) { + return a.second.blockHeight < b.second.blockHeight; } UniValue getaddressutxos(const UniValue& params, bool fHelp) @@ -468,6 +471,8 @@ UniValue getaddressutxos(const UniValue& params, bool fHelp) } } + std::sort(unspentOutputs.begin(), unspentOutputs.end(), heightSort); + UniValue result(UniValue::VARR); for (std::vector >::const_iterator it=unspentOutputs.begin(); it!=unspentOutputs.end(); it++) { @@ -478,6 +483,7 @@ UniValue getaddressutxos(const UniValue& params, bool fHelp) output.push_back(Pair("outputIndex", it->first.index)); output.push_back(Pair("script", HexStr(it->second.script.begin(), it->second.script.end()))); output.push_back(Pair("satoshis", it->second.satoshis)); + output.push_back(Pair("height", it->second.blockHeight)); result.push_back(output); } From b66eff47cff0c4b1135276b3cd30f108ea7fee54 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 4 Apr 2016 16:37:43 -0400 Subject: [PATCH 28/59] main: mempool address index --- qa/rpc-tests/addressindex.py | 40 ++++++++++++++++++- src/addressindex.h | 77 ++++++++++++++++++++++++++++++++++++ src/main.cpp | 3 ++ src/rpcmisc.cpp | 46 +++++++++++++++++++++ src/rpcserver.cpp | 1 + src/rpcserver.h | 1 + src/txmempool.cpp | 75 +++++++++++++++++++++++++++++++++++ src/txmempool.h | 12 ++++++ 8 files changed, 254 insertions(+), 1 deletion(-) create mode 100644 src/addressindex.h diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index ad79c7416..d28cd393f 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -7,6 +7,7 @@ # Test addressindex generation and fetching # +import time from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * from test_framework.script import * @@ -25,7 +26,7 @@ class AddressIndexTest(BitcoinTestFramework): self.nodes.append(start_node(0, self.options.tmpdir, ["-debug"])) self.nodes.append(start_node(1, self.options.tmpdir, ["-debug", "-addressindex"])) # Nodes 2/3 are used for testing - self.nodes.append(start_node(2, self.options.tmpdir, ["-debug"])) + self.nodes.append(start_node(2, self.options.tmpdir, ["-debug", "-addressindex"])) self.nodes.append(start_node(3, self.options.tmpdir, ["-debug", "-addressindex"])) connect_nodes(self.nodes[0], 1) connect_nodes(self.nodes[0], 2) @@ -207,6 +208,43 @@ class AddressIndexTest(BitcoinTestFramework): assert_equal(utxos3[1]["height"], 264) assert_equal(utxos3[2]["height"], 265) + # Check mempool indexing + print "Testing mempool indexing..." + + privKey3 = "cVfUn53hAbRrDEuMexyfgDpZPhF7KqXpS8UZevsyTDaugB7HZ3CD" + address3 = "mw4ynwhS7MmrQ27hr82kgqu7zryNDK26JB" + addressHash3 = "aa9872b5bbcdb511d89e0e11aa27da73fd2c3f50".decode("hex") + scriptPubKey3 = CScript([OP_DUP, OP_HASH160, addressHash3, OP_EQUALVERIFY, OP_CHECKSIG]) + unspent = self.nodes[2].listunspent() + + tx = CTransaction() + tx.vin = [CTxIn(COutPoint(int(unspent[0]["txid"], 16), unspent[0]["vout"]))] + amount = unspent[0]["amount"] * 100000000 + tx.vout = [CTxOut(amount, scriptPubKey3)] + tx.rehash() + signed_tx = self.nodes[2].signrawtransaction(binascii.hexlify(tx.serialize()).decode("utf-8")) + memtxid1 = self.nodes[2].sendrawtransaction(signed_tx["hex"], True) + time.sleep(2) + + tx2 = CTransaction() + tx2.vin = [CTxIn(COutPoint(int(unspent[1]["txid"], 16), unspent[1]["vout"]))] + amount = unspent[1]["amount"] * 100000000 + tx2.vout = [CTxOut(amount, scriptPubKey3)] + tx2.rehash() + signed_tx2 = self.nodes[2].signrawtransaction(binascii.hexlify(tx2.serialize()).decode("utf-8")) + memtxid2 = self.nodes[2].sendrawtransaction(signed_tx2["hex"], True) + time.sleep(2) + + mempool = self.nodes[2].getaddressmempool({"addresses": [address3]}) + assert_equal(len(mempool), 2) + assert_equal(mempool[0]["txid"], memtxid1) + assert_equal(mempool[1]["txid"], memtxid2) + + self.nodes[2].generate(1); + self.sync_all(); + mempool2 = self.nodes[2].getaddressmempool({"addresses": [address3]}) + assert_equal(len(mempool2), 0) + print "Passed\n" diff --git a/src/addressindex.h b/src/addressindex.h new file mode 100644 index 000000000..b6e3587c1 --- /dev/null +++ b/src/addressindex.h @@ -0,0 +1,77 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2015 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_ADDRESSINDEX_H +#define BITCOIN_ADDRESSINDEX_H + +#include "uint256.h" +#include "amount.h" + +struct CMempoolAddressDelta +{ + int64_t time; + CAmount amount; + uint256 prevhash; + unsigned int prevout; + + CMempoolAddressDelta(int64_t t, CAmount a, uint256 hash, unsigned int out) { + time = t; + amount = a; + prevhash = hash; + prevout = out; + } + + CMempoolAddressDelta(int64_t t, CAmount a) { + time = t; + amount = a; + prevhash.SetNull(); + prevout = 0; + } +}; + +struct CMempoolAddressDeltaKey +{ + int type; + uint160 addressBytes; + uint256 txhash; + unsigned int index; + bool spending; + + CMempoolAddressDeltaKey(int addressType, uint160 addressHash, uint256 hash, unsigned int i, bool s) { + type = addressType; + addressBytes = addressHash; + txhash = hash; + index = i; + spending = s; + } + + CMempoolAddressDeltaKey(int addressType, uint160 addressHash) { + type = addressType; + addressBytes = addressHash; + txhash.SetNull(); + index = 0; + } +}; + +struct CMempoolAddressDeltaKeyCompare +{ + bool operator()(const CMempoolAddressDeltaKey& a, const CMempoolAddressDeltaKey& b) { + if (a.type == b.type) { + if (a.addressBytes == b.addressBytes) { + if (a.txhash == b.txhash) { + return a.index < b.index; + } else { + return a.txhash < b.txhash; + } + } else { + return a.addressBytes < b.addressBytes; + } + } else { + return a.type < b.type; + } + } +}; + +#endif // BITCOIN_ADDRESSINDEX_H diff --git a/src/main.cpp b/src/main.cpp index b4c7d8e24..5af73848b 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1414,6 +1414,9 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState &state, const C // Store transaction in memory pool.addUnchecked(hash, entry, setAncestors, !IsInitialBlockDownload()); + if (fAddressIndex) { + pool.addAddressIndex(entry, view); + } // trim mempool and check if tx was trimmed if (!fOverrideMempoolLimit) { diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index ec0774660..f06baa042 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -11,6 +11,7 @@ #include "netbase.h" #include "rpcserver.h" #include "timedata.h" +#include "txmempool.h" #include "util.h" #include "utilstrencodings.h" #ifdef ENABLE_WALLET @@ -438,6 +439,51 @@ bool heightSort(std::pair a, return a.second.blockHeight < b.second.blockHeight; } +bool timestampSort(std::pair a, + std::pair b) { + return a.second.time < b.second.time; +} + +UniValue getaddressmempool(const UniValue& params, bool fHelp) +{ + if (fHelp || params.size() != 1) + throw runtime_error( + "getaddressmempool\n" + "\nReturns all mempool deltas for an address (requires addressindex to be enabled).\n" + ); + + std::vector > addresses; + + if (!getAddressesFromParams(params, addresses)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); + } + + std::vector > indexes; + + if (!mempool.getAddressIndex(addresses, indexes)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); + } + + std::sort(indexes.begin(), indexes.end(), timestampSort); + + UniValue result(UniValue::VARR); + + for (std::vector >::iterator it = indexes.begin(); + it != indexes.end(); it++) { + + UniValue delta(UniValue::VOBJ); + delta.push_back(Pair("addressType", (int)it->first.type)); + delta.push_back(Pair("addressHash", it->first.addressBytes.GetHex())); + delta.push_back(Pair("txid", it->first.txhash.GetHex())); + delta.push_back(Pair("index", (int)it->first.index)); + delta.push_back(Pair("satoshis", it->second.amount)); + delta.push_back(Pair("timestamp", it->second.time)); + result.push_back(delta); + } + + return result; +} + UniValue getaddressutxos(const UniValue& params, bool fHelp) { if (fHelp || params.size() != 1) diff --git a/src/rpcserver.cpp b/src/rpcserver.cpp index 39af0513b..1d5399df3 100644 --- a/src/rpcserver.cpp +++ b/src/rpcserver.cpp @@ -315,6 +315,7 @@ static const CRPCCommand vRPCCommands[] = #endif /* Address index */ + { "addressindex", "getaddressmempool", &getaddressmempool, false }, { "addressindex", "getaddressutxos", &getaddressutxos, false }, { "addressindex", "getaddressdeltas", &getaddressdeltas, false }, { "addressindex", "getaddresstxids", &getaddresstxids, false }, diff --git a/src/rpcserver.h b/src/rpcserver.h index 5ad147dec..74ecd1191 100644 --- a/src/rpcserver.h +++ b/src/rpcserver.h @@ -166,6 +166,7 @@ extern std::string HelpExampleRpc(const std::string& methodname, const std::stri extern void EnsureWalletIsUnlocked(); extern UniValue getconnectioncount(const UniValue& params, bool fHelp); // in rpcnet.cpp +extern UniValue getaddressmempool(const UniValue& params, bool fHelp); extern UniValue getaddressutxos(const UniValue& params, bool fHelp); extern UniValue getaddressdeltas(const UniValue& params, bool fHelp); extern UniValue getaddresstxids(const UniValue& params, bool fHelp); diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 5f814749b..adbe662ab 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -422,6 +422,80 @@ bool CTxMemPool::addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry, return true; } +void CTxMemPool::addAddressIndex(const CTxMemPoolEntry &entry, const CCoinsViewCache &view) +{ + LOCK(cs); + const CTransaction& tx = entry.GetTx(); + std::vector inserted; + + uint256 txhash = tx.GetHash(); + for (unsigned int j = 0; j < tx.vin.size(); j++) { + const CTxIn input = tx.vin[j]; + const CTxOut &prevout = view.GetOutputFor(input); + if (prevout.scriptPubKey.IsPayToScriptHash()) { + vector hashBytes(prevout.scriptPubKey.begin()+2, prevout.scriptPubKey.begin()+22); + CMempoolAddressDeltaKey key(2, uint160(hashBytes), txhash, j, true); + CMempoolAddressDelta delta(entry.GetTime(), prevout.nValue * -1, input.prevout.hash, input.prevout.n); + mapAddress.insert(make_pair(key, delta)); + inserted.push_back(key); + } else if (prevout.scriptPubKey.IsPayToPublicKeyHash()) { + vector hashBytes(prevout.scriptPubKey.begin()+3, prevout.scriptPubKey.begin()+23); + CMempoolAddressDeltaKey key(1, uint160(hashBytes), txhash, j, true); + CMempoolAddressDelta delta(entry.GetTime(), prevout.nValue * -1, input.prevout.hash, input.prevout.n); + mapAddress.insert(make_pair(key, delta)); + inserted.push_back(key); + } + } + + for (unsigned int k = 0; k < tx.vout.size(); k++) { + const CTxOut &out = tx.vout[k]; + if (out.scriptPubKey.IsPayToScriptHash()) { + vector hashBytes(out.scriptPubKey.begin()+2, out.scriptPubKey.begin()+22); + CMempoolAddressDeltaKey key(2, uint160(hashBytes), txhash, k, false); + mapAddress.insert(make_pair(key, CMempoolAddressDelta(entry.GetTime(), out.nValue))); + inserted.push_back(key); + } else if (out.scriptPubKey.IsPayToPublicKeyHash()) { + vector hashBytes(out.scriptPubKey.begin()+3, out.scriptPubKey.begin()+23); + std::pair ret; + CMempoolAddressDeltaKey key(1, uint160(hashBytes), txhash, k, false); + mapAddress.insert(make_pair(key, CMempoolAddressDelta(entry.GetTime(), out.nValue))); + inserted.push_back(key); + } + } + + mapAddressInserted.insert(make_pair(txhash, inserted)); +} + +bool CTxMemPool::getAddressIndex(std::vector > &addresses, + std::vector > &results) +{ + LOCK(cs); + for (std::vector >::iterator it = addresses.begin(); it != addresses.end(); it++) { + addressDeltaMap::iterator ait = mapAddress.lower_bound(CMempoolAddressDeltaKey((*it).second, (*it).first)); + while (ait != mapAddress.end() && (*ait).first.addressBytes == (*it).first) { + results.push_back(*ait); + ait++; + } + } + return true; +} + +bool CTxMemPool::removeAddressIndex(const uint256 txhash) +{ + LOCK(cs); + addressDeltaMapInserted::iterator it = mapAddressInserted.find(txhash); + + if (it != mapAddressInserted.end()) { + std::vector keys = (*it).second; + for (std::vector::iterator mit = keys.begin(); mit != keys.end(); mit++) { + mapAddress.erase(*mit); + } + mapAddressInserted.erase(it); + } + + return true; +} + void CTxMemPool::removeUnchecked(txiter it) { const uint256 hash = it->GetTx().GetHash(); @@ -435,6 +509,7 @@ void CTxMemPool::removeUnchecked(txiter it) mapTx.erase(it); nTransactionsUpdated++; minerPolicyEstimator->removeTx(hash); + removeAddressIndex(hash); } // Calculates descendants of entry that are not already in setDescendants, and adds to diff --git a/src/txmempool.h b/src/txmempool.h index 5997346b0..57091a02e 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -9,6 +9,7 @@ #include #include +#include "addressindex.h" #include "amount.h" #include "coins.h" #include "primitives/transaction.h" @@ -419,6 +420,12 @@ private: typedef std::map txlinksMap; txlinksMap mapLinks; + typedef std::map addressDeltaMap; + addressDeltaMap mapAddress; + + typedef std::map > addressDeltaMapInserted; + addressDeltaMapInserted mapAddressInserted; + void UpdateParent(txiter entry, txiter parent, bool add); void UpdateChild(txiter entry, txiter child, bool add); @@ -450,6 +457,11 @@ public: bool addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry, bool fCurrentEstimate = true); bool addUnchecked(const uint256& hash, const CTxMemPoolEntry &entry, setEntries &setAncestors, bool fCurrentEstimate = true); + void addAddressIndex(const CTxMemPoolEntry &entry, const CCoinsViewCache &view); + bool getAddressIndex(std::vector > &addresses, + std::vector > &results); + bool removeAddressIndex(const uint256 txhash); + void remove(const CTransaction &tx, std::list& removed, bool fRecursive = false); void removeForReorg(const CCoinsViewCache *pcoins, unsigned int nMemPoolHeight, int flags); void removeConflicts(const CTransaction &tx, std::list& removed); From d99f17de1a708f67e473298f638cba04596f6c98 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 5 Apr 2016 10:49:11 -0400 Subject: [PATCH 29/59] rpc: give back base58 encoded address format in utxos --- src/rpcmisc.cpp | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index f06baa042..c79a4ccdd 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -523,8 +523,16 @@ UniValue getaddressutxos(const UniValue& params, bool fHelp) for (std::vector >::const_iterator it=unspentOutputs.begin(); it!=unspentOutputs.end(); it++) { UniValue output(UniValue::VOBJ); - output.push_back(Pair("addressType", (int)it->first.type)); - output.push_back(Pair("addressHash", it->first.hashBytes.GetHex())); + std::string address; + if (it->first.type == 2) { + address = CBitcoinAddress(CScriptID(it->first.hashBytes)).ToString(); + } else if (it->first.type == 1) { + address = CBitcoinAddress(CKeyID(it->first.hashBytes)).ToString(); + } else { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Unknown address type"); + } + + output.push_back(Pair("address", address)); output.push_back(Pair("txid", it->first.txhash.GetHex())); output.push_back(Pair("outputIndex", it->first.index)); output.push_back(Pair("script", HexStr(it->second.script.begin(), it->second.script.end()))); From 7c68235cf4a9c51b079660338e1fd9490dfa0859 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 5 Apr 2016 15:41:19 -0400 Subject: [PATCH 30/59] main: include timestampindex in help --- src/init.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/init.cpp b/src/init.cpp index 16598ada4..7f10a32e4 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -351,6 +351,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-txindex", strprintf(_("Maintain a full transaction index, used by the getrawtransaction rpc call (default: %u)"), DEFAULT_TXINDEX)); strUsage += HelpMessageOpt("-addressindex", strprintf(_("Maintain a full address index, used to query for the balance, txids and unspent outputs for addresses (default: %u)"), DEFAULT_ADDRESSINDEX)); + strUsage += HelpMessageOpt("-timestampindex", strprintf(_("Maintain a timestamp index for block hashes, used to query blocks hashes by a range of timestamps (default: %u)"), DEFAULT_TIMESTAMPINDEX)); strUsage += HelpMessageGroup(_("Connection options:")); strUsage += HelpMessageOpt("-addnode=", _("Add a node to connect to and attempt to keep the connection open")); From 4678f2d4389f9cce467f51a55f1da7d0e8debd63 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 7 Apr 2016 12:35:25 -0400 Subject: [PATCH 31/59] build: add addressindex.h to make --- src/Makefile.am | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Makefile.am b/src/Makefile.am index 52316a9fd..9632d6567 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -79,6 +79,7 @@ endif .PHONY: FORCE check-symbols check-security # bitcoin core # BITCOIN_CORE_H = \ + addressindex.h \ addrman.h \ alert.h \ amount.h \ From 8636f364770089418cb613403cbe1288da9842b6 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 7 Apr 2016 13:52:40 -0400 Subject: [PATCH 32/59] rpc: cast indexes to ints --- src/rpcmisc.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index c79a4ccdd..4b0d04ae9 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -534,7 +534,7 @@ UniValue getaddressutxos(const UniValue& params, bool fHelp) output.push_back(Pair("address", address)); output.push_back(Pair("txid", it->first.txhash.GetHex())); - output.push_back(Pair("outputIndex", it->first.index)); + output.push_back(Pair("outputIndex", (int)it->first.index)); output.push_back(Pair("script", HexStr(it->second.script.begin(), it->second.script.end()))); output.push_back(Pair("satoshis", it->second.satoshis)); output.push_back(Pair("height", it->second.blockHeight)); @@ -594,7 +594,7 @@ UniValue getaddressdeltas(const UniValue& params, bool fHelp) UniValue delta(UniValue::VOBJ); delta.push_back(Pair("satoshis", it->second)); delta.push_back(Pair("txid", it->first.txhash.GetHex())); - delta.push_back(Pair("index", it->first.index)); + delta.push_back(Pair("index", (int)it->first.index)); delta.push_back(Pair("height", it->first.blockHeight)); delta.push_back(Pair("hash", it->first.hashBytes.GetHex())); delta.push_back(Pair("type", (int)it->first.type)); From e3d9207e5acd3ec53176d8f0d455a541419429a5 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 5 Apr 2016 15:53:38 -0400 Subject: [PATCH 33/59] main: add spentindex option --- qa/pull-tester/rpc-tests.py | 1 + qa/rpc-tests/spentindex.py | 73 +++++++++++++++++++++++++++++++ src/init.cpp | 1 + src/main.cpp | 86 ++++++++++++++++++++++++++++--------- src/main.h | 65 ++++++++++++++++++++++++++++ src/rpcmisc.cpp | 39 +++++++++++++++++ src/rpcserver.cpp | 1 + src/rpcserver.h | 1 + src/txdb.cpp | 17 ++++++++ src/txdb.h | 4 ++ 10 files changed, 267 insertions(+), 21 deletions(-) create mode 100755 qa/rpc-tests/spentindex.py diff --git a/qa/pull-tester/rpc-tests.py b/qa/pull-tester/rpc-tests.py index 73bfe1e28..9c07091f7 100755 --- a/qa/pull-tester/rpc-tests.py +++ b/qa/pull-tester/rpc-tests.py @@ -99,6 +99,7 @@ testScripts = [ 'reindex.py', 'addressindex.py', 'timestampindex.py', + 'spentindex.py', 'decodescript.py', 'p2p-fullblocktest.py', 'blockchain.py', diff --git a/qa/rpc-tests/spentindex.py b/qa/rpc-tests/spentindex.py new file mode 100755 index 000000000..0fe98dcbd --- /dev/null +++ b/qa/rpc-tests/spentindex.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python2 +# Copyright (c) 2014-2015 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +# +# Test addressindex generation and fetching +# + +import time +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import * +from test_framework.script import * +from test_framework.mininode import * +import binascii + +class SpentIndexTest(BitcoinTestFramework): + + def setup_chain(self): + print("Initializing test directory "+self.options.tmpdir) + initialize_chain_clean(self.options.tmpdir, 4) + + def setup_network(self): + self.nodes = [] + # Nodes 0/1 are "wallet" nodes + self.nodes.append(start_node(0, self.options.tmpdir, ["-debug"])) + self.nodes.append(start_node(1, self.options.tmpdir, ["-debug", "-spentindex"])) + # Nodes 2/3 are used for testing + self.nodes.append(start_node(2, self.options.tmpdir, ["-debug", "-spentindex"])) + self.nodes.append(start_node(3, self.options.tmpdir, ["-debug", "-spentindex"])) + connect_nodes(self.nodes[0], 1) + connect_nodes(self.nodes[0], 2) + connect_nodes(self.nodes[0], 3) + + self.is_network_split = False + self.sync_all() + + def run_test(self): + print "Mining blocks..." + self.nodes[0].generate(105) + self.sync_all() + + chain_height = self.nodes[1].getblockcount() + assert_equal(chain_height, 105) + + # Check that + print "Testing spent index..." + + privkey = "cSdkPxkAjA4HDr5VHgsebAPDEh9Gyub4HK8UJr2DFGGqKKy4K5sG" + address = "mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW" + addressHash = "0b2f0a0c31bfe0406b0ccc1381fdbe311946dadc".decode("hex") + scriptPubKey = CScript([OP_DUP, OP_HASH160, addressHash, OP_EQUALVERIFY, OP_CHECKSIG]) + unspent = self.nodes[0].listunspent() + tx = CTransaction() + amount = unspent[0]["amount"] * 100000000 + tx.vin = [CTxIn(COutPoint(int(unspent[0]["txid"], 16), unspent[0]["vout"]))] + tx.vout = [CTxOut(amount, scriptPubKey)] + tx.rehash() + + signed_tx = self.nodes[0].signrawtransaction(binascii.hexlify(tx.serialize()).decode("utf-8")) + txid = self.nodes[0].sendrawtransaction(signed_tx["hex"], True) + self.nodes[0].generate(1) + self.sync_all() + + info = self.nodes[1].getspentinfo({"txid": unspent[0]["txid"], "index": unspent[0]["vout"]}) + assert_equal(info["txid"], txid) + assert_equal(info["index"], 0) + + print "Passed\n" + + +if __name__ == '__main__': + SpentIndexTest().main() diff --git a/src/init.cpp b/src/init.cpp index 7f10a32e4..389350092 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -352,6 +352,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-addressindex", strprintf(_("Maintain a full address index, used to query for the balance, txids and unspent outputs for addresses (default: %u)"), DEFAULT_ADDRESSINDEX)); strUsage += HelpMessageOpt("-timestampindex", strprintf(_("Maintain a timestamp index for block hashes, used to query blocks hashes by a range of timestamps (default: %u)"), DEFAULT_TIMESTAMPINDEX)); + strUsage += HelpMessageOpt("-spentindex", strprintf(_("Maintain a full spent index, used to query the spending txid and input index for an outpoint (default: %u)"), DEFAULT_SPENTINDEX)); strUsage += HelpMessageGroup(_("Connection options:")); strUsage += HelpMessageOpt("-addnode=", _("Add a node to connect to and attempt to keep the connection open")); diff --git a/src/main.cpp b/src/main.cpp index 5af73848b..e96d58aa6 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -68,6 +68,7 @@ bool fReindex = false; bool fTxIndex = false; bool fAddressIndex = false; bool fTimestampIndex = false; +bool fSpentIndex = false; bool fHavePruned = false; bool fPruneMode = false; bool fIsBareMultisigStd = DEFAULT_PERMIT_BAREMULTISIG; @@ -1454,6 +1455,17 @@ bool GetTimestampIndex(const unsigned int &high, const unsigned int &low, std::v return true; } +bool GetSpentIndex(CSpentIndexKey &key, CSpentIndexValue &value) +{ + if (!fSpentIndex) + return error("spent index not enabled"); + + if (!pblocktree->ReadSpentIndex(key, value)) + return error("unable to get spent info"); + + return true; +} + bool GetAddressIndex(uint160 addressHash, int type, std::vector > &addressIndex, int start, int end) { @@ -2058,6 +2070,7 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI std::vector > addressIndex; std::vector > addressUnspentIndex; + std::vector > spentIndex; // undo transactions in reverse order for (int i = block.vtx.size() - 1; i >= 0; i--) { @@ -2125,8 +2138,14 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI if (!ApplyTxInUndo(undo, view, out)) fClean = false; + const CTxIn input = tx.vin[j]; + + if (fSpentIndex) { + // undo and delete the spent index + spentIndex.push_back(make_pair(CSpentIndexKey(input.prevout.hash, input.prevout.n), CSpentIndexValue())); + } + if (fAddressIndex) { - const CTxIn input = tx.vin[j]; const CTxOut &prevout = view.GetOutputFor(tx.vin[j]); if (prevout.scriptPubKey.IsPayToScriptHash()) { vector hashBytes(prevout.scriptPubKey.begin()+2, prevout.scriptPubKey.begin()+22); @@ -2151,6 +2170,7 @@ bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockI continue; } } + } } } @@ -2435,6 +2455,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin blockundo.vtxundo.reserve(block.vtx.size() - 1); std::vector > addressIndex; std::vector > addressUnspentIndex; + std::vector > spentIndex; for (unsigned int i = 0; i < block.vtx.size(); i++) { @@ -2466,31 +2487,43 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin REJECT_INVALID, "bad-txns-nonfinal"); } - if (fAddressIndex) + if (fAddressIndex || fSpentIndex) { for (size_t j = 0; j < tx.vin.size(); j++) { + const CTxIn input = tx.vin[j]; - const CTxOut &prevout = view.GetOutputFor(tx.vin[j]); - if (prevout.scriptPubKey.IsPayToScriptHash()) { - vector hashBytes(prevout.scriptPubKey.begin()+2, prevout.scriptPubKey.begin()+22); - // record spending activity - addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, j, true), prevout.nValue * -1)); - - // remove address from unspent index - addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), input.prevout.hash, input.prevout.n), CAddressUnspentValue())); - } else if (prevout.scriptPubKey.IsPayToPublicKeyHash()) { - vector hashBytes(prevout.scriptPubKey.begin()+3, prevout.scriptPubKey.begin()+23); - - // record spending activity - addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, j, true), prevout.nValue * -1)); - - // remove address from unspent index - addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), input.prevout.hash, input.prevout.n), CAddressUnspentValue())); - - } else { - continue; + if (fSpentIndex) { + // add the spent index to determine the txid and input that spent an output + spentIndex.push_back(make_pair(CSpentIndexKey(input.prevout.hash, input.prevout.n), CSpentIndexValue(txhash, j, pindex->nHeight))); } + + if (fAddressIndex) { + + const CTxOut &prevout = view.GetOutputFor(tx.vin[j]); + + if (prevout.scriptPubKey.IsPayToScriptHash()) { + vector hashBytes(prevout.scriptPubKey.begin()+2, prevout.scriptPubKey.begin()+22); + + // record spending activity + addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, j, true), prevout.nValue * -1)); + + // remove address from unspent index + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), input.prevout.hash, input.prevout.n), CAddressUnspentValue())); + } else if (prevout.scriptPubKey.IsPayToPublicKeyHash()) { + vector hashBytes(prevout.scriptPubKey.begin()+3, prevout.scriptPubKey.begin()+23); + + // record spending activity + addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, j, true), prevout.nValue * -1)); + + // remove address from unspent index + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), input.prevout.hash, input.prevout.n), CAddressUnspentValue())); + + } else { + continue; + } + } + } } @@ -2604,6 +2637,10 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin } } + if (fSpentIndex) + if (!pblocktree->UpdateSpentIndex(spentIndex)) + return AbortNode(state, "Failed to write transaction index"); + if (fTimestampIndex) if (!pblocktree->WriteTimestampIndex(CTimestampIndexKey(pindex->nTime, pindex->GetBlockHash()))) return AbortNode(state, "Failed to write timestamp index"); @@ -4007,6 +4044,10 @@ bool static LoadBlockIndexDB() pblocktree->ReadFlag("timestampindex", fTimestampIndex); LogPrintf("%s: timestamp index %s\n", __func__, fTimestampIndex ? "enabled" : "disabled"); + // Check whether we have a spent index + pblocktree->ReadFlag("spentindex", fSpentIndex); + LogPrintf("%s: spent index %s\n", __func__, fSpentIndex ? "enabled" : "disabled"); + // Load pointer to end of best chain BlockMap::iterator it = mapBlockIndex.find(pcoinsTip->GetBestBlock()); if (it == mapBlockIndex.end()) @@ -4176,6 +4217,9 @@ bool InitBlockIndex(const CChainParams& chainparams) fTimestampIndex = GetBoolArg("-timestampindex", DEFAULT_TIMESTAMPINDEX); pblocktree->WriteFlag("timestampindex", fTimestampIndex); + fSpentIndex = GetBoolArg("-spentindex", DEFAULT_SPENTINDEX); + pblocktree->WriteFlag("spentindex", fSpentIndex); + LogPrintf("Initializing databases...\n"); // Only add the genesis block if not reindexing (in which case we reuse the one already on disk) diff --git a/src/main.h b/src/main.h index 90ed55e92..7e54909c1 100644 --- a/src/main.h +++ b/src/main.h @@ -113,6 +113,7 @@ static const bool DEFAULT_CHECKPOINTS_ENABLED = true; static const bool DEFAULT_TXINDEX = false; static const bool DEFAULT_ADDRESSINDEX = false; static const bool DEFAULT_TIMESTAMPINDEX = false; +static const bool DEFAULT_SPENTINDEX = false; static const unsigned int DEFAULT_BANSCORE_THRESHOLD = 100; static const bool DEFAULT_TESTSAFEMODE = false; @@ -292,6 +293,69 @@ struct CNodeStateStats { std::vector vHeightInFlight; }; +struct CSpentIndexKey { + uint256 txid; + unsigned int outputIndex; + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + READWRITE(txid); + READWRITE(outputIndex); + } + + CSpentIndexKey(uint256 t, unsigned int i) { + txid = t; + outputIndex = i; + } + + CSpentIndexKey() { + SetNull(); + } + + void SetNull() { + txid.SetNull(); + outputIndex = 0; + } + +}; + +struct CSpentIndexValue { + uint256 txid; + unsigned int inputIndex; + int blockHeight; + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + READWRITE(txid); + READWRITE(inputIndex); + READWRITE(blockHeight); + } + + CSpentIndexValue(uint256 t, unsigned int i, int h) { + txid = t; + inputIndex = i; + blockHeight = h; + } + + CSpentIndexValue() { + SetNull(); + } + + void SetNull() { + txid.SetNull(); + inputIndex = 0; + blockHeight = 0; + } + + bool IsNull() const { + return txid.IsNull(); + } +}; + struct CTimestampIndexIteratorKey { unsigned int timestamp; @@ -676,6 +740,7 @@ public: }; bool GetTimestampIndex(const unsigned int &high, const unsigned int &low, std::vector &hashes); +bool GetSpentIndex(CSpentIndexKey &key, CSpentIndexValue &value); bool GetAddressIndex(uint160 addressHash, int type, std::vector > &addressIndex, int start = 0, int end = 0); diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index 4b0d04ae9..b31864993 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -701,3 +701,42 @@ UniValue getaddresstxids(const UniValue& params, bool fHelp) return result; } + +UniValue getspentinfo(const UniValue& params, bool fHelp) +{ + + if (fHelp || params.size() != 1 || !params[0].isObject()) + throw runtime_error( + "getspentinfo\n" + "\nReturns the txid and index where an output is spent.\n" + "\nResult\n" + "{\n" + " \"txid\" (string) The transaction id\n" + " \"index\" (number) The spending input index\n" + " ,...\n" + "}\n" + ); + + UniValue txidValue = find_value(params[0].get_obj(), "txid"); + UniValue indexValue = find_value(params[0].get_obj(), "index"); + + if (!txidValue.isStr() || !indexValue.isNum()) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid txid or index"); + } + + uint256 txid = ParseHashV(txidValue, "txid"); + int outputIndex = indexValue.get_int(); + + CSpentIndexKey key(txid, outputIndex); + CSpentIndexValue value; + + if (!GetSpentIndex(key, value)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Unable to get spent info"); + } + + UniValue obj(UniValue::VOBJ); + obj.push_back(Pair("txid", value.txid.GetHex())); + obj.push_back(Pair("index", (int)value.inputIndex)); + + return obj; +} diff --git a/src/rpcserver.cpp b/src/rpcserver.cpp index 1d5399df3..6717592fa 100644 --- a/src/rpcserver.cpp +++ b/src/rpcserver.cpp @@ -290,6 +290,7 @@ static const CRPCCommand vRPCCommands[] = { "blockchain", "verifytxoutproof", &verifytxoutproof, true }, { "blockchain", "gettxoutsetinfo", &gettxoutsetinfo, true }, { "blockchain", "verifychain", &verifychain, true }, + { "blockchain", "getspentinfo", &getspentinfo, true }, /* Mining */ { "mining", "getblocktemplate", &getblocktemplate, true }, diff --git a/src/rpcserver.h b/src/rpcserver.h index 74ecd1191..581370c26 100644 --- a/src/rpcserver.h +++ b/src/rpcserver.h @@ -271,6 +271,7 @@ extern UniValue verifychain(const UniValue& params, bool fHelp); extern UniValue getchaintips(const UniValue& params, bool fHelp); extern UniValue invalidateblock(const UniValue& params, bool fHelp); extern UniValue reconsiderblock(const UniValue& params, bool fHelp); +extern UniValue getspentinfo(const UniValue& params, bool fHelp); bool StartRPC(); void InterruptRPC(); diff --git a/src/txdb.cpp b/src/txdb.cpp index 8e485e9eb..d3a29d5fd 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -24,6 +24,7 @@ static const char DB_TXINDEX = 't'; static const char DB_ADDRESSINDEX = 'a'; static const char DB_ADDRESSUNSPENTINDEX = 'u'; static const char DB_TIMESTAMPINDEX = 's'; +static const char DB_SPENTINDEX = 'p'; static const char DB_BLOCK_INDEX = 'b'; static const char DB_BEST_BLOCK = 'B'; @@ -166,6 +167,22 @@ bool CBlockTreeDB::WriteTxIndex(const std::vector return WriteBatch(batch); } +bool CBlockTreeDB::ReadSpentIndex(CSpentIndexKey &key, CSpentIndexValue &value) { + return Read(make_pair(DB_SPENTINDEX, key), value); +} + +bool CBlockTreeDB::UpdateSpentIndex(const std::vector >&vect) { + CDBBatch batch(&GetObfuscateKey()); + for (std::vector >::const_iterator it=vect.begin(); it!=vect.end(); it++) { + if (it->second.IsNull()) { + batch.Erase(make_pair(DB_SPENTINDEX, it->first)); + } else { + batch.Write(make_pair(DB_SPENTINDEX, it->first), it->second); + } + } + return WriteBatch(batch); +} + bool CBlockTreeDB::UpdateAddressUnspentIndex(const std::vector >&vect) { CDBBatch batch(&GetObfuscateKey()); for (std::vector >::const_iterator it=vect.begin(); it!=vect.end(); it++) { diff --git a/src/txdb.h b/src/txdb.h index 547a48fa8..62b115707 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -23,6 +23,8 @@ struct CAddressIndexKey; struct CAddressIndexIteratorKey; struct CTimestampIndexKey; struct CTimestampIndexIteratorKey; +struct CSpentIndexKey; +struct CSpentIndexValue; class uint256; //! -dbcache default (MiB) @@ -63,6 +65,8 @@ public: bool ReadReindexing(bool &fReindex); bool ReadTxIndex(const uint256 &txid, CDiskTxPos &pos); bool WriteTxIndex(const std::vector > &list); + bool ReadSpentIndex(CSpentIndexKey &key, CSpentIndexValue &value); + bool UpdateSpentIndex(const std::vector >&vect); bool UpdateAddressUnspentIndex(const std::vector >&vect); bool ReadAddressUnspentIndex(uint160 addressHash, int type, std::vector > &vect); From b752fbe09e73e22101c151f139c2001f08d9bd00 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 12 Apr 2016 12:31:21 -0400 Subject: [PATCH 34/59] rpc: include spent info if spentindex enabled with getrawtransaction verbose --- qa/rpc-tests/spentindex.py | 8 +++++++- src/main.cpp | 2 +- src/rpcrawtransaction.cpp | 12 +++++++++++- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/qa/rpc-tests/spentindex.py b/qa/rpc-tests/spentindex.py index 0fe98dcbd..44ba28660 100755 --- a/qa/rpc-tests/spentindex.py +++ b/qa/rpc-tests/spentindex.py @@ -27,7 +27,7 @@ class SpentIndexTest(BitcoinTestFramework): self.nodes.append(start_node(1, self.options.tmpdir, ["-debug", "-spentindex"])) # Nodes 2/3 are used for testing self.nodes.append(start_node(2, self.options.tmpdir, ["-debug", "-spentindex"])) - self.nodes.append(start_node(3, self.options.tmpdir, ["-debug", "-spentindex"])) + self.nodes.append(start_node(3, self.options.tmpdir, ["-debug", "-spentindex", "-txindex"])) connect_nodes(self.nodes[0], 1) connect_nodes(self.nodes[0], 2) connect_nodes(self.nodes[0], 3) @@ -62,10 +62,16 @@ class SpentIndexTest(BitcoinTestFramework): self.nodes[0].generate(1) self.sync_all() + # Check that the spentinfo works standalone info = self.nodes[1].getspentinfo({"txid": unspent[0]["txid"], "index": unspent[0]["vout"]}) assert_equal(info["txid"], txid) assert_equal(info["index"], 0) + # Check that verbose raw transaction includes spent info + txVerbose = self.nodes[3].getrawtransaction(unspent[0]["txid"], 1) + assert_equal(txVerbose["vout"][unspent[0]["vout"]]["spentTxId"], txid) + assert_equal(txVerbose["vout"][unspent[0]["vout"]]["spentIndex"], 0) + print "Passed\n" diff --git a/src/main.cpp b/src/main.cpp index e96d58aa6..d750477e9 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1458,7 +1458,7 @@ bool GetTimestampIndex(const unsigned int &high, const unsigned int &low, std::v bool GetSpentIndex(CSpentIndexKey &key, CSpentIndexValue &value) { if (!fSpentIndex) - return error("spent index not enabled"); + return false; if (!pblocktree->ReadSpentIndex(key, value)) return error("unable to get spent info"); diff --git a/src/rpcrawtransaction.cpp b/src/rpcrawtransaction.cpp index 6ab1807d4..0e8df7fae 100644 --- a/src/rpcrawtransaction.cpp +++ b/src/rpcrawtransaction.cpp @@ -61,7 +61,8 @@ void ScriptPubKeyToJSON(const CScript& scriptPubKey, UniValue& out, bool fInclud void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry) { - entry.push_back(Pair("txid", tx.GetHash().GetHex())); + uint256 txid = tx.GetHash(); + entry.push_back(Pair("txid", txid.GetHex())); entry.push_back(Pair("size", (int)::GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION))); entry.push_back(Pair("version", tx.nVersion)); entry.push_back(Pair("locktime", (int64_t)tx.nLockTime)); @@ -91,6 +92,15 @@ void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry) UniValue o(UniValue::VOBJ); ScriptPubKeyToJSON(txout.scriptPubKey, o, true); out.push_back(Pair("scriptPubKey", o)); + + // Add spent information if spentindex is enabled + CSpentIndexValue spentInfo; + CSpentIndexKey spentKey(txid, i); + if (GetSpentIndex(spentKey, spentInfo)) { + out.push_back(Pair("spentTxId", spentInfo.txid.GetHex())); + out.push_back(Pair("spentIndex", (int)spentInfo.inputIndex)); + } + vout.push_back(out); } entry.push_back(Pair("vout", vout)); From abe40712ce25f2383e60db9bfeb59d02473502ec Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 12 Apr 2016 15:31:19 -0400 Subject: [PATCH 35/59] rpc: include height in spentinfo --- qa/rpc-tests/spentindex.py | 2 ++ src/rpcmisc.cpp | 1 + src/rpcrawtransaction.cpp | 1 + 3 files changed, 4 insertions(+) diff --git a/qa/rpc-tests/spentindex.py b/qa/rpc-tests/spentindex.py index 44ba28660..6669df462 100755 --- a/qa/rpc-tests/spentindex.py +++ b/qa/rpc-tests/spentindex.py @@ -66,11 +66,13 @@ class SpentIndexTest(BitcoinTestFramework): info = self.nodes[1].getspentinfo({"txid": unspent[0]["txid"], "index": unspent[0]["vout"]}) assert_equal(info["txid"], txid) assert_equal(info["index"], 0) + assert_equal(info["height"], 106) # Check that verbose raw transaction includes spent info txVerbose = self.nodes[3].getrawtransaction(unspent[0]["txid"], 1) assert_equal(txVerbose["vout"][unspent[0]["vout"]]["spentTxId"], txid) assert_equal(txVerbose["vout"][unspent[0]["vout"]]["spentIndex"], 0) + assert_equal(txVerbose["vout"][unspent[0]["vout"]]["spentHeight"], 106) print "Passed\n" diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index b31864993..fc55f3df2 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -737,6 +737,7 @@ UniValue getspentinfo(const UniValue& params, bool fHelp) UniValue obj(UniValue::VOBJ); obj.push_back(Pair("txid", value.txid.GetHex())); obj.push_back(Pair("index", (int)value.inputIndex)); + obj.push_back(Pair("height", value.blockHeight)); return obj; } diff --git a/src/rpcrawtransaction.cpp b/src/rpcrawtransaction.cpp index 0e8df7fae..6fbec480e 100644 --- a/src/rpcrawtransaction.cpp +++ b/src/rpcrawtransaction.cpp @@ -99,6 +99,7 @@ void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry) if (GetSpentIndex(spentKey, spentInfo)) { out.push_back(Pair("spentTxId", spentInfo.txid.GetHex())); out.push_back(Pair("spentIndex", (int)spentInfo.inputIndex)); + out.push_back(Pair("spentHeight", spentInfo.blockHeight)); } vout.push_back(out); From 96d8307b559535fcf0809c4734b83895a5f8ab43 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 12 Apr 2016 16:04:10 -0400 Subject: [PATCH 36/59] rpc: query txids for addresses within block height range --- qa/rpc-tests/addressindex.py | 10 ++++++++++ src/rpcmisc.cpp | 21 +++++++++++++++++++-- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index d28cd393f..3ef5bcf67 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -84,6 +84,16 @@ class AddressIndexTest(BitcoinTestFramework): assert_equal(txidsb[1], txidb1) assert_equal(txidsb[2], txidb2) + # Check that limiting by height works + chain_height = self.nodes[1].getblockcount() + height_txids = self.nodes[1].getaddresstxids({ + "addresses": ["2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br"], + "start": 111, + "end": 111 + }) + assert_equal(len(height_txids), 1) + assert_equal(height_txids[0], txidb2) + # Check that multiple addresses works multitxids = self.nodes[1].getaddresstxids({"addresses": ["2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br", "mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs"]}) assert_equal(len(multitxids), 6) diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index fc55f3df2..9e5266e31 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -668,11 +668,28 @@ UniValue getaddresstxids(const UniValue& params, bool fHelp) throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); } + int start = 0; + int end = 0; + if (params[0].isObject()) { + UniValue startValue = find_value(params[0].get_obj(), "start"); + UniValue endValue = find_value(params[0].get_obj(), "end"); + if (startValue.isNum() && endValue.isNum()) { + start = startValue.get_int(); + end = startValue.get_int(); + } + } + std::vector > addressIndex; for (std::vector >::iterator it = addresses.begin(); it != addresses.end(); it++) { - if (!GetAddressIndex((*it).first, (*it).second, addressIndex)) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); + if (start > 0 && end > 0) { + if (!GetAddressIndex((*it).first, (*it).second, addressIndex, start, end)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); + } + } else { + if (!GetAddressIndex((*it).first, (*it).second, addressIndex)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); + } } } From 94ea69a4d59852b06c89b484a3dab324ff0c39fa Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 12 Apr 2016 20:33:18 -0400 Subject: [PATCH 37/59] rpc: fix issue with querying txids by block heights --- qa/rpc-tests/addressindex.py | 11 +++---- src/main.h | 58 ++++++++++++++++++++++++------------ src/rpcmisc.cpp | 2 +- src/txdb.cpp | 2 +- src/txdb.h | 1 + 5 files changed, 48 insertions(+), 26 deletions(-) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index 3ef5bcf67..e45efdbda 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -85,14 +85,15 @@ class AddressIndexTest(BitcoinTestFramework): assert_equal(txidsb[2], txidb2) # Check that limiting by height works - chain_height = self.nodes[1].getblockcount() + print "Testing querying txids by range of block heights.." height_txids = self.nodes[1].getaddresstxids({ "addresses": ["2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br"], - "start": 111, - "end": 111 + "start": 105, + "end": 110 }) - assert_equal(len(height_txids), 1) - assert_equal(height_txids[0], txidb2) + assert_equal(len(height_txids), 2) + assert_equal(height_txids[0], txidb0) + assert_equal(height_txids[1], txidb1) # Check that multiple addresses works multitxids = self.nodes[1].getaddresstxids({"addresses": ["2N2JD6wb56AfK4tfmM6PwdVmoYk2dCKf4Br", "mo9ncXisMeAoXwqcV5EWuyncbmCcQN4rVs"]}) diff --git a/src/main.h b/src/main.h index 7e54909c1..6c685e68f 100644 --- a/src/main.h +++ b/src/main.h @@ -561,23 +561,14 @@ struct CAddressIndexKey { struct CAddressIndexIteratorKey { unsigned int type; uint160 hashBytes; - bool includeHeight; - int blockHeight; size_t GetSerializeSize(int nType, int nVersion) const { - if (includeHeight) { - return 25; - } else { - return 21; - } + return 21; } template void Serialize(Stream& s, int nType, int nVersion) const { ser_writedata8(s, type); hashBytes.Serialize(s, nType, nVersion); - if (includeHeight) { - ser_writedata32be(s, blockHeight); - } } template void Unserialize(Stream& s, int nType, int nVersion) { @@ -588,14 +579,6 @@ struct CAddressIndexIteratorKey { CAddressIndexIteratorKey(unsigned int addressType, uint160 addressHash) { type = addressType; hashBytes = addressHash; - includeHeight = false; - } - - CAddressIndexIteratorKey(unsigned int addressType, uint160 addressHash, int height) { - type = addressType; - hashBytes = addressHash; - blockHeight = height; - includeHeight = true; } CAddressIndexIteratorKey() { @@ -605,7 +588,44 @@ struct CAddressIndexIteratorKey { void SetNull() { type = 0; hashBytes.SetNull(); - includeHeight = false; + } +}; + +struct CAddressIndexIteratorHeightKey { + unsigned int type; + uint160 hashBytes; + int blockHeight; + + size_t GetSerializeSize(int nType, int nVersion) const { + return 25; + } + template + void Serialize(Stream& s, int nType, int nVersion) const { + ser_writedata8(s, type); + hashBytes.Serialize(s, nType, nVersion); + ser_writedata32be(s, blockHeight); + } + template + void Unserialize(Stream& s, int nType, int nVersion) { + type = ser_readdata8(s); + hashBytes.Unserialize(s, nType, nVersion); + blockHeight = ser_readdata32be(s); + } + + CAddressIndexIteratorHeightKey(unsigned int addressType, uint160 addressHash, int height) { + type = addressType; + hashBytes = addressHash; + blockHeight = height; + } + + CAddressIndexIteratorHeightKey() { + SetNull(); + } + + void SetNull() { + type = 0; + hashBytes.SetNull(); + blockHeight = 0; } }; diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index 9e5266e31..91a45813a 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -675,7 +675,7 @@ UniValue getaddresstxids(const UniValue& params, bool fHelp) UniValue endValue = find_value(params[0].get_obj(), "end"); if (startValue.isNum() && endValue.isNum()) { start = startValue.get_int(); - end = startValue.get_int(); + end = endValue.get_int(); } } diff --git a/src/txdb.cpp b/src/txdb.cpp index d3a29d5fd..8a4d71578 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -242,7 +242,7 @@ bool CBlockTreeDB::ReadAddressIndex(uint160 addressHash, int type, boost::scoped_ptr pcursor(NewIterator()); if (start > 0 && end > 0) { - pcursor->Seek(make_pair(DB_ADDRESSINDEX, CAddressIndexIteratorKey(type, addressHash, start))); + pcursor->Seek(make_pair(DB_ADDRESSINDEX, CAddressIndexIteratorHeightKey(type, addressHash, start))); } else { pcursor->Seek(make_pair(DB_ADDRESSINDEX, CAddressIndexIteratorKey(type, addressHash))); } diff --git a/src/txdb.h b/src/txdb.h index 62b115707..14d501278 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -21,6 +21,7 @@ struct CAddressUnspentKey; struct CAddressUnspentValue; struct CAddressIndexKey; struct CAddressIndexIteratorKey; +struct CAddressIndexIteratorHeightKey; struct CTimestampIndexKey; struct CTimestampIndexIteratorKey; struct CSpentIndexKey; From 04da9306c62c062536a309e99178c9b742a01560 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 12 Apr 2016 23:25:55 -0400 Subject: [PATCH 38/59] build: fix darwin build --- src/addressindex.h | 2 +- src/txdb.cpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/addressindex.h b/src/addressindex.h index b6e3587c1..43b49dca9 100644 --- a/src/addressindex.h +++ b/src/addressindex.h @@ -57,7 +57,7 @@ struct CMempoolAddressDeltaKey struct CMempoolAddressDeltaKeyCompare { - bool operator()(const CMempoolAddressDeltaKey& a, const CMempoolAddressDeltaKey& b) { + bool operator()(const CMempoolAddressDeltaKey& a, const CMempoolAddressDeltaKey& b) const { if (a.type == b.type) { if (a.addressBytes == b.addressBytes) { if (a.txhash == b.txhash) { diff --git a/src/txdb.cpp b/src/txdb.cpp index 8a4d71578..48150198f 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -271,7 +271,7 @@ bool CBlockTreeDB::ReadAddressIndex(uint160 addressHash, int type, bool CBlockTreeDB::WriteTimestampIndex(const CTimestampIndexKey ×tampIndex) { CDBBatch batch(&GetObfuscateKey()); - batch.Write(make_pair(DB_TIMESTAMPINDEX, timestampIndex), NULL); + batch.Write(make_pair(DB_TIMESTAMPINDEX, timestampIndex), 0); return WriteBatch(batch); } From 5c3cf5f631a427fbe05dd9edc10bda6ddf02358a Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 21 Apr 2016 15:07:01 -0400 Subject: [PATCH 39/59] rpc: include prevhash and prevout information for spending deltas --- src/rpcmisc.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index 91a45813a..05455fb8b 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -478,6 +478,10 @@ UniValue getaddressmempool(const UniValue& params, bool fHelp) delta.push_back(Pair("index", (int)it->first.index)); delta.push_back(Pair("satoshis", it->second.amount)); delta.push_back(Pair("timestamp", it->second.time)); + if (it->second.amount < 0) { + delta.push_back(Pair("prevtxid", it->second.prevhash.GetHex())); + delta.push_back(Pair("prevout", (int)it->second.prevout)); + } result.push_back(delta); } From 28f9ae7853b0f335334b029029fe671eb59a886f Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 21 Apr 2016 15:23:52 -0400 Subject: [PATCH 40/59] test: test for getaddressmempool prevhash and prevout values --- qa/rpc-tests/addressindex.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index e45efdbda..c6910cd1e 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -256,6 +256,20 @@ class AddressIndexTest(BitcoinTestFramework): mempool2 = self.nodes[2].getaddressmempool({"addresses": [address3]}) assert_equal(len(mempool2), 0) + tx = CTransaction() + tx.vin = [CTxIn(COutPoint(int(memtxid2, 16), 0))] + tx.vout = [CTxOut(amount - 10000, scriptPubKey2)] + tx.rehash() + self.nodes[2].importprivkey(privKey3) + signed_tx3 = self.nodes[2].signrawtransaction(binascii.hexlify(tx.serialize()).decode("utf-8")) + memtxid3 = self.nodes[2].sendrawtransaction(signed_tx3["hex"], True) + time.sleep(2) + + mempool3 = self.nodes[2].getaddressmempool({"addresses": [address3]}) + assert_equal(len(mempool3), 1) + assert_equal(mempool3[0]["prevtxid"], memtxid2) + assert_equal(mempool3[0]["prevout"], 0) + print "Passed\n" From 8391ff0b0ab51f25d0930084ce1a5ccef7b6eba0 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 21 Apr 2016 15:59:51 -0400 Subject: [PATCH 41/59] rpc: include base58check encoded address in results --- qa/rpc-tests/addressindex.py | 2 ++ src/rpcmisc.cpp | 37 +++++++++++++++++++++++++----------- 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index c6910cd1e..7ec00cc4f 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -176,6 +176,7 @@ class AddressIndexTest(BitcoinTestFramework): for delta in deltas: balance3 += delta["satoshis"] assert_equal(balance3, change_amount) + assert_equal(deltas[0]["address"], address2) # Check that deltas can be returned from range of block heights deltas = self.nodes[1].getaddressdeltas({"addresses": [address2], "start": 113, "end": 113}) @@ -250,6 +251,7 @@ class AddressIndexTest(BitcoinTestFramework): assert_equal(len(mempool), 2) assert_equal(mempool[0]["txid"], memtxid1) assert_equal(mempool[1]["txid"], memtxid2) + assert_equal(mempool[0]["address"], address3) self.nodes[2].generate(1); self.sync_all(); diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index 05455fb8b..8f5a741e2 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -398,6 +398,18 @@ UniValue setmocktime(const UniValue& params, bool fHelp) return NullUniValue; } +bool getAddressFromIndex(const int &type, const uint160 &hash, std::string &address) +{ + if (type == 2) { + address = CBitcoinAddress(CScriptID(hash)).ToString(); + } else if (type == 1) { + address = CBitcoinAddress(CKeyID(hash)).ToString(); + } else { + return false; + } + return true; +} + bool getAddressesFromParams(const UniValue& params, std::vector > &addresses) { if (params[0].isStr()) { @@ -471,9 +483,13 @@ UniValue getaddressmempool(const UniValue& params, bool fHelp) for (std::vector >::iterator it = indexes.begin(); it != indexes.end(); it++) { + std::string address; + if (!getAddressFromIndex(it->first.type, it->first.addressBytes, address)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Unknown address type"); + } + UniValue delta(UniValue::VOBJ); - delta.push_back(Pair("addressType", (int)it->first.type)); - delta.push_back(Pair("addressHash", it->first.addressBytes.GetHex())); + delta.push_back(Pair("address", address)); delta.push_back(Pair("txid", it->first.txhash.GetHex())); delta.push_back(Pair("index", (int)it->first.index)); delta.push_back(Pair("satoshis", it->second.amount)); @@ -528,11 +544,7 @@ UniValue getaddressutxos(const UniValue& params, bool fHelp) for (std::vector >::const_iterator it=unspentOutputs.begin(); it!=unspentOutputs.end(); it++) { UniValue output(UniValue::VOBJ); std::string address; - if (it->first.type == 2) { - address = CBitcoinAddress(CScriptID(it->first.hashBytes)).ToString(); - } else if (it->first.type == 1) { - address = CBitcoinAddress(CKeyID(it->first.hashBytes)).ToString(); - } else { + if (!getAddressFromIndex(it->first.type, it->first.hashBytes, address)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Unknown address type"); } @@ -561,8 +573,7 @@ UniValue getaddressdeltas(const UniValue& params, bool fHelp) " \"txid\" (string) The related txid\n" " \"index\" (number) The related input or output index\n" " \"height\" (number) The block height\n" - " \"hash\" (string) The address hash\n" - " \"type\" (number) The address type 0 for pubkeyhash 1 for scripthash\n" + " \"address\" (string) The base58check encoded address\n" " }\n" "]\n" ); @@ -595,13 +606,17 @@ UniValue getaddressdeltas(const UniValue& params, bool fHelp) UniValue result(UniValue::VARR); for (std::vector >::const_iterator it=addressIndex.begin(); it!=addressIndex.end(); it++) { + std::string address; + if (!getAddressFromIndex(it->first.type, it->first.hashBytes, address)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Unknown address type"); + } + UniValue delta(UniValue::VOBJ); delta.push_back(Pair("satoshis", it->second)); delta.push_back(Pair("txid", it->first.txhash.GetHex())); delta.push_back(Pair("index", (int)it->first.index)); delta.push_back(Pair("height", it->first.blockHeight)); - delta.push_back(Pair("hash", it->first.hashBytes.GetHex())); - delta.push_back(Pair("type", (int)it->first.type)); + delta.push_back(Pair("address", address)); result.push_back(delta); } From 98f8fdddc858f86e48d09b4b1a289749dc66bda2 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 21 Apr 2016 16:07:42 -0400 Subject: [PATCH 42/59] rpc: optional "start" and "end" params for getaddressdeltas --- qa/rpc-tests/addressindex.py | 4 ++++ src/rpcmisc.cpp | 24 +++++++++++++++++------- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index 7ec00cc4f..a07d4de1e 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -178,6 +178,10 @@ class AddressIndexTest(BitcoinTestFramework): assert_equal(balance3, change_amount) assert_equal(deltas[0]["address"], address2) + # Check that entire range will be queried + deltasAll = self.nodes[1].getaddressdeltas({"addresses": [address2]}) + assert_equal(len(deltasAll), len(deltas)) + # Check that deltas can be returned from range of block heights deltas = self.nodes[1].getaddressdeltas({"addresses": [address2], "start": 113, "end": 113}) assert_equal(len(deltas), 1) diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index 8f5a741e2..04eb5ecef 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -582,12 +582,16 @@ UniValue getaddressdeltas(const UniValue& params, bool fHelp) UniValue startValue = find_value(params[0].get_obj(), "start"); UniValue endValue = find_value(params[0].get_obj(), "end"); - if (!startValue.isNum() || !endValue.isNum()) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Start and end values are expected to be block heights"); - } + int start = 0; + int end = 0; - int start = startValue.get_int(); - int end = startValue.get_int(); + if (startValue.isNum() && endValue.isNum()) { + start = startValue.get_int(); + end = endValue.get_int(); + if (end < start) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "End value is expected to be greater than start"); + } + } std::vector > addresses; @@ -598,8 +602,14 @@ UniValue getaddressdeltas(const UniValue& params, bool fHelp) std::vector > addressIndex; for (std::vector >::iterator it = addresses.begin(); it != addresses.end(); it++) { - if (!GetAddressIndex((*it).first, (*it).second, addressIndex, start, end)) { - throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); + if (start > 0 && end > 0) { + if (!GetAddressIndex((*it).first, (*it).second, addressIndex, start, end)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); + } + } else { + if (!GetAddressIndex((*it).first, (*it).second, addressIndex)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); + } } } From 5fa85bc9ae38528ee7b69f272f3bc3ea83c90599 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 25 Apr 2016 13:08:52 -0400 Subject: [PATCH 43/59] rpc: update oksafemode for address commands --- src/rpcserver.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/rpcserver.cpp b/src/rpcserver.cpp index 6717592fa..eb3060c51 100644 --- a/src/rpcserver.cpp +++ b/src/rpcserver.cpp @@ -290,7 +290,7 @@ static const CRPCCommand vRPCCommands[] = { "blockchain", "verifytxoutproof", &verifytxoutproof, true }, { "blockchain", "gettxoutsetinfo", &gettxoutsetinfo, true }, { "blockchain", "verifychain", &verifychain, true }, - { "blockchain", "getspentinfo", &getspentinfo, true }, + { "blockchain", "getspentinfo", &getspentinfo, false }, /* Mining */ { "mining", "getblocktemplate", &getblocktemplate, true }, @@ -316,7 +316,7 @@ static const CRPCCommand vRPCCommands[] = #endif /* Address index */ - { "addressindex", "getaddressmempool", &getaddressmempool, false }, + { "addressindex", "getaddressmempool", &getaddressmempool, true }, { "addressindex", "getaddressutxos", &getaddressutxos, false }, { "addressindex", "getaddressdeltas", &getaddressdeltas, false }, { "addressindex", "getaddresstxids", &getaddresstxids, false }, From 9c5b709c6f5093e602d5a69a58301e5903d73b18 Mon Sep 17 00:00:00 2001 From: "Wladimir J. van der Laan" Date: Tue, 9 Feb 2016 12:37:05 +0100 Subject: [PATCH 44/59] tests: Make proxy_test work on travis servers without IPv6 Github-Pull: #7489 Rebased-From: 7539f1aae3b41279dc5d49e09f448a78a071e114 Cherry-picked-From: 9ca957bcd401de69c4c03904b9ee8b8b41052905 --- qa/rpc-tests/proxy_test.py | 74 +++++++++++++++----------- qa/rpc-tests/test_framework/netutil.py | 15 ++++++ 2 files changed, 58 insertions(+), 31 deletions(-) diff --git a/qa/rpc-tests/proxy_test.py b/qa/rpc-tests/proxy_test.py index 7f77e664d..b3c65573e 100755 --- a/qa/rpc-tests/proxy_test.py +++ b/qa/rpc-tests/proxy_test.py @@ -7,6 +7,7 @@ import socket from test_framework.socks5 import Socks5Configuration, Socks5Command, Socks5Server, AddressType from test_framework.test_framework import BitcoinTestFramework from test_framework.util import * +from test_framework.netutil import test_ipv6_local ''' Test plan: - Start bitcoind's with different proxy configurations @@ -34,6 +35,7 @@ addnode connect to generic DNS name class ProxyTest(BitcoinTestFramework): def __init__(self): + self.have_ipv6 = test_ipv6_local() # Create two proxies on different ports # ... one unauthenticated self.conf1 = Socks5Configuration() @@ -45,29 +47,36 @@ class ProxyTest(BitcoinTestFramework): self.conf2.addr = ('127.0.0.1', 14000 + (os.getpid() % 1000)) self.conf2.unauth = True self.conf2.auth = True - # ... one on IPv6 with similar configuration - self.conf3 = Socks5Configuration() - self.conf3.af = socket.AF_INET6 - self.conf3.addr = ('::1', 15000 + (os.getpid() % 1000)) - self.conf3.unauth = True - self.conf3.auth = True + if self.have_ipv6: + # ... one on IPv6 with similar configuration + self.conf3 = Socks5Configuration() + self.conf3.af = socket.AF_INET6 + self.conf3.addr = ('::1', 15000 + (os.getpid() % 1000)) + self.conf3.unauth = True + self.conf3.auth = True + else: + print "Warning: testing without local IPv6 support" self.serv1 = Socks5Server(self.conf1) self.serv1.start() self.serv2 = Socks5Server(self.conf2) self.serv2.start() - self.serv3 = Socks5Server(self.conf3) - self.serv3.start() + if self.have_ipv6: + self.serv3 = Socks5Server(self.conf3) + self.serv3.start() def setup_nodes(self): # Note: proxies are not used to connect to local nodes # this is because the proxy to use is based on CService.GetNetwork(), which return NET_UNROUTABLE for localhost - return start_nodes(4, self.options.tmpdir, extra_args=[ + args = [ ['-listen', '-debug=net', '-debug=proxy', '-proxy=%s:%i' % (self.conf1.addr),'-proxyrandomize=1'], ['-listen', '-debug=net', '-debug=proxy', '-proxy=%s:%i' % (self.conf1.addr),'-onion=%s:%i' % (self.conf2.addr),'-proxyrandomize=0'], ['-listen', '-debug=net', '-debug=proxy', '-proxy=%s:%i' % (self.conf2.addr),'-proxyrandomize=1'], - ['-listen', '-debug=net', '-debug=proxy', '-proxy=[%s]:%i' % (self.conf3.addr),'-proxyrandomize=0', '-noonion'] - ]) + [] + ] + if self.have_ipv6: + args[3] = ['-listen', '-debug=net', '-debug=proxy', '-proxy=[%s]:%i' % (self.conf3.addr),'-proxyrandomize=0', '-noonion'] + return start_nodes(4, self.options.tmpdir, extra_args=args) def node_test(self, node, proxies, auth, test_onion=True): rv = [] @@ -84,18 +93,19 @@ class ProxyTest(BitcoinTestFramework): assert_equal(cmd.password, None) rv.append(cmd) - # Test: outgoing IPv6 connection through node - node.addnode("[1233:3432:2434:2343:3234:2345:6546:4534]:5443", "onetry") - cmd = proxies[1].queue.get() - assert(isinstance(cmd, Socks5Command)) - # Note: bitcoind's SOCKS5 implementation only sends atyp DOMAINNAME, even if connecting directly to IPv4/IPv6 - assert_equal(cmd.atyp, AddressType.DOMAINNAME) - assert_equal(cmd.addr, "1233:3432:2434:2343:3234:2345:6546:4534") - assert_equal(cmd.port, 5443) - if not auth: - assert_equal(cmd.username, None) - assert_equal(cmd.password, None) - rv.append(cmd) + if self.have_ipv6: + # Test: outgoing IPv6 connection through node + node.addnode("[1233:3432:2434:2343:3234:2345:6546:4534]:5443", "onetry") + cmd = proxies[1].queue.get() + assert(isinstance(cmd, Socks5Command)) + # Note: bitcoind's SOCKS5 implementation only sends atyp DOMAINNAME, even if connecting directly to IPv4/IPv6 + assert_equal(cmd.atyp, AddressType.DOMAINNAME) + assert_equal(cmd.addr, "1233:3432:2434:2343:3234:2345:6546:4534") + assert_equal(cmd.port, 5443) + if not auth: + assert_equal(cmd.username, None) + assert_equal(cmd.password, None) + rv.append(cmd) if test_onion: # Test: outgoing onion connection through node @@ -135,10 +145,11 @@ class ProxyTest(BitcoinTestFramework): rv = self.node_test(self.nodes[2], [self.serv2, self.serv2, self.serv2, self.serv2], True) # Check that credentials as used for -proxyrandomize connections are unique credentials = set((x.username,x.password) for x in rv) - assert_equal(len(credentials), 4) + assert_equal(len(credentials), len(rv)) - # proxy on IPv6 localhost - self.node_test(self.nodes[3], [self.serv3, self.serv3, self.serv3, self.serv3], False, False) + if self.have_ipv6: + # proxy on IPv6 localhost + self.node_test(self.nodes[3], [self.serv3, self.serv3, self.serv3, self.serv3], False, False) def networks_dict(d): r = {} @@ -167,11 +178,12 @@ class ProxyTest(BitcoinTestFramework): assert_equal(n2[net]['proxy_randomize_credentials'], True) assert_equal(n2['onion']['reachable'], True) - n3 = networks_dict(self.nodes[3].getnetworkinfo()) - for net in ['ipv4','ipv6']: - assert_equal(n3[net]['proxy'], '[%s]:%i' % (self.conf3.addr)) - assert_equal(n3[net]['proxy_randomize_credentials'], False) - assert_equal(n3['onion']['reachable'], False) + if self.have_ipv6: + n3 = networks_dict(self.nodes[3].getnetworkinfo()) + for net in ['ipv4','ipv6']: + assert_equal(n3[net]['proxy'], '[%s]:%i' % (self.conf3.addr)) + assert_equal(n3[net]['proxy_randomize_credentials'], False) + assert_equal(n3['onion']['reachable'], False) if __name__ == '__main__': ProxyTest().main() diff --git a/qa/rpc-tests/test_framework/netutil.py b/qa/rpc-tests/test_framework/netutil.py index 50daa8793..bfdef76ad 100644 --- a/qa/rpc-tests/test_framework/netutil.py +++ b/qa/rpc-tests/test_framework/netutil.py @@ -137,3 +137,18 @@ def addr_to_hex(addr): else: raise ValueError('Could not parse address %s' % addr) return binascii.hexlify(bytearray(addr)) + +def test_ipv6_local(): + ''' + Check for (local) IPv6 support. + ''' + import socket + # By using SOCK_DGRAM this will not actually make a connection, but it will + # fail if there is no route to IPv6 localhost. + have_ipv6 = True + try: + s = socket.socket(socket.AF_INET6, socket.SOCK_DGRAM) + s.connect(('::1', 0)) + except socket.error: + have_ipv6 = False + return have_ipv6 From 128c5e123213c77b313f1384775339c72819aedc Mon Sep 17 00:00:00 2001 From: Luke Dashjr Date: Tue, 9 Feb 2016 22:17:09 +0000 Subject: [PATCH 45/59] Workaround Travis-side CI issues Github-Pull: #7487 Rebased-From: 149641e8fc9996da01eb76ffe4578828c40d37b5 c01f08db127883ff985889214eebdbe9513c729a 5d1148cb79856ac4695a0c7ac1cd28ada04eff34 1ecbb3b0f717c277f3db1a923fff16f7fc39432c Cherry-pick-From: 00d57b4d3a9a8e0a6e59ac639229debe14385ceb --- depends/builders/darwin.mk | 2 +- depends/builders/linux.mk | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/depends/builders/darwin.mk b/depends/builders/darwin.mk index 200d6ed22..cedbddc57 100644 --- a/depends/builders/darwin.mk +++ b/depends/builders/darwin.mk @@ -7,7 +7,7 @@ build_darwin_OTOOL: = $(shell xcrun -f otool) build_darwin_NM: = $(shell xcrun -f nm) build_darwin_INSTALL_NAME_TOOL:=$(shell xcrun -f install_name_tool) build_darwin_SHA256SUM = shasum -a 256 -build_darwin_DOWNLOAD = curl --location --fail --connect-timeout $(DOWNLOAD_CONNECT_TIMEOUT) --retry $(DOWNLOAD_RETRIES) -o +build_darwin_DOWNLOAD = curl --location --fail --connect-timeout $(DOWNLOAD_CONNECT_TIMEOUT) --retry $(DOWNLOAD_RETRIES) -L -o #darwin host on darwin builder. overrides darwin host preferences. darwin_CC=$(shell xcrun -f clang) -mmacosx-version-min=$(OSX_MIN_VERSION) diff --git a/depends/builders/linux.mk b/depends/builders/linux.mk index b03f42401..d6a304e4b 100644 --- a/depends/builders/linux.mk +++ b/depends/builders/linux.mk @@ -1,2 +1,2 @@ build_linux_SHA256SUM = sha256sum -build_linux_DOWNLOAD = curl --location --fail --connect-timeout $(DOWNLOAD_CONNECT_TIMEOUT) --retry $(DOWNLOAD_RETRIES) -o +build_linux_DOWNLOAD = curl --location --fail --connect-timeout $(DOWNLOAD_CONNECT_TIMEOUT) --retry $(DOWNLOAD_RETRIES) -L -o From 6c44620e5af7f51167271513917f087f193da243 Mon Sep 17 00:00:00 2001 From: Cory Fields Date: Wed, 20 Apr 2016 22:36:55 +0200 Subject: [PATCH 46/59] travis: switch to Trusty Github-Pull: #7920 Rebased-From: 06fdffd222ba0a00add4abe9fab9ad2c3e220d8f, 9267a47d86d0673eae9e504ee566aa4e0410d923, cf77fcdb1fe525b63b004ef729173f04bdb48882, 174023c9b008fc02316bce972b0c1031de3feee3, a33b7c9cb545985771d074748c0e368ca2d06702 Cherry-pick-From: 564aaa2cd0ee057d9eedc8da1c900d90f43c89c6 --- .travis.yml | 17 +++++++++-------- depends/packages/qt.mk | 2 +- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1a5731f3a..f2fb8a4fc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,8 +7,7 @@ # IPv6 support sudo: required -dist: precise -group: legacy +dist: trusty os: linux language: cpp @@ -37,22 +36,25 @@ matrix: - compiler: ": ARM" env: HOST=arm-linux-gnueabihf PACKAGES="g++-arm-linux-gnueabihf" DEP_OPTS="NO_QT=1" GOAL="install" BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports" - compiler: ": Win32" - env: HOST=i686-w64-mingw32 PPA="ppa:ubuntu-wine/ppa" PACKAGES="nsis gcc-mingw-w64-i686 g++-mingw-w64-i686 binutils-mingw-w64-i686 mingw-w64-dev wine1.7 bc" RUN_TESTS=true GOAL="deploy" BITCOIN_CONFIG="--enable-gui --enable-reduce-exports" MAKEJOBS="-j2" + env: HOST=i686-w64-mingw32 DPKG_ADD_ARCH="i386" DEP_OPTS="NO_QT=1" PACKAGES="nsis g++-mingw-w64-i686 wine1.6 bc" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-reduce-exports" - compiler: ": 32-bit + dash" - env: HOST=i686-pc-linux-gnu PACKAGES="g++-multilib bc python-zmq" PPA="ppa:chris-lea/zeromq" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-zmq --enable-glibc-back-compat --enable-reduce-exports LDFLAGS=-static-libstdc++" USE_SHELL="/bin/dash" + env: HOST=i686-pc-linux-gnu PACKAGES="g++-multilib bc python-zmq" DEP_OPTS="NO_QT=1" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-zmq --enable-glibc-back-compat --enable-reduce-exports LDFLAGS=-static-libstdc++" USE_SHELL="/bin/dash" - compiler: ": Win64" - env: HOST=x86_64-w64-mingw32 PPA="ppa:ubuntu-wine/ppa" PACKAGES="nsis gcc-mingw-w64-x86-64 g++-mingw-w64-x86-64 binutils-mingw-w64-x86-64 mingw-w64-dev wine1.7 bc" RUN_TESTS=true GOAL="deploy" BITCOIN_CONFIG="--enable-gui --enable-reduce-exports" MAKEJOBS="-j2" + env: HOST=x86_64-w64-mingw32 DPKG_ADD_ARCH="i386" DEP_OPTS="NO_QT=1" PACKAGES="nsis g++-mingw-w64-x86-64 wine1.6 bc" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-reduce-exports" - compiler: ": bitcoind" - env: HOST=x86_64-unknown-linux-gnu PACKAGES="bc python-zmq" PPA="ppa:chris-lea/zeromq" DEP_OPTS="NO_QT=1 NO_UPNP=1 DEBUG=1" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-zmq --enable-glibc-back-compat --enable-reduce-exports CPPFLAGS=-DDEBUG_LOCKORDER" + env: HOST=x86_64-unknown-linux-gnu PACKAGES="bc python-zmq" DEP_OPTS="NO_QT=1 NO_UPNP=1 DEBUG=1" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-zmq --enable-glibc-back-compat --enable-reduce-exports CPPFLAGS=-DDEBUG_LOCKORDER" - compiler: ": No wallet" env: HOST=x86_64-unknown-linux-gnu DEP_OPTS="NO_WALLET=1" RUN_TESTS=true GOAL="install" BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports" - compiler: ": Cross-Mac" env: HOST=x86_64-apple-darwin11 PACKAGES="cmake libcap-dev libz-dev libbz2-dev" BITCOIN_CONFIG="--enable-reduce-exports" OSX_SDK=10.9 GOAL="deploy" exclude: - compiler: gcc +before_install: + - export PATH=$(echo $PATH | tr ':' "\n" | sed '/\/opt\/python/d' | tr "\n" ":" | sed "s|::|:|g") install: - - if [ -n "$PACKAGES" ]; then sudo rm -f /etc/apt/sources.list.d/travis_ci_zeromq3-source.list; fi + - if [ -n "$PACKAGES" ]; then sudo rm -f /etc/apt/sources.list.d/google-chrome.list; fi - if [ -n "$PPA" ]; then travis_retry sudo add-apt-repository "$PPA" -y; fi + - if [ -n "$DPKG_ADD_ARCH" ]; then sudo dpkg --add-architecture "$DPKG_ADD_ARCH" ; fi - if [ -n "$PACKAGES" ]; then travis_retry sudo apt-get update; fi - if [ -n "$PACKAGES" ]; then travis_retry sudo apt-get install --no-install-recommends --no-upgrade -qq $PACKAGES; fi before_script: @@ -66,7 +68,6 @@ script: - OUTDIR=$BASE_OUTDIR/$TRAVIS_PULL_REQUEST/$TRAVIS_JOB_NUMBER-$HOST - BITCOIN_CONFIG_ALL="--disable-dependency-tracking --prefix=$TRAVIS_BUILD_DIR/depends/$HOST --bindir=$OUTDIR/bin --libdir=$OUTDIR/lib" - depends/$HOST/native/bin/ccache --max-size=$CCACHE_SIZE - - if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then export CCACHE_READONLY=1; fi - test -n "$USE_SHELL" && eval '"$USE_SHELL" -c "./autogen.sh"' || ./autogen.sh - ./configure --cache-file=config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( cat config.log && false) - make distdir PACKAGE=bitcoin VERSION=$HOST diff --git a/depends/packages/qt.mk b/depends/packages/qt.mk index 901b761fd..b2ed21f9d 100644 --- a/depends/packages/qt.mk +++ b/depends/packages/qt.mk @@ -31,7 +31,7 @@ $(package)_config_opts += -no-iconv $(package)_config_opts += -no-gif $(package)_config_opts += -no-freetype $(package)_config_opts += -no-nis -$(package)_config_opts += -no-pch +$(package)_config_opts += -pch $(package)_config_opts += -no-qml-debug $(package)_config_opts += -nomake examples $(package)_config_opts += -nomake tests From eb82f39655b6a35d52a04ce6a0f42176e20b06c7 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 11 May 2016 17:17:08 -0400 Subject: [PATCH 47/59] rpcclient: add params to be parsed as JSON There was an issue where getblockhashes wouldn't work from bitcoin-cli as the two params would be strings instead of integers. This fixes that issue, and will parse the first param as JSON for other addressindex related rpc methods. --- src/rpcclient.cpp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/rpcclient.cpp b/src/rpcclient.cpp index 047158023..27a09c0fc 100644 --- a/src/rpcclient.cpp +++ b/src/rpcclient.cpp @@ -102,6 +102,14 @@ static const CRPCConvertParam vRPCConvertParams[] = { "prioritisetransaction", 2 }, { "setban", 2 }, { "setban", 3 }, + { "getblockhashes", 0 }, + { "getblockhashes", 1 }, + { "getspentinfo", 0}, + { "getaddresstxids", 0}, + { "getaddressbalance", 0}, + { "getaddressdeltas", 0}, + { "getaddressutxos", 0}, + { "getaddressmempool", 0}, }; class CRPCConvertTable From 3c74fff5521a0c8d8f68730a0f7dbffaf91b4b99 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 11 May 2016 18:33:22 -0400 Subject: [PATCH 48/59] rpc: include help text for addressindex and related commands --- src/rpcblockchain.cpp | 6 ++- src/rpcmisc.cpp | 99 ++++++++++++++++++++++++++++++++++++++----- 2 files changed, 93 insertions(+), 12 deletions(-) diff --git a/src/rpcblockchain.cpp b/src/rpcblockchain.cpp index 276e99400..70ab3a98e 100644 --- a/src/rpcblockchain.cpp +++ b/src/rpcblockchain.cpp @@ -285,10 +285,12 @@ UniValue getblockhashes(const UniValue& params, bool fHelp) "1. high (numeric, required) The newer block timestamp\n" "2. low (numeric, required) The older block timestamp\n" "\nResult:\n" - "[" + "[\n" " \"hash\" (string) The block hash\n" - "]" + "]\n" "\nExamples:\n" + + HelpExampleCli("getblockhashes", "1231614698 1231024505") + + HelpExampleRpc("getblockhashes", "1231614698, 1231024505") ); unsigned int high = params[0].get_int(); diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index 04eb5ecef..8dc8f7146 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -462,7 +462,30 @@ UniValue getaddressmempool(const UniValue& params, bool fHelp) throw runtime_error( "getaddressmempool\n" "\nReturns all mempool deltas for an address (requires addressindex to be enabled).\n" - ); + "\nArguments:\n" + "{\n" + " \"addresses\"\n" + " [\n" + " \"address\" (string) The base58check encoded address\n" + " ,...\n" + " ]\n" + "}\n" + "\nResult:\n" + "[\n" + " {\n" + " \"address\" (string) The base58check encoded address\n" + " \"txid\" (string) The related txid\n" + " \"index\" (number) The related input or output index\n" + " \"satoshis\" (number) The difference of satoshis\n" + " \"timestamp\" (number) The time the transaction entered the mempool (seconds)\n" + " \"prevtxid\" (string) The previous txid (if spending)\n" + " \"prevout\" (string) The previous transaction output index (if spending)\n" + " }\n" + "]\n" + "\nExamples:\n" + + HelpExampleCli("getaddressmempool", "'{\"addresses\": [\"12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX\"]}'") + + HelpExampleRpc("getaddressmempool", "{\"addresses\": [\"12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX\"]}") + ); std::vector > addresses; @@ -510,6 +533,14 @@ UniValue getaddressutxos(const UniValue& params, bool fHelp) throw runtime_error( "getaddressutxos\n" "\nReturns all unspent outputs for an address (requires addressindex to be enabled).\n" + "\nArguments:\n" + "{\n" + " \"addresses\"\n" + " [\n" + " \"address\" (string) The base58check encoded address\n" + " ,...\n" + " ]\n" + "}\n" "\nResult\n" "[\n" " {\n" @@ -521,7 +552,10 @@ UniValue getaddressutxos(const UniValue& params, bool fHelp) " \"satoshis\" (number) The number of satoshis of the output\n" " }\n" "]\n" - ); + "\nExamples:\n" + + HelpExampleCli("getaddressutxos", "'{\"addresses\": [\"12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX\"]}'") + + HelpExampleRpc("getaddressutxos", "{\"addresses\": [\"12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX\"]}") + ); std::vector > addresses; @@ -566,7 +600,17 @@ UniValue getaddressdeltas(const UniValue& params, bool fHelp) throw runtime_error( "getaddressdeltas\n" "\nReturns all changes for an address (requires addressindex to be enabled).\n" - "\nResult\n" + "\nArguments:\n" + "{\n" + " \"addresses\"\n" + " [\n" + " \"address\" (string) The base58check encoded address\n" + " ,...\n" + " ]\n" + " \"start\" (number) The start block height\n" + " \"end\" (number) The end block height\n" + "}\n" + "\nResult:\n" "[\n" " {\n" " \"satoshis\" (number) The difference of satoshis\n" @@ -576,6 +620,9 @@ UniValue getaddressdeltas(const UniValue& params, bool fHelp) " \"address\" (string) The base58check encoded address\n" " }\n" "]\n" + "\nExamples:\n" + + HelpExampleCli("getaddressdeltas", "'{\"addresses\": [\"12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX\"]}'") + + HelpExampleRpc("getaddressdeltas", "{\"addresses\": [\"12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX\"]}") ); @@ -638,12 +685,23 @@ UniValue getaddressbalance(const UniValue& params, bool fHelp) if (fHelp || params.size() != 1) throw runtime_error( "getaddressbalance\n" - "\nReturns the balance for an address (requires addressindex to be enabled).\n" - "\nResult\n" + "\nReturns the balance for an address(es) (requires addressindex to be enabled).\n" + "\nArguments:\n" "{\n" - " \"balance\" (string) The current balance\n" - " ,...\n" + " \"addresses\"\n" + " [\n" + " \"address\" (string) The base58check encoded address\n" + " ,...\n" + " ]\n" "}\n" + "\nResult:\n" + "{\n" + " \"balance\" (string) The current balance in satoshis\n" + " \"received\" (string) The total number of satoshis received (including change)\n" + "}\n" + "\nExamples:\n" + + HelpExampleCli("getaddressbalance", "'{\"addresses\": [\"12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX\"]}'") + + HelpExampleRpc("getaddressbalance", "{\"addresses\": [\"12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX\"]}") ); std::vector > addresses; @@ -683,12 +741,25 @@ UniValue getaddresstxids(const UniValue& params, bool fHelp) if (fHelp || params.size() != 1) throw runtime_error( "getaddresstxids\n" - "\nReturns the txids for an address (requires addressindex to be enabled).\n" - "\nResult\n" + "\nReturns the txids for an address(es) (requires addressindex to be enabled).\n" + "\nArguments:\n" + "{\n" + " \"addresses\"\n" + " [\n" + " \"address\" (string) The base58check encoded address\n" + " ,...\n" + " ]\n" + " \"start\" (number) The start block height\n" + " \"end\" (number) The end block height\n" + "}\n" + "\nResult:\n" "[\n" " \"transactionid\" (string) The transaction id\n" " ,...\n" "]\n" + "\nExamples:\n" + + HelpExampleCli("getaddresstxids", "'{\"addresses\": [\"12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX\"]}'") + + HelpExampleRpc("getaddresstxids", "{\"addresses\": [\"12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX\"]}") ); std::vector > addresses; @@ -755,12 +826,20 @@ UniValue getspentinfo(const UniValue& params, bool fHelp) throw runtime_error( "getspentinfo\n" "\nReturns the txid and index where an output is spent.\n" - "\nResult\n" + "\nArguments:\n" + "{\n" + " \"txid\" (string) The hex string of the txid\n" + " \"index\" (number) The start block height\n" + "}\n" + "\nResult:\n" "{\n" " \"txid\" (string) The transaction id\n" " \"index\" (number) The spending input index\n" " ,...\n" "}\n" + "\nExamples:\n" + + HelpExampleCli("getspentinfo", "'{\"txid\": \"0437cd7f8525ceed2324359c2d0ba26006d92d856a9c20fa0241106ee5a597c9\", \"index\": 0}'") + + HelpExampleRpc("getspentinfo", "{\"txid\": \"0437cd7f8525ceed2324359c2d0ba26006d92d856a9c20fa0241106ee5a597c9\", \"index\": 0}") ); UniValue txidValue = find_value(params[0].get_obj(), "txid"); From 1c022b9fc2773be9dfdca3b58b5daac26e04a382 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 10 May 2016 10:27:03 -0400 Subject: [PATCH 49/59] rpc: add blockindex to getaddressdeltas method for the purposes of secondary sorting by block order --- qa/rpc-tests/addressindex.py | 1 + src/rpcmisc.cpp | 1 + 2 files changed, 2 insertions(+) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index a07d4de1e..47102a02c 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -177,6 +177,7 @@ class AddressIndexTest(BitcoinTestFramework): balance3 += delta["satoshis"] assert_equal(balance3, change_amount) assert_equal(deltas[0]["address"], address2) + assert_equal(deltas[0]["blockindex"], 1) # Check that entire range will be queried deltasAll = self.nodes[1].getaddressdeltas({"addresses": [address2]}) diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index 8dc8f7146..328e9eb29 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -672,6 +672,7 @@ UniValue getaddressdeltas(const UniValue& params, bool fHelp) delta.push_back(Pair("satoshis", it->second)); delta.push_back(Pair("txid", it->first.txhash.GetHex())); delta.push_back(Pair("index", (int)it->first.index)); + delta.push_back(Pair("blockindex", (int)it->first.txindex)); delta.push_back(Pair("height", it->first.blockHeight)); delta.push_back(Pair("address", address)); result.push_back(delta); From 87dfd136948d9834319895dacbcf87e786a16818 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 12 May 2016 16:16:44 -0400 Subject: [PATCH 50/59] rpc: include satoshis in verbose raw transaction --- qa/rpc-tests/txindex.py | 73 +++++++++++++++++++++++++++++++++++++++ src/rpcrawtransaction.cpp | 1 + 2 files changed, 74 insertions(+) create mode 100755 qa/rpc-tests/txindex.py diff --git a/qa/rpc-tests/txindex.py b/qa/rpc-tests/txindex.py new file mode 100755 index 000000000..b139253b7 --- /dev/null +++ b/qa/rpc-tests/txindex.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python2 +# Copyright (c) 2014-2015 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +# +# Test txindex generation and fetching +# + +import time +from test_framework.test_framework import BitcoinTestFramework +from test_framework.util import * +from test_framework.script import * +from test_framework.mininode import * +import binascii + +class TxIndexTest(BitcoinTestFramework): + + def setup_chain(self): + print("Initializing test directory "+self.options.tmpdir) + initialize_chain_clean(self.options.tmpdir, 4) + + def setup_network(self): + self.nodes = [] + # Nodes 0/1 are "wallet" nodes + self.nodes.append(start_node(0, self.options.tmpdir, ["-debug"])) + self.nodes.append(start_node(1, self.options.tmpdir, ["-debug", "-txindex"])) + # Nodes 2/3 are used for testing + self.nodes.append(start_node(2, self.options.tmpdir, ["-debug", "-txindex"])) + self.nodes.append(start_node(3, self.options.tmpdir, ["-debug", "-txindex"])) + connect_nodes(self.nodes[0], 1) + connect_nodes(self.nodes[0], 2) + connect_nodes(self.nodes[0], 3) + + self.is_network_split = False + self.sync_all() + + def run_test(self): + print "Mining blocks..." + self.nodes[0].generate(105) + self.sync_all() + + chain_height = self.nodes[1].getblockcount() + assert_equal(chain_height, 105) + + print "Testing transaction index..." + + privkey = "cSdkPxkAjA4HDr5VHgsebAPDEh9Gyub4HK8UJr2DFGGqKKy4K5sG" + address = "mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW" + addressHash = "0b2f0a0c31bfe0406b0ccc1381fdbe311946dadc".decode("hex") + scriptPubKey = CScript([OP_DUP, OP_HASH160, addressHash, OP_EQUALVERIFY, OP_CHECKSIG]) + unspent = self.nodes[0].listunspent() + tx = CTransaction() + amount = unspent[0]["amount"] * 100000000 + tx.vin = [CTxIn(COutPoint(int(unspent[0]["txid"], 16), unspent[0]["vout"]))] + tx.vout = [CTxOut(amount, scriptPubKey)] + tx.rehash() + + signed_tx = self.nodes[0].signrawtransaction(binascii.hexlify(tx.serialize()).decode("utf-8")) + txid = self.nodes[0].sendrawtransaction(signed_tx["hex"], True) + self.nodes[0].generate(1) + self.sync_all() + + # Check verbose raw transaction results + verbose = self.nodes[3].getrawtransaction(unspent[0]["txid"], 1) + assert_equal(verbose["vout"][0]["valueSat"], 5000000000); + assert_equal(verbose["vout"][0]["value"], 50); + + print "Passed\n" + + +if __name__ == '__main__': + TxIndexTest().main() diff --git a/src/rpcrawtransaction.cpp b/src/rpcrawtransaction.cpp index 6fbec480e..1d32da4aa 100644 --- a/src/rpcrawtransaction.cpp +++ b/src/rpcrawtransaction.cpp @@ -88,6 +88,7 @@ void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry) const CTxOut& txout = tx.vout[i]; UniValue out(UniValue::VOBJ); out.push_back(Pair("value", ValueFromAmount(txout.nValue))); + out.push_back(Pair("valueSat", txout.nValue)); out.push_back(Pair("n", (int64_t)i)); UniValue o(UniValue::VOBJ); ScriptPubKeyToJSON(txout.scriptPubKey, o, true); From 16d35eb228232ed53f87cee233d0c8c3a9ca39eb Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Fri, 13 May 2016 11:43:01 -0400 Subject: [PATCH 51/59] main: add amount and address to spentindex value --- src/main.cpp | 53 +++++++++++++++++++++++++--------------------------- src/main.h | 14 +++++++++++++- 2 files changed, 38 insertions(+), 29 deletions(-) diff --git a/src/main.cpp b/src/main.cpp index d750477e9..83e48801d 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -2492,39 +2492,36 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin for (size_t j = 0; j < tx.vin.size(); j++) { const CTxIn input = tx.vin[j]; + const CTxOut &prevout = view.GetOutputFor(tx.vin[j]); + uint160 hashBytes; + int addressType; + + if (prevout.scriptPubKey.IsPayToScriptHash()) { + hashBytes = uint160(vector (prevout.scriptPubKey.begin()+2, prevout.scriptPubKey.begin()+22)); + addressType = 2; + } else if (prevout.scriptPubKey.IsPayToPublicKeyHash()) { + hashBytes = uint160(vector (prevout.scriptPubKey.begin()+3, prevout.scriptPubKey.begin()+23)); + addressType = 1; + } else { + hashBytes.SetNull(); + addressType = 0; + } + + if (fAddressIndex && addressType > 0) { + // record spending activity + addressIndex.push_back(make_pair(CAddressIndexKey(addressType, hashBytes, pindex->nHeight, i, txhash, j, true), prevout.nValue * -1)); + + // remove address from unspent index + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(addressType, hashBytes, input.prevout.hash, input.prevout.n), CAddressUnspentValue())); + } if (fSpentIndex) { // add the spent index to determine the txid and input that spent an output - spentIndex.push_back(make_pair(CSpentIndexKey(input.prevout.hash, input.prevout.n), CSpentIndexValue(txhash, j, pindex->nHeight))); + // and to find the amount and address from an input + spentIndex.push_back(make_pair(CSpentIndexKey(input.prevout.hash, input.prevout.n), CSpentIndexValue(txhash, j, pindex->nHeight, prevout.nValue, addressType, hashBytes))); } - - if (fAddressIndex) { - - const CTxOut &prevout = view.GetOutputFor(tx.vin[j]); - - if (prevout.scriptPubKey.IsPayToScriptHash()) { - vector hashBytes(prevout.scriptPubKey.begin()+2, prevout.scriptPubKey.begin()+22); - - // record spending activity - addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, j, true), prevout.nValue * -1)); - - // remove address from unspent index - addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), input.prevout.hash, input.prevout.n), CAddressUnspentValue())); - } else if (prevout.scriptPubKey.IsPayToPublicKeyHash()) { - vector hashBytes(prevout.scriptPubKey.begin()+3, prevout.scriptPubKey.begin()+23); - - // record spending activity - addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, j, true), prevout.nValue * -1)); - - // remove address from unspent index - addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), input.prevout.hash, input.prevout.n), CAddressUnspentValue())); - - } else { - continue; - } - } - } + } if (fStrictPayToScriptHash) diff --git a/src/main.h b/src/main.h index 6c685e68f..c343a264c 100644 --- a/src/main.h +++ b/src/main.h @@ -325,6 +325,9 @@ struct CSpentIndexValue { uint256 txid; unsigned int inputIndex; int blockHeight; + CAmount satoshis; + int addressType; + uint160 addressHash; ADD_SERIALIZE_METHODS; @@ -333,12 +336,18 @@ struct CSpentIndexValue { READWRITE(txid); READWRITE(inputIndex); READWRITE(blockHeight); + READWRITE(satoshis); + READWRITE(addressType); + READWRITE(addressHash); } - CSpentIndexValue(uint256 t, unsigned int i, int h) { + CSpentIndexValue(uint256 t, unsigned int i, int h, CAmount s, int type, uint160 a) { txid = t; inputIndex = i; blockHeight = h; + satoshis = s; + addressType = type; + addressHash = a; } CSpentIndexValue() { @@ -349,6 +358,9 @@ struct CSpentIndexValue { txid.SetNull(); inputIndex = 0; blockHeight = 0; + satoshis = 0; + addressType = 0; + addressHash.SetNull(); } bool IsNull() const { From 4c7dc871d1061ecab8238b5fb9efc0622d143e6a Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Fri, 13 May 2016 11:43:29 -0400 Subject: [PATCH 52/59] rpc: add input value and address to getrawtransaction if spentindex enabled --- qa/rpc-tests/spentindex.py | 29 +++++++++++++++++++++++++++++ src/rpcrawtransaction.cpp | 14 ++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/qa/rpc-tests/spentindex.py b/qa/rpc-tests/spentindex.py index 6669df462..5d0ab58b6 100755 --- a/qa/rpc-tests/spentindex.py +++ b/qa/rpc-tests/spentindex.py @@ -62,18 +62,47 @@ class SpentIndexTest(BitcoinTestFramework): self.nodes[0].generate(1) self.sync_all() + print "Testing getspentinfo method..." + # Check that the spentinfo works standalone info = self.nodes[1].getspentinfo({"txid": unspent[0]["txid"], "index": unspent[0]["vout"]}) assert_equal(info["txid"], txid) assert_equal(info["index"], 0) assert_equal(info["height"], 106) + print "Testing getrawtransaction method..." + # Check that verbose raw transaction includes spent info txVerbose = self.nodes[3].getrawtransaction(unspent[0]["txid"], 1) assert_equal(txVerbose["vout"][unspent[0]["vout"]]["spentTxId"], txid) assert_equal(txVerbose["vout"][unspent[0]["vout"]]["spentIndex"], 0) assert_equal(txVerbose["vout"][unspent[0]["vout"]]["spentHeight"], 106) + # Check that verbose raw transaction includes input values + txVerbose2 = self.nodes[3].getrawtransaction(txid, 1) + assert_equal(txVerbose2["vin"][0]["value"], Decimal(unspent[0]["amount"])) + assert_equal(txVerbose2["vin"][0]["valueSat"], amount) + + # Check that verbose raw transaction includes address values and input values + privkey2 = "cSdkPxkAjA4HDr5VHgsebAPDEh9Gyub4HK8UJr2DFGGqKKy4K5sG" + address2 = "mgY65WSfEmsyYaYPQaXhmXMeBhwp4EcsQW" + addressHash2 = "0b2f0a0c31bfe0406b0ccc1381fdbe311946dadc".decode("hex") + scriptPubKey2 = CScript([OP_DUP, OP_HASH160, addressHash2, OP_EQUALVERIFY, OP_CHECKSIG]) + tx2 = CTransaction() + tx2.vin = [CTxIn(COutPoint(int(txid, 16), 0))] + tx2.vout = [CTxOut(amount, scriptPubKey2)] + tx.rehash() + self.nodes[0].importprivkey(privkey) + signed_tx2 = self.nodes[0].signrawtransaction(binascii.hexlify(tx2.serialize()).decode("utf-8")) + txid2 = self.nodes[0].sendrawtransaction(signed_tx2["hex"], True) + self.nodes[0].generate(1) + self.sync_all() + + txVerbose3 = self.nodes[3].getrawtransaction(txid2, 1) + assert_equal(txVerbose3["vin"][0]["address"], address2) + assert_equal(txVerbose2["vin"][0]["value"], Decimal(unspent[0]["amount"])) + assert_equal(txVerbose2["vin"][0]["valueSat"], amount) + print "Passed\n" diff --git a/src/rpcrawtransaction.cpp b/src/rpcrawtransaction.cpp index 1d32da4aa..ee39fbefd 100644 --- a/src/rpcrawtransaction.cpp +++ b/src/rpcrawtransaction.cpp @@ -78,6 +78,20 @@ void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry) o.push_back(Pair("asm", ScriptToAsmStr(txin.scriptSig, true))); o.push_back(Pair("hex", HexStr(txin.scriptSig.begin(), txin.scriptSig.end()))); in.push_back(Pair("scriptSig", o)); + + // Add address and value info if spentindex enabled + CSpentIndexValue spentInfo; + CSpentIndexKey spentKey(txin.prevout.hash, txin.prevout.n); + if (GetSpentIndex(spentKey, spentInfo)) { + in.push_back(Pair("value", ValueFromAmount(spentInfo.satoshis))); + in.push_back(Pair("valueSat", spentInfo.satoshis)); + if (spentInfo.addressType == 1) { + in.push_back(Pair("address", CBitcoinAddress(CKeyID(spentInfo.addressHash)).ToString())); + } else if (spentInfo.addressType == 2) { + in.push_back(Pair("address", CBitcoinAddress(CScriptID(spentInfo.addressHash)).ToString())); + } + } + } in.push_back(Pair("sequence", (int64_t)txin.nSequence)); vin.push_back(in); From 55fa4798ebbef084b46b047b8d16b22d713a3102 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Mon, 16 May 2016 14:23:01 -0400 Subject: [PATCH 53/59] main: spentindex for the mempool --- qa/rpc-tests/spentindex.py | 17 +++++-- src/Makefile.am | 1 + src/main.cpp | 10 ++++ src/main.h | 76 +---------------------------- src/spentindex.h | 98 ++++++++++++++++++++++++++++++++++++++ src/txmempool.cpp | 66 +++++++++++++++++++++++++ src/txmempool.h | 11 +++++ 7 files changed, 200 insertions(+), 79 deletions(-) create mode 100644 src/spentindex.h diff --git a/qa/rpc-tests/spentindex.py b/qa/rpc-tests/spentindex.py index 5d0ab58b6..c233ca019 100755 --- a/qa/rpc-tests/spentindex.py +++ b/qa/rpc-tests/spentindex.py @@ -95,13 +95,22 @@ class SpentIndexTest(BitcoinTestFramework): self.nodes[0].importprivkey(privkey) signed_tx2 = self.nodes[0].signrawtransaction(binascii.hexlify(tx2.serialize()).decode("utf-8")) txid2 = self.nodes[0].sendrawtransaction(signed_tx2["hex"], True) + + # Check the mempool index + self.sync_all() + txVerbose3 = self.nodes[1].getrawtransaction(txid2, 1) + assert_equal(txVerbose3["vin"][0]["address"], address2) + assert_equal(txVerbose3["vin"][0]["value"], Decimal(unspent[0]["amount"])) + assert_equal(txVerbose3["vin"][0]["valueSat"], amount) + + # Check the database index self.nodes[0].generate(1) self.sync_all() - txVerbose3 = self.nodes[3].getrawtransaction(txid2, 1) - assert_equal(txVerbose3["vin"][0]["address"], address2) - assert_equal(txVerbose2["vin"][0]["value"], Decimal(unspent[0]["amount"])) - assert_equal(txVerbose2["vin"][0]["valueSat"], amount) + txVerbose4 = self.nodes[3].getrawtransaction(txid2, 1) + assert_equal(txVerbose4["vin"][0]["address"], address2) + assert_equal(txVerbose4["vin"][0]["value"], Decimal(unspent[0]["amount"])) + assert_equal(txVerbose4["vin"][0]["valueSat"], amount) print "Passed\n" diff --git a/src/Makefile.am b/src/Makefile.am index 9632d6567..c52371bea 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -80,6 +80,7 @@ endif # bitcoin core # BITCOIN_CORE_H = \ addressindex.h \ + spentindex.h \ addrman.h \ alert.h \ amount.h \ diff --git a/src/main.cpp b/src/main.cpp index 83e48801d..f384df004 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1415,10 +1415,17 @@ bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState &state, const C // Store transaction in memory pool.addUnchecked(hash, entry, setAncestors, !IsInitialBlockDownload()); + + // Add memory address index if (fAddressIndex) { pool.addAddressIndex(entry, view); } + // Add memory spent index + if (fSpentIndex) { + pool.addSpentIndex(entry, view); + } + // trim mempool and check if tx was trimmed if (!fOverrideMempoolLimit) { LimitMempoolSize(pool, GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000, GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY) * 60 * 60); @@ -1460,6 +1467,9 @@ bool GetSpentIndex(CSpentIndexKey &key, CSpentIndexValue &value) if (!fSpentIndex) return false; + if (mempool.getSpentIndex(key, value)) + return true; + if (!pblocktree->ReadSpentIndex(key, value)) return error("unable to get spent info"); diff --git a/src/main.h b/src/main.h index c343a264c..3f8a3fb32 100644 --- a/src/main.h +++ b/src/main.h @@ -17,6 +17,7 @@ #include "script/script_error.h" #include "sync.h" #include "versionbits.h" +#include "spentindex.h" #include #include @@ -293,81 +294,6 @@ struct CNodeStateStats { std::vector vHeightInFlight; }; -struct CSpentIndexKey { - uint256 txid; - unsigned int outputIndex; - - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { - READWRITE(txid); - READWRITE(outputIndex); - } - - CSpentIndexKey(uint256 t, unsigned int i) { - txid = t; - outputIndex = i; - } - - CSpentIndexKey() { - SetNull(); - } - - void SetNull() { - txid.SetNull(); - outputIndex = 0; - } - -}; - -struct CSpentIndexValue { - uint256 txid; - unsigned int inputIndex; - int blockHeight; - CAmount satoshis; - int addressType; - uint160 addressHash; - - ADD_SERIALIZE_METHODS; - - template - inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { - READWRITE(txid); - READWRITE(inputIndex); - READWRITE(blockHeight); - READWRITE(satoshis); - READWRITE(addressType); - READWRITE(addressHash); - } - - CSpentIndexValue(uint256 t, unsigned int i, int h, CAmount s, int type, uint160 a) { - txid = t; - inputIndex = i; - blockHeight = h; - satoshis = s; - addressType = type; - addressHash = a; - } - - CSpentIndexValue() { - SetNull(); - } - - void SetNull() { - txid.SetNull(); - inputIndex = 0; - blockHeight = 0; - satoshis = 0; - addressType = 0; - addressHash.SetNull(); - } - - bool IsNull() const { - return txid.IsNull(); - } -}; - struct CTimestampIndexIteratorKey { unsigned int timestamp; diff --git a/src/spentindex.h b/src/spentindex.h new file mode 100644 index 000000000..bd5da45d6 --- /dev/null +++ b/src/spentindex.h @@ -0,0 +1,98 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2015 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_SPENTINDEX_H +#define BITCOIN_SPENTINDEX_H + +#include "uint256.h" +#include "amount.h" + +struct CSpentIndexKey { + uint256 txid; + unsigned int outputIndex; + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + READWRITE(txid); + READWRITE(outputIndex); + } + + CSpentIndexKey(uint256 t, unsigned int i) { + txid = t; + outputIndex = i; + } + + CSpentIndexKey() { + SetNull(); + } + + void SetNull() { + txid.SetNull(); + outputIndex = 0; + } + +}; + +struct CSpentIndexValue { + uint256 txid; + unsigned int inputIndex; + int blockHeight; + CAmount satoshis; + int addressType; + uint160 addressHash; + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + READWRITE(txid); + READWRITE(inputIndex); + READWRITE(blockHeight); + READWRITE(satoshis); + READWRITE(addressType); + READWRITE(addressHash); + } + + CSpentIndexValue(uint256 t, unsigned int i, int h, CAmount s, int type, uint160 a) { + txid = t; + inputIndex = i; + blockHeight = h; + satoshis = s; + addressType = type; + addressHash = a; + } + + CSpentIndexValue() { + SetNull(); + } + + void SetNull() { + txid.SetNull(); + inputIndex = 0; + blockHeight = 0; + satoshis = 0; + addressType = 0; + addressHash.SetNull(); + } + + bool IsNull() const { + return txid.IsNull(); + } +}; + +struct CSpentIndexKeyCompare +{ + bool operator()(const CSpentIndexKey& a, const CSpentIndexKey& b) const { + if (a.txid == b.txid) { + return a.outputIndex < b.outputIndex; + } else { + return a.txid < b.txid; + } + } +}; + +#endif // BITCOIN_SPENTINDEX_H diff --git a/src/txmempool.cpp b/src/txmempool.cpp index adbe662ab..6ed4edbfd 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -496,6 +496,71 @@ bool CTxMemPool::removeAddressIndex(const uint256 txhash) return true; } +void CTxMemPool::addSpentIndex(const CTxMemPoolEntry &entry, const CCoinsViewCache &view) +{ + LOCK(cs); + + const CTransaction& tx = entry.GetTx(); + std::vector inserted; + + uint256 txhash = tx.GetHash(); + for (unsigned int j = 0; j < tx.vin.size(); j++) { + const CTxIn input = tx.vin[j]; + const CTxOut &prevout = view.GetOutputFor(input); + uint160 addressHash; + int addressType; + + if (prevout.scriptPubKey.IsPayToScriptHash()) { + addressHash = uint160(vector (prevout.scriptPubKey.begin()+2, prevout.scriptPubKey.begin()+22)); + addressType = 2; + } else if (prevout.scriptPubKey.IsPayToPublicKeyHash()) { + addressHash = uint160(vector (prevout.scriptPubKey.begin()+3, prevout.scriptPubKey.begin()+23)); + addressType = 1; + } else { + addressHash.SetNull(); + addressType = 0; + } + + CSpentIndexKey key = CSpentIndexKey(input.prevout.hash, input.prevout.n); + CSpentIndexValue value = CSpentIndexValue(txhash, j, -1, prevout.nValue, addressType, addressHash); + + mapSpent.insert(make_pair(key, value)); + inserted.push_back(key); + + } + + mapSpentInserted.insert(make_pair(txhash, inserted)); +} + +bool CTxMemPool::getSpentIndex(CSpentIndexKey &key, CSpentIndexValue &value) +{ + LOCK(cs); + mapSpentIndex::iterator it; + + it = mapSpent.find(key); + if (it != mapSpent.end()) { + value = it->second; + return true; + } + return false; +} + +bool CTxMemPool::removeSpentIndex(const uint256 txhash) +{ + LOCK(cs); + mapSpentIndexInserted::iterator it = mapSpentInserted.find(txhash); + + if (it != mapSpentInserted.end()) { + std::vector keys = (*it).second; + for (std::vector::iterator mit = keys.begin(); mit != keys.end(); mit++) { + mapSpent.erase(*mit); + } + mapSpentInserted.erase(it); + } + + return true; +} + void CTxMemPool::removeUnchecked(txiter it) { const uint256 hash = it->GetTx().GetHash(); @@ -510,6 +575,7 @@ void CTxMemPool::removeUnchecked(txiter it) nTransactionsUpdated++; minerPolicyEstimator->removeTx(hash); removeAddressIndex(hash); + removeSpentIndex(hash); } // Calculates descendants of entry that are not already in setDescendants, and adds to diff --git a/src/txmempool.h b/src/txmempool.h index 57091a02e..a94ff607b 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -10,6 +10,7 @@ #include #include "addressindex.h" +#include "spentindex.h" #include "amount.h" #include "coins.h" #include "primitives/transaction.h" @@ -426,6 +427,12 @@ private: typedef std::map > addressDeltaMapInserted; addressDeltaMapInserted mapAddressInserted; + typedef std::map mapSpentIndex; + mapSpentIndex mapSpent; + + typedef std::map > mapSpentIndexInserted; + mapSpentIndexInserted mapSpentInserted; + void UpdateParent(txiter entry, txiter parent, bool add); void UpdateChild(txiter entry, txiter child, bool add); @@ -462,6 +469,10 @@ public: std::vector > &results); bool removeAddressIndex(const uint256 txhash); + void addSpentIndex(const CTxMemPoolEntry &entry, const CCoinsViewCache &view); + bool getSpentIndex(CSpentIndexKey &key, CSpentIndexValue &value); + bool removeSpentIndex(const uint256 txhash); + void remove(const CTransaction &tx, std::list& removed, bool fRecursive = false); void removeForReorg(const CCoinsViewCache *pcoins, unsigned int nMemPoolHeight, int flags); void removeConflicts(const CTransaction &tx, std::list& removed); From fea930aa8c4ca3a83af507960fedfcebc5a2ed3b Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 19 May 2016 20:10:02 -0400 Subject: [PATCH 54/59] rpc: add input confirmations to getrawtransaction --- qa/rpc-tests/spentindex.py | 8 ++++++++ src/rpcrawtransaction.cpp | 7 +++++++ 2 files changed, 15 insertions(+) diff --git a/qa/rpc-tests/spentindex.py b/qa/rpc-tests/spentindex.py index c233ca019..0a3f93e72 100755 --- a/qa/rpc-tests/spentindex.py +++ b/qa/rpc-tests/spentindex.py @@ -103,6 +103,10 @@ class SpentIndexTest(BitcoinTestFramework): assert_equal(txVerbose3["vin"][0]["value"], Decimal(unspent[0]["amount"])) assert_equal(txVerbose3["vin"][0]["valueSat"], amount) + # Check that the input confirmations work for mempool unconfirmed transactions + assert_equal(txVerbose3["vin"][0].has_key("height"), False) + assert_equal(txVerbose3["vin"][0]["confirmations"], 0) + # Check the database index self.nodes[0].generate(1) self.sync_all() @@ -112,6 +116,10 @@ class SpentIndexTest(BitcoinTestFramework): assert_equal(txVerbose4["vin"][0]["value"], Decimal(unspent[0]["amount"])) assert_equal(txVerbose4["vin"][0]["valueSat"], amount) + # Check that the input confirmations work + assert_equal(txVerbose4["vin"][0]["height"], 107) + assert_equal(txVerbose4["vin"][0]["confirmations"], 1) + print "Passed\n" diff --git a/src/rpcrawtransaction.cpp b/src/rpcrawtransaction.cpp index ee39fbefd..386cfbcc4 100644 --- a/src/rpcrawtransaction.cpp +++ b/src/rpcrawtransaction.cpp @@ -83,6 +83,13 @@ void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry) CSpentIndexValue spentInfo; CSpentIndexKey spentKey(txin.prevout.hash, txin.prevout.n); if (GetSpentIndex(spentKey, spentInfo)) { + // Unconfirmed spentInfo have a height of -1, block 0 is unspendable + if (spentInfo.blockHeight > 0) { + in.push_back(Pair("height", spentInfo.blockHeight)); + in.push_back(Pair("confirmations", 1 + chainActive.Height() - spentInfo.blockHeight)); + } else { + in.push_back(Pair("confirmations", 0)); + } in.push_back(Pair("value", ValueFromAmount(spentInfo.satoshis))); in.push_back(Pair("valueSat", spentInfo.satoshis)); if (spentInfo.addressType == 1) { From 347f0d1ed4e4f7d6ec06e7034272ac5745ff1ec3 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 1 Jun 2016 14:57:39 -0400 Subject: [PATCH 55/59] main: do not log error when spent info not found --- src/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.cpp b/src/main.cpp index f384df004..1092d3563 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1471,7 +1471,7 @@ bool GetSpentIndex(CSpentIndexKey &key, CSpentIndexValue &value) return true; if (!pblocktree->ReadSpentIndex(key, value)) - return error("unable to get spent info"); + return false; return true; } From 809a8abff69fbf8c04641ccd352f0b1cb1c08047 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Thu, 9 Jun 2016 14:02:30 -0400 Subject: [PATCH 56/59] tests: expanded address index mempool testing --- qa/rpc-tests/addressindex.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index 47102a02c..54c018e3d 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -246,17 +246,21 @@ class AddressIndexTest(BitcoinTestFramework): tx2 = CTransaction() tx2.vin = [CTxIn(COutPoint(int(unspent[1]["txid"], 16), unspent[1]["vout"]))] amount = unspent[1]["amount"] * 100000000 - tx2.vout = [CTxOut(amount, scriptPubKey3)] + tx2.vout = [CTxOut(amount / 2, scriptPubKey3), CTxOut(amount / 2, scriptPubKey3)] tx2.rehash() signed_tx2 = self.nodes[2].signrawtransaction(binascii.hexlify(tx2.serialize()).decode("utf-8")) memtxid2 = self.nodes[2].sendrawtransaction(signed_tx2["hex"], True) time.sleep(2) mempool = self.nodes[2].getaddressmempool({"addresses": [address3]}) - assert_equal(len(mempool), 2) + assert_equal(len(mempool), 3) assert_equal(mempool[0]["txid"], memtxid1) - assert_equal(mempool[1]["txid"], memtxid2) assert_equal(mempool[0]["address"], address3) + assert_equal(mempool[0]["index"], 0) + assert_equal(mempool[1]["txid"], memtxid2) + assert_equal(mempool[1]["index"], 0) + assert_equal(mempool[2]["txid"], memtxid2) + assert_equal(mempool[2]["index"], 1) self.nodes[2].generate(1); self.sync_all(); @@ -264,7 +268,7 @@ class AddressIndexTest(BitcoinTestFramework): assert_equal(len(mempool2), 0) tx = CTransaction() - tx.vin = [CTxIn(COutPoint(int(memtxid2, 16), 0))] + tx.vin = [CTxIn(COutPoint(int(memtxid2, 16), 0)), CTxIn(COutPoint(int(memtxid2, 16), 1))] tx.vout = [CTxOut(amount - 10000, scriptPubKey2)] tx.rehash() self.nodes[2].importprivkey(privKey3) @@ -273,9 +277,11 @@ class AddressIndexTest(BitcoinTestFramework): time.sleep(2) mempool3 = self.nodes[2].getaddressmempool({"addresses": [address3]}) - assert_equal(len(mempool3), 1) + assert_equal(len(mempool3), 2) assert_equal(mempool3[0]["prevtxid"], memtxid2) assert_equal(mempool3[0]["prevout"], 0) + assert_equal(mempool3[1]["prevtxid"], memtxid2) + assert_equal(mempool3[1]["prevout"], 1) print "Passed\n" From 4dcf3e821cc6b77476a61548238b2bf9dbd0f2d9 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Fri, 10 Jun 2016 14:02:51 -0400 Subject: [PATCH 57/59] mempool: fix bug with mempool address index iteration fixes a minor bug where iteration would not end when there are matching hashes for a p2sh and p2pkh address, and would return results for both addresses --- qa/rpc-tests/addressindex.py | 16 +++++++++++++--- src/txmempool.cpp | 2 +- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index 54c018e3d..1ab5db616 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -232,6 +232,8 @@ class AddressIndexTest(BitcoinTestFramework): address3 = "mw4ynwhS7MmrQ27hr82kgqu7zryNDK26JB" addressHash3 = "aa9872b5bbcdb511d89e0e11aa27da73fd2c3f50".decode("hex") scriptPubKey3 = CScript([OP_DUP, OP_HASH160, addressHash3, OP_EQUALVERIFY, OP_CHECKSIG]) + address4 = "2N8oFVB2vThAKury4vnLquW2zVjsYjjAkYQ" + scriptPubKey4 = CScript([OP_HASH160, addressHash3, OP_EQUAL]) unspent = self.nodes[2].listunspent() tx = CTransaction() @@ -246,7 +248,12 @@ class AddressIndexTest(BitcoinTestFramework): tx2 = CTransaction() tx2.vin = [CTxIn(COutPoint(int(unspent[1]["txid"], 16), unspent[1]["vout"]))] amount = unspent[1]["amount"] * 100000000 - tx2.vout = [CTxOut(amount / 2, scriptPubKey3), CTxOut(amount / 2, scriptPubKey3)] + tx2.vout = [ + CTxOut(amount / 4, scriptPubKey3), + CTxOut(amount / 4, scriptPubKey3), + CTxOut(amount / 4, scriptPubKey4), + CTxOut(amount / 4, scriptPubKey4) + ] tx2.rehash() signed_tx2 = self.nodes[2].signrawtransaction(binascii.hexlify(tx2.serialize()).decode("utf-8")) memtxid2 = self.nodes[2].sendrawtransaction(signed_tx2["hex"], True) @@ -268,8 +275,11 @@ class AddressIndexTest(BitcoinTestFramework): assert_equal(len(mempool2), 0) tx = CTransaction() - tx.vin = [CTxIn(COutPoint(int(memtxid2, 16), 0)), CTxIn(COutPoint(int(memtxid2, 16), 1))] - tx.vout = [CTxOut(amount - 10000, scriptPubKey2)] + tx.vin = [ + CTxIn(COutPoint(int(memtxid2, 16), 0)), + CTxIn(COutPoint(int(memtxid2, 16), 1)) + ] + tx.vout = [CTxOut(amount / 2 - 10000, scriptPubKey2)] tx.rehash() self.nodes[2].importprivkey(privKey3) signed_tx3 = self.nodes[2].signrawtransaction(binascii.hexlify(tx.serialize()).decode("utf-8")) diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 6ed4edbfd..384d33bf7 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -472,7 +472,7 @@ bool CTxMemPool::getAddressIndex(std::vector > &addresse LOCK(cs); for (std::vector >::iterator it = addresses.begin(); it != addresses.end(); it++) { addressDeltaMap::iterator ait = mapAddress.lower_bound(CMempoolAddressDeltaKey((*it).second, (*it).first)); - while (ait != mapAddress.end() && (*ait).first.addressBytes == (*it).first) { + while (ait != mapAddress.end() && (*ait).first.addressBytes == (*it).first && (*ait).first.type == (*it).second) { results.push_back(*ait); ait++; } From c01f78375e20bfe8819ec03f9b5f3760e716fa79 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Fri, 10 Jun 2016 14:41:51 -0400 Subject: [PATCH 58/59] mempool: same address and index for an input and output bug fixes a bug that would happen when an output would match an input with the same address and index, and would lead to the outputs not appearing in results. --- qa/rpc-tests/addressindex.py | 28 ++++++++++++++++++++++++++++ src/addressindex.h | 11 ++++++++--- src/txmempool.cpp | 8 ++++---- 3 files changed, 40 insertions(+), 7 deletions(-) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index 1ab5db616..5c4dab85a 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -293,6 +293,34 @@ class AddressIndexTest(BitcoinTestFramework): assert_equal(mempool3[1]["prevtxid"], memtxid2) assert_equal(mempool3[1]["prevout"], 1) + # sending and receiving to the same address + privkey1 = "cQY2s58LhzUCmEXN8jtAp1Etnijx78YRZ466w4ikX1V4UpTpbsf8" + address1 = "myAUWSHnwsQrhuMWv4Br6QsCnpB41vFwHn" + address1hash = "c192bff751af8efec15135d42bfeedf91a6f3e34".decode("hex") + address1script = CScript([OP_DUP, OP_HASH160, address1hash, OP_EQUALVERIFY, OP_CHECKSIG]) + + self.nodes[0].sendtoaddress(address1, 10) + self.nodes[0].generate(1) + self.sync_all() + + utxos = self.nodes[1].getaddressutxos({"addresses": [address1]}) + assert_equal(len(utxos), 1) + + tx = CTransaction() + tx.vin = [ + CTxIn(COutPoint(int(utxos[0]["txid"], 16), utxos[0]["outputIndex"])) + ] + amount = utxos[0]["satoshis"] - 1000 + tx.vout = [CTxOut(amount, address1script)] + tx.rehash() + self.nodes[0].importprivkey(privkey1) + signed_tx = self.nodes[0].signrawtransaction(binascii.hexlify(tx.serialize()).decode("utf-8")) + mem_txid = self.nodes[0].sendrawtransaction(signed_tx["hex"], True) + + self.sync_all() + mempool_deltas = self.nodes[2].getaddressmempool({"addresses": [address1]}) + assert_equal(len(mempool_deltas), 2) + print "Passed\n" diff --git a/src/addressindex.h b/src/addressindex.h index 43b49dca9..9e734b84d 100644 --- a/src/addressindex.h +++ b/src/addressindex.h @@ -37,9 +37,9 @@ struct CMempoolAddressDeltaKey uint160 addressBytes; uint256 txhash; unsigned int index; - bool spending; + int spending; - CMempoolAddressDeltaKey(int addressType, uint160 addressHash, uint256 hash, unsigned int i, bool s) { + CMempoolAddressDeltaKey(int addressType, uint160 addressHash, uint256 hash, unsigned int i, int s) { type = addressType; addressBytes = addressHash; txhash = hash; @@ -52,6 +52,7 @@ struct CMempoolAddressDeltaKey addressBytes = addressHash; txhash.SetNull(); index = 0; + spending = 0; } }; @@ -61,7 +62,11 @@ struct CMempoolAddressDeltaKeyCompare if (a.type == b.type) { if (a.addressBytes == b.addressBytes) { if (a.txhash == b.txhash) { - return a.index < b.index; + if (a.index == b.index) { + return a.spending < b.spending; + } else { + return a.index < b.index; + } } else { return a.txhash < b.txhash; } diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 384d33bf7..2bb902f83 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -434,13 +434,13 @@ void CTxMemPool::addAddressIndex(const CTxMemPoolEntry &entry, const CCoinsViewC const CTxOut &prevout = view.GetOutputFor(input); if (prevout.scriptPubKey.IsPayToScriptHash()) { vector hashBytes(prevout.scriptPubKey.begin()+2, prevout.scriptPubKey.begin()+22); - CMempoolAddressDeltaKey key(2, uint160(hashBytes), txhash, j, true); + CMempoolAddressDeltaKey key(2, uint160(hashBytes), txhash, j, 1); CMempoolAddressDelta delta(entry.GetTime(), prevout.nValue * -1, input.prevout.hash, input.prevout.n); mapAddress.insert(make_pair(key, delta)); inserted.push_back(key); } else if (prevout.scriptPubKey.IsPayToPublicKeyHash()) { vector hashBytes(prevout.scriptPubKey.begin()+3, prevout.scriptPubKey.begin()+23); - CMempoolAddressDeltaKey key(1, uint160(hashBytes), txhash, j, true); + CMempoolAddressDeltaKey key(1, uint160(hashBytes), txhash, j, 1); CMempoolAddressDelta delta(entry.GetTime(), prevout.nValue * -1, input.prevout.hash, input.prevout.n); mapAddress.insert(make_pair(key, delta)); inserted.push_back(key); @@ -451,13 +451,13 @@ void CTxMemPool::addAddressIndex(const CTxMemPoolEntry &entry, const CCoinsViewC const CTxOut &out = tx.vout[k]; if (out.scriptPubKey.IsPayToScriptHash()) { vector hashBytes(out.scriptPubKey.begin()+2, out.scriptPubKey.begin()+22); - CMempoolAddressDeltaKey key(2, uint160(hashBytes), txhash, k, false); + CMempoolAddressDeltaKey key(2, uint160(hashBytes), txhash, k, 0); mapAddress.insert(make_pair(key, CMempoolAddressDelta(entry.GetTime(), out.nValue))); inserted.push_back(key); } else if (out.scriptPubKey.IsPayToPublicKeyHash()) { vector hashBytes(out.scriptPubKey.begin()+3, out.scriptPubKey.begin()+23); std::pair ret; - CMempoolAddressDeltaKey key(1, uint160(hashBytes), txhash, k, false); + CMempoolAddressDeltaKey key(1, uint160(hashBytes), txhash, k, 0); mapAddress.insert(make_pair(key, CMempoolAddressDelta(entry.GetTime(), out.nValue))); inserted.push_back(key); } From d28f8866845da6e6b893fd2d0b95342e39819e02 Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Wed, 15 Jun 2016 20:22:35 -0400 Subject: [PATCH 59/59] Revert "rpc: add input confirmations to getrawtransaction" --- qa/rpc-tests/spentindex.py | 8 -------- src/rpcrawtransaction.cpp | 7 ------- 2 files changed, 15 deletions(-) diff --git a/qa/rpc-tests/spentindex.py b/qa/rpc-tests/spentindex.py index 0a3f93e72..c233ca019 100755 --- a/qa/rpc-tests/spentindex.py +++ b/qa/rpc-tests/spentindex.py @@ -103,10 +103,6 @@ class SpentIndexTest(BitcoinTestFramework): assert_equal(txVerbose3["vin"][0]["value"], Decimal(unspent[0]["amount"])) assert_equal(txVerbose3["vin"][0]["valueSat"], amount) - # Check that the input confirmations work for mempool unconfirmed transactions - assert_equal(txVerbose3["vin"][0].has_key("height"), False) - assert_equal(txVerbose3["vin"][0]["confirmations"], 0) - # Check the database index self.nodes[0].generate(1) self.sync_all() @@ -116,10 +112,6 @@ class SpentIndexTest(BitcoinTestFramework): assert_equal(txVerbose4["vin"][0]["value"], Decimal(unspent[0]["amount"])) assert_equal(txVerbose4["vin"][0]["valueSat"], amount) - # Check that the input confirmations work - assert_equal(txVerbose4["vin"][0]["height"], 107) - assert_equal(txVerbose4["vin"][0]["confirmations"], 1) - print "Passed\n" diff --git a/src/rpcrawtransaction.cpp b/src/rpcrawtransaction.cpp index 386cfbcc4..ee39fbefd 100644 --- a/src/rpcrawtransaction.cpp +++ b/src/rpcrawtransaction.cpp @@ -83,13 +83,6 @@ void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry) CSpentIndexValue spentInfo; CSpentIndexKey spentKey(txin.prevout.hash, txin.prevout.n); if (GetSpentIndex(spentKey, spentInfo)) { - // Unconfirmed spentInfo have a height of -1, block 0 is unspendable - if (spentInfo.blockHeight > 0) { - in.push_back(Pair("height", spentInfo.blockHeight)); - in.push_back(Pair("confirmations", 1 + chainActive.Height() - spentInfo.blockHeight)); - } else { - in.push_back(Pair("confirmations", 0)); - } in.push_back(Pair("value", ValueFromAmount(spentInfo.satoshis))); in.push_back(Pair("valueSat", spentInfo.satoshis)); if (spentInfo.addressType == 1) {