From 0b42ba227af7fe537e7bad095c0015f9201d9cca Mon Sep 17 00:00:00 2001 From: Braydon Fuller Date: Tue, 29 Mar 2016 15:17:30 -0400 Subject: [PATCH] main: index unspent outputs by address --- qa/rpc-tests/addressindex.py | 6 +++ src/main.cpp | 46 +++++++++++++++++- src/main.h | 91 +++++++++++++++++++++++++++++++++++- src/rpcmisc.cpp | 50 ++++++++++++++++++++ src/rpcserver.cpp | 1 + src/rpcserver.h | 1 + src/txdb.cpp | 39 ++++++++++++++++ src/txdb.h | 5 ++ 8 files changed, 236 insertions(+), 3 deletions(-) diff --git a/qa/rpc-tests/addressindex.py b/qa/rpc-tests/addressindex.py index a60c845b3..5448af6c6 100755 --- a/qa/rpc-tests/addressindex.py +++ b/qa/rpc-tests/addressindex.py @@ -169,6 +169,12 @@ class AddressIndexTest(BitcoinTestFramework): deltas = self.nodes[1].getaddressdeltas({"addresses": [address2], "start": 113, "end": 113}) assert_equal(len(deltas), 1) + # Check that unspent outputs can be queried + print "Testing utxos..." + utxos = self.nodes[1].getaddressutxos({"addresses": [address2]}) + assert_equal(len(utxos), 2) + assert_equal(utxos[0]["satoshis"], 5000000000) + # Check that indexes will be updated with a reorg print "Testing reorg..." diff --git a/src/main.cpp b/src/main.cpp index 6616c290e..dbbeef8f9 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1463,6 +1463,18 @@ bool GetAddressIndex(uint160 addressHash, int type, return true; } +bool GetAddressUnspent(uint160 addressHash, int type, + std::vector > &unspentOutputs) +{ + if (!fAddressIndex) + return error("address index not enabled"); + + if (!pblocktree->ReadAddressUnspentIndex(addressHash, type, unspentOutputs)) + return error("unable to get txids for address"); + + return true; +} + /** Return transaction in tx, and if it was found inside a block, its hash is placed in hashBlock */ bool GetTransaction(const uint256 &hash, CTransaction &txOut, const Consensus::Params& consensusParams, uint256 &hashBlock, bool fAllowSlow) { @@ -2389,6 +2401,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin vPos.reserve(block.vtx.size()); blockundo.vtxundo.reserve(block.vtx.size() - 1); std::vector > addressIndex; + std::vector > addressUnspentIndex; for (unsigned int i = 0; i < block.vtx.size(); i++) { @@ -2426,10 +2439,21 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin const CTxOut &prevout = view.GetOutputFor(tx.vin[j]); if (prevout.scriptPubKey.IsPayToScriptHash()) { vector hashBytes(prevout.scriptPubKey.begin()+2, prevout.scriptPubKey.begin()+22); + + // record spending activity addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, j, true), prevout.nValue * -1)); + + // remove address from unspent index + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, j), CAddressUnspentValue())); } else if (prevout.scriptPubKey.IsPayToPublicKeyHash()) { vector hashBytes(prevout.scriptPubKey.begin()+3, prevout.scriptPubKey.begin()+23); + + // record spending activity addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, j, true), prevout.nValue * -1)); + + // remove address from unspent index + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, j), CAddressUnspentValue())); + } else { continue; } @@ -2463,10 +2487,22 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin if (out.scriptPubKey.IsPayToScriptHash()) { vector hashBytes(out.scriptPubKey.begin()+2, out.scriptPubKey.begin()+22); + + // record receiving activity addressIndex.push_back(make_pair(CAddressIndexKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, k, false), out.nValue)); + + // record unspent output + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(2, uint160(hashBytes), pindex->nHeight, i, txhash, k), CAddressUnspentValue(out.nValue, out.scriptPubKey))); + } else if (out.scriptPubKey.IsPayToPublicKeyHash()) { vector hashBytes(out.scriptPubKey.begin()+3, out.scriptPubKey.begin()+23); + + // record receiving activity addressIndex.push_back(make_pair(CAddressIndexKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, k, false), out.nValue)); + + // record unspent output + addressUnspentIndex.push_back(make_pair(CAddressUnspentKey(1, uint160(hashBytes), pindex->nHeight, i, txhash, k), CAddressUnspentValue(out.nValue, out.scriptPubKey))); + } else { continue; } @@ -2524,9 +2560,15 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin if (!pblocktree->WriteTxIndex(vPos)) return AbortNode(state, "Failed to write transaction index"); - if (fAddressIndex) - if (!pblocktree->WriteAddressIndex(addressIndex)) + if (fAddressIndex) { + if (!pblocktree->WriteAddressIndex(addressIndex)) { return AbortNode(state, "Failed to write address index"); + } + + if (!pblocktree->UpdateAddressUnspentIndex(addressUnspentIndex)) { + return AbortNode(state, "Failed to write address unspent index"); + } + } if (fTimestampIndex) if (!pblocktree->WriteTimestampIndex(CTimestampIndexKey(pindex->nTime, pindex->GetBlockHash()))) diff --git a/src/main.h b/src/main.h index 181f88993..0964e6dc3 100644 --- a/src/main.h +++ b/src/main.h @@ -353,6 +353,93 @@ struct CTimestampIndexKey { } }; +struct CAddressUnspentKey { + unsigned int type; + uint160 hashBytes; + int blockHeight; + unsigned int txindex; + uint256 txhash; + size_t index; + + size_t GetSerializeSize(int nType, int nVersion) const { + return 65; + } + template + void Serialize(Stream& s, int nType, int nVersion) const { + ser_writedata8(s, type); + hashBytes.Serialize(s, nType, nVersion); + // Heights are stored big-endian for key sorting in LevelDB + ser_writedata32be(s, blockHeight); + ser_writedata32be(s, txindex); + txhash.Serialize(s, nType, nVersion); + ser_writedata32(s, index); + } + template + void Unserialize(Stream& s, int nType, int nVersion) { + type = ser_readdata8(s); + hashBytes.Unserialize(s, nType, nVersion); + blockHeight = ser_readdata32be(s); + txindex = ser_readdata32be(s); + txhash.Unserialize(s, nType, nVersion); + index = ser_readdata32(s); + } + + CAddressUnspentKey(unsigned int addressType, uint160 addressHash, int height, int blockindex, + uint256 txid, size_t indexValue) { + type = addressType; + hashBytes = addressHash; + blockHeight = height; + txindex = blockindex; + txhash = txid; + index = indexValue; + } + + CAddressUnspentKey() { + SetNull(); + } + + void SetNull() { + type = 0; + hashBytes.SetNull(); + blockHeight = 0; + txindex = 0; + txhash.SetNull(); + index = 0; + } +}; + +struct CAddressUnspentValue { + CAmount satoshis; + CScript script; + + ADD_SERIALIZE_METHODS; + + template + inline void SerializationOp(Stream& s, Operation ser_action, int nType, int nVersion) { + READWRITE(satoshis); + READWRITE(*(CScriptBase*)(&script)); + } + + CAddressUnspentValue(CAmount sats, CScript scriptPubKey) { + satoshis = sats; + script = scriptPubKey; + } + + CAddressUnspentValue() { + SetNull(); + } + + void SetNull() { + satoshis = 0; + script = CScript(); + } + + bool IsNull() const { + return (satoshis == 0); + } + +}; + struct CAddressIndexKey { unsigned int type; uint160 hashBytes; @@ -363,7 +450,7 @@ struct CAddressIndexKey { bool spending; size_t GetSerializeSize(int nType, int nVersion) const { - return 65; + return 66; } template void Serialize(Stream& s, int nType, int nVersion) const { @@ -601,6 +688,8 @@ bool GetTimestampIndex(const unsigned int &high, const unsigned int &low, std::v bool GetAddressIndex(uint160 addressHash, int type, std::vector > &addressIndex, int start = 0, int end = 0); +bool GetAddressUnspent(uint160 addressHash, int type, + std::vector > &unspentOutputs); /** Functions for disk access for blocks */ bool WriteBlockToDisk(const CBlock& block, CDiskBlockPos& pos, const CMessageHeader::MessageStartChars& messageStart); diff --git a/src/rpcmisc.cpp b/src/rpcmisc.cpp index 8d939ffea..1eeded608 100644 --- a/src/rpcmisc.cpp +++ b/src/rpcmisc.cpp @@ -435,6 +435,56 @@ bool getAddressesFromParams(const UniValue& params, std::vector > addresses; + + if (!getAddressesFromParams(params, addresses)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid address"); + } + + std::vector > unspentOutputs; + + for (std::vector >::iterator it = addresses.begin(); it != addresses.end(); it++) { + if (!GetAddressUnspent((*it).first, (*it).second, unspentOutputs)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "No information available for address"); + } + } + + UniValue result(UniValue::VARR); + + for (std::vector >::const_iterator it=unspentOutputs.begin(); it!=unspentOutputs.end(); it++) { + UniValue output(UniValue::VOBJ); + output.push_back(Pair("addressType", (int)it->first.type)); + output.push_back(Pair("addressHash", it->first.hashBytes.GetHex())); + output.push_back(Pair("txid", it->first.txhash.GetHex())); + output.push_back(Pair("height", it->first.blockHeight)); + output.push_back(Pair("outputIndex", it->first.index)); + output.push_back(Pair("script", HexStr(it->second.script.begin(), it->second.script.end()))); + output.push_back(Pair("satoshis", it->second.satoshis)); + result.push_back(output); + } + + return result; +} + UniValue getaddressdeltas(const UniValue& params, bool fHelp) { if (fHelp || params.size() != 1 || !params[0].isObject()) diff --git a/src/rpcserver.cpp b/src/rpcserver.cpp index c95548462..39af0513b 100644 --- a/src/rpcserver.cpp +++ b/src/rpcserver.cpp @@ -315,6 +315,7 @@ static const CRPCCommand vRPCCommands[] = #endif /* Address index */ + { "addressindex", "getaddressutxos", &getaddressutxos, false }, { "addressindex", "getaddressdeltas", &getaddressdeltas, false }, { "addressindex", "getaddresstxids", &getaddresstxids, false }, { "addressindex", "getaddressbalance", &getaddressbalance, false }, diff --git a/src/rpcserver.h b/src/rpcserver.h index fc57aa16b..5ad147dec 100644 --- a/src/rpcserver.h +++ b/src/rpcserver.h @@ -166,6 +166,7 @@ extern std::string HelpExampleRpc(const std::string& methodname, const std::stri extern void EnsureWalletIsUnlocked(); extern UniValue getconnectioncount(const UniValue& params, bool fHelp); // in rpcnet.cpp +extern UniValue getaddressutxos(const UniValue& params, bool fHelp); extern UniValue getaddressdeltas(const UniValue& params, bool fHelp); extern UniValue getaddresstxids(const UniValue& params, bool fHelp); extern UniValue getaddressbalance(const UniValue& params, bool fHelp); diff --git a/src/txdb.cpp b/src/txdb.cpp index 2128c181f..8e485e9eb 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -22,6 +22,7 @@ static const char DB_COINS = 'c'; static const char DB_BLOCK_FILES = 'f'; static const char DB_TXINDEX = 't'; static const char DB_ADDRESSINDEX = 'a'; +static const char DB_ADDRESSUNSPENTINDEX = 'u'; static const char DB_TIMESTAMPINDEX = 's'; static const char DB_BLOCK_INDEX = 'b'; @@ -165,6 +166,44 @@ bool CBlockTreeDB::WriteTxIndex(const std::vector return WriteBatch(batch); } +bool CBlockTreeDB::UpdateAddressUnspentIndex(const std::vector >&vect) { + CDBBatch batch(&GetObfuscateKey()); + for (std::vector >::const_iterator it=vect.begin(); it!=vect.end(); it++) { + if (it->second.IsNull()) { + batch.Erase(make_pair(DB_ADDRESSUNSPENTINDEX, it->first)); + } else { + batch.Write(make_pair(DB_ADDRESSUNSPENTINDEX, it->first), it->second); + } + } + return WriteBatch(batch); +} + +bool CBlockTreeDB::ReadAddressUnspentIndex(uint160 addressHash, int type, + std::vector > &unspentOutputs) { + + boost::scoped_ptr pcursor(NewIterator()); + + pcursor->Seek(make_pair(DB_ADDRESSUNSPENTINDEX, CAddressIndexIteratorKey(type, addressHash))); + + while (pcursor->Valid()) { + boost::this_thread::interruption_point(); + std::pair key; + if (pcursor->GetKey(key) && key.first == DB_ADDRESSUNSPENTINDEX && key.second.hashBytes == addressHash) { + CAddressUnspentValue nValue; + if (pcursor->GetValue(nValue)) { + unspentOutputs.push_back(make_pair(key.second, nValue)); + pcursor->Next(); + } else { + return error("failed to get address unspent value"); + } + } else { + break; + } + } + + return true; +} + bool CBlockTreeDB::WriteAddressIndex(const std::vector >&vect) { CDBBatch batch(&GetObfuscateKey()); for (std::vector >::const_iterator it=vect.begin(); it!=vect.end(); it++) diff --git a/src/txdb.h b/src/txdb.h index 93b5c0f5c..547a48fa8 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -17,6 +17,8 @@ class CBlockFileInfo; class CBlockIndex; struct CDiskTxPos; +struct CAddressUnspentKey; +struct CAddressUnspentValue; struct CAddressIndexKey; struct CAddressIndexIteratorKey; struct CTimestampIndexKey; @@ -61,6 +63,9 @@ public: bool ReadReindexing(bool &fReindex); bool ReadTxIndex(const uint256 &txid, CDiskTxPos &pos); bool WriteTxIndex(const std::vector > &list); + bool UpdateAddressUnspentIndex(const std::vector >&vect); + bool ReadAddressUnspentIndex(uint160 addressHash, int type, + std::vector > &vect); bool WriteAddressIndex(const std::vector > &vect); bool EraseAddressIndex(const std::vector > &vect); bool ReadAddressIndex(uint160 addressHash, int type,