merge bitcoin#17631: Expose block filters over REST

This commit is contained in:
Kittywhiskers Van Gogh 2019-11-28 22:39:12 -05:00
parent d60f15ec33
commit 427d07f4db
No known key found for this signature in database
GPG Key ID: 30CD0C065E5C4AAD
3 changed files with 229 additions and 5 deletions

View File

@ -52,6 +52,20 @@ With the /notxdetails/ option JSON response will only contain the transaction ha
Given a block hash: returns <COUNT> amount of blockheaders in upward direction. Given a block hash: returns <COUNT> amount of blockheaders in upward direction.
Returns empty if the block doesn't exist or it isn't in the active chain. Returns empty if the block doesn't exist or it isn't in the active chain.
#### Blockfilter Headers
`GET /rest/blockfilterheaders/<FILTERTYPE>/<COUNT>/<BLOCK-HASH>.<bin|hex|json>`
Given a block hash: returns <COUNT> amount of blockfilter headers in upward
direction for the filter type <FILTERTYPE>.
Returns empty if the block doesn't exist or it isn't in the active chain.
#### Blockfilters
`GET /rest/blockfilter/<FILTERTYPE>/<BLOCK-HASH>.<bin|hex|json>`
Given a block hash: returns the block filter of the given block of type
<FILTERTYPE>.
Responds with 404 if the block doesn't exist.
#### Blockhash by height #### Blockhash by height
`GET /rest/blockhashbyheight/<HEIGHT>.<bin|hex|json>` `GET /rest/blockhashbyheight/<HEIGHT>.<bin|hex|json>`

View File

@ -3,11 +3,13 @@
// Distributed under the MIT software license, see the accompanying // Distributed under the MIT software license, see the accompanying
// file COPYING or http://www.opensource.org/licenses/mit-license.php. // file COPYING or http://www.opensource.org/licenses/mit-license.php.
#include <blockfilter.h>
#include <chain.h> #include <chain.h>
#include <chainparams.h> #include <chainparams.h>
#include <context.h> #include <context.h>
#include <core_io.h> #include <core_io.h>
#include <httpserver.h> #include <httpserver.h>
#include <index/blockfilterindex.h>
#include <index/txindex.h> #include <index/txindex.h>
#include <llmq/chainlocks.h> #include <llmq/chainlocks.h>
#include <llmq/context.h> #include <llmq/context.h>
@ -30,6 +32,7 @@
#include <univalue.h> #include <univalue.h>
static const size_t MAX_GETUTXOS_OUTPOINTS = 15; //allow a max of 15 outpoints to be queried at once static const size_t MAX_GETUTXOS_OUTPOINTS = 15; //allow a max of 15 outpoints to be queried at once
static constexpr unsigned int MAX_REST_HEADERS_RESULTS = 2000;
enum class RetFormat { enum class RetFormat {
UNDEF, UNDEF,
@ -189,8 +192,8 @@ static bool rest_headers(const CoreContext& context,
return RESTERR(req, HTTP_BAD_REQUEST, "No header count specified. Use /rest/headers/<count>/<hash>.<ext>."); return RESTERR(req, HTTP_BAD_REQUEST, "No header count specified. Use /rest/headers/<count>/<hash>.<ext>.");
const auto parsed_count{ToIntegral<size_t>(path[0])}; const auto parsed_count{ToIntegral<size_t>(path[0])};
if (!parsed_count.has_value() || *parsed_count < 1 || *parsed_count > 2000) { if (!parsed_count.has_value() || *parsed_count < 1 || *parsed_count > MAX_REST_HEADERS_RESULTS) {
return RESTERR(req, HTTP_BAD_REQUEST, "Header count out of range: " + path[0]); return RESTERR(req, HTTP_BAD_REQUEST, strprintf("Header count out of acceptable range (1-%u): %s", MAX_REST_HEADERS_RESULTS, path[0]));
} }
std::string hashStr = path[1]; std::string hashStr = path[1];
@ -253,7 +256,7 @@ static bool rest_headers(const CoreContext& context,
return true; return true;
} }
default: { default: {
return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: .bin, .hex, .json)"); return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: " + AvailableDataFormatsString() + ")");
} }
} }
} }
@ -336,6 +339,208 @@ static bool rest_block_notxdetails(const CoreContext& context, HTTPRequest* req,
return rest_block(context, req, strURIPart, false); return rest_block(context, req, strURIPart, false);
} }
static bool rest_filter_header(const CoreContext& context, HTTPRequest* req, const std::string& strURIPart)
{
if (!CheckWarmup(req))
return false;
std::string param;
const RetFormat rf = ParseDataFormat(param, strURIPart);
std::vector<std::string> uri_parts = SplitString(param, '/');
if (uri_parts.size() != 3) {
return RESTERR(req, HTTP_BAD_REQUEST, "Invalid URI format. Expected /rest/blockfilterheaders/<filtertype>/<count>/<blockhash>");
}
uint256 block_hash;
if (!ParseHashStr(uri_parts[2], block_hash)) {
return RESTERR(req, HTTP_BAD_REQUEST, "Invalid hash: " + uri_parts[2]);
}
BlockFilterType filtertype;
if (!BlockFilterTypeByName(uri_parts[0], filtertype)) {
return RESTERR(req, HTTP_BAD_REQUEST, "Unknown filtertype " + uri_parts[0]);
}
BlockFilterIndex* index = GetBlockFilterIndex(filtertype);
if (!index) {
return RESTERR(req, HTTP_BAD_REQUEST, "Index is not enabled for filtertype " + uri_parts[0]);
}
const auto parsed_count{ToIntegral<size_t>(uri_parts[1])};
if (!parsed_count.has_value() || *parsed_count < 1 || *parsed_count > MAX_REST_HEADERS_RESULTS) {
return RESTERR(req, HTTP_BAD_REQUEST, strprintf("Header count out of acceptable range (1-%u): %s", MAX_REST_HEADERS_RESULTS, uri_parts[1]));
}
std::vector<const CBlockIndex *> headers;
headers.reserve(*parsed_count);
{
ChainstateManager* maybe_chainman = GetChainman(context, req);
if (!maybe_chainman) return false;
ChainstateManager& chainman = *maybe_chainman;
LOCK(cs_main);
CChain& active_chain = chainman.ActiveChain();
const CBlockIndex* pindex = chainman.m_blockman.LookupBlockIndex(block_hash);
while (pindex != nullptr && active_chain.Contains(pindex)) {
headers.push_back(pindex);
if (headers.size() == *parsed_count)
break;
pindex = active_chain.Next(pindex);
}
}
bool index_ready = index->BlockUntilSyncedToCurrentChain();
std::vector<uint256> filter_headers;
filter_headers.reserve(*parsed_count);
for (const CBlockIndex *pindex : headers) {
uint256 filter_header;
if (!index->LookupFilterHeader(pindex, filter_header)) {
std::string errmsg = "Filter not found.";
if (!index_ready) {
errmsg += " Block filters are still in the process of being indexed.";
} else {
errmsg += " This error is unexpected and indicates index corruption.";
}
return RESTERR(req, HTTP_NOT_FOUND, errmsg);
}
filter_headers.push_back(filter_header);
}
switch (rf) {
case RetFormat::BINARY: {
CDataStream ssHeader(SER_NETWORK, PROTOCOL_VERSION);
for (const uint256& header : filter_headers) {
ssHeader << header;
}
std::string binaryHeader = ssHeader.str();
req->WriteHeader("Content-Type", "application/octet-stream");
req->WriteReply(HTTP_OK, binaryHeader);
return true;
}
case RetFormat::HEX: {
CDataStream ssHeader(SER_NETWORK, PROTOCOL_VERSION);
for (const uint256& header : filter_headers) {
ssHeader << header;
}
std::string strHex = HexStr(ssHeader) + "\n";
req->WriteHeader("Content-Type", "text/plain");
req->WriteReply(HTTP_OK, strHex);
return true;
}
case RetFormat::JSON: {
UniValue jsonHeaders(UniValue::VARR);
for (const uint256& header : filter_headers) {
jsonHeaders.push_back(header.GetHex());
}
std::string strJSON = jsonHeaders.write() + "\n";
req->WriteHeader("Content-Type", "application/json");
req->WriteReply(HTTP_OK, strJSON);
return true;
}
default: {
return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: " + AvailableDataFormatsString() + ")");
}
}
}
static bool rest_block_filter(const CoreContext& context, HTTPRequest* req, const std::string& strURIPart)
{
if (!CheckWarmup(req))
return false;
std::string param;
const RetFormat rf = ParseDataFormat(param, strURIPart);
// request is sent over URI scheme /rest/blockfilter/filtertype/blockhash
std::vector<std::string> uri_parts = SplitString(param, '/');
if (uri_parts.size() != 2) {
return RESTERR(req, HTTP_BAD_REQUEST, "Invalid URI format. Expected /rest/blockfilter/<filtertype>/<blockhash>");
}
uint256 block_hash;
if (!ParseHashStr(uri_parts[1], block_hash)) {
return RESTERR(req, HTTP_BAD_REQUEST, "Invalid hash: " + uri_parts[1]);
}
BlockFilterType filtertype;
if (!BlockFilterTypeByName(uri_parts[0], filtertype)) {
return RESTERR(req, HTTP_BAD_REQUEST, "Unknown filtertype " + uri_parts[0]);
}
BlockFilterIndex* index = GetBlockFilterIndex(filtertype);
if (!index) {
return RESTERR(req, HTTP_BAD_REQUEST, "Index is not enabled for filtertype " + uri_parts[0]);
}
const CBlockIndex* block_index;
bool block_was_connected;
{
ChainstateManager* maybe_chainman = GetChainman(context, req);
if (!maybe_chainman) return false;
ChainstateManager& chainman = *maybe_chainman;
LOCK(cs_main);
block_index = chainman.m_blockman.LookupBlockIndex(block_hash);
if (!block_index) {
return RESTERR(req, HTTP_NOT_FOUND, uri_parts[1] + " not found");
}
block_was_connected = block_index->IsValid(BLOCK_VALID_SCRIPTS);
}
bool index_ready = index->BlockUntilSyncedToCurrentChain();
BlockFilter filter;
if (!index->LookupFilter(block_index, filter)) {
std::string errmsg = "Filter not found.";
if (!block_was_connected) {
errmsg += " Block was not connected to active chain.";
} else if (!index_ready) {
errmsg += " Block filters are still in the process of being indexed.";
} else {
errmsg += " This error is unexpected and indicates index corruption.";
}
return RESTERR(req, HTTP_NOT_FOUND, errmsg);
}
switch (rf) {
case RetFormat::BINARY: {
CDataStream ssResp(SER_NETWORK, PROTOCOL_VERSION);
ssResp << filter;
std::string binaryResp = ssResp.str();
req->WriteHeader("Content-Type", "application/octet-stream");
req->WriteReply(HTTP_OK, binaryResp);
return true;
}
case RetFormat::HEX: {
CDataStream ssResp(SER_NETWORK, PROTOCOL_VERSION);
ssResp << filter;
std::string strHex = HexStr(ssResp) + "\n";
req->WriteHeader("Content-Type", "text/plain");
req->WriteReply(HTTP_OK, strHex);
return true;
}
case RetFormat::JSON: {
UniValue ret(UniValue::VOBJ);
ret.pushKV("filter", HexStr(filter.GetEncodedFilter()));
std::string strJSON = ret.write() + "\n";
req->WriteHeader("Content-Type", "application/json");
req->WriteReply(HTTP_OK, strJSON);
return true;
}
default: {
return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: " + AvailableDataFormatsString() + ")");
}
}
}
// A bit of a hack - dependency on a function defined in rpc/blockchain.cpp // A bit of a hack - dependency on a function defined in rpc/blockchain.cpp
RPCHelpMan getblockchaininfo(); RPCHelpMan getblockchaininfo();
@ -716,6 +921,8 @@ static const struct {
{"/rest/tx/", rest_tx}, {"/rest/tx/", rest_tx},
{"/rest/block/notxdetails/", rest_block_notxdetails}, {"/rest/block/notxdetails/", rest_block_notxdetails},
{"/rest/block/", rest_block_extended}, {"/rest/block/", rest_block_extended},
{"/rest/blockfilter/", rest_block_filter},
{"/rest/blockfilterheaders/", rest_filter_header},
{"/rest/chaininfo", rest_chaininfo}, {"/rest/chaininfo", rest_chaininfo},
{"/rest/mempool/info", rest_mempool_info}, {"/rest/mempool/info", rest_mempool_info},
{"/rest/mempool/contents", rest_mempool_contents}, {"/rest/mempool/contents", rest_mempool_contents},

View File

@ -45,7 +45,7 @@ class RESTTest (BitcoinTestFramework):
def set_test_params(self): def set_test_params(self):
self.setup_clean_chain = True self.setup_clean_chain = True
self.num_nodes = 2 self.num_nodes = 2
self.extra_args = [["-rest"], []] self.extra_args = [["-rest", "-blockfilterindex=1"], []]
self.supports_cli = False self.supports_cli = False
def skip_test_if_missing_module(self): def skip_test_if_missing_module(self):
@ -282,11 +282,14 @@ class RESTTest (BitcoinTestFramework):
self.generate(self.nodes[1], 5) self.generate(self.nodes[1], 5)
json_obj = self.test_rest_request("/headers/5/{}".format(bb_hash)) json_obj = self.test_rest_request("/headers/5/{}".format(bb_hash))
assert_equal(len(json_obj), 5) # now we should have 5 header objects assert_equal(len(json_obj), 5) # now we should have 5 header objects
json_obj = self.test_rest_request(f"/blockfilterheaders/basic/5/{bb_hash}")
assert_equal(len(json_obj), 5) # now we should have 5 filter header objects
self.test_rest_request(f"/blockfilter/basic/{bb_hash}", req_type=ReqType.BIN, ret_type=RetType.OBJ)
# Test number parsing # Test number parsing
for num in ['5a', '-5', '0', '2001', '99999999999999999999999999999999999']: for num in ['5a', '-5', '0', '2001', '99999999999999999999999999999999999']:
assert_equal( assert_equal(
bytes(f'Header count out of range: {num}\r\n', 'ascii'), bytes(f'Header count out of acceptable range (1-2000): {num}\r\n', 'ascii'),
self.test_rest_request(f"/headers/{num}/{bb_hash}", ret_type=RetType.BYTES, status=400), self.test_rest_request(f"/headers/{num}/{bb_hash}", ret_type=RetType.BYTES, status=400),
) )