merge bitcoin#16899: UTXO snapshot creation (dumptxoutset)

Co-authored-by: UdjinM6 <UdjinM6@users.noreply.github.com>

contrib/devtools/utxo_snapshot.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2019 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+#
+export LC_ALL=C
+
+set -ueo pipefail
+
+if (( $# < 3 )); then
+  echo 'Usage: utxo_snapshot.sh <generate-at-height> <snapshot-out-path> <dash-cli-call ...>'
+  echo
+  echo "  if <snapshot-out-path> is '-', don't produce a snapshot file but instead print the "
+  echo "  expected assumeutxo hash"
+  echo
+  echo 'Examples:'
+  echo
+  echo "  ./contrib/devtools/utxo_snapshot.sh 570000 utxo.dat ./src/dash-cli -datadir=\$(pwd)/testdata"
+  echo '  ./contrib/devtools/utxo_snapshot.sh 570000 - ./src/dash-cli'
+  exit 1
+fi
+
+GENERATE_AT_HEIGHT="${1}"; shift;
+OUTPUT_PATH="${1}"; shift;
+# Most of the calls we make take a while to run, so pad with a lengthy timeout.
+BITCOIN_CLI_CALL="${*} -rpcclienttimeout=9999999"
+
+# Block we'll invalidate/reconsider to rewind/fast-forward the chain.
+PIVOT_BLOCKHASH=$($BITCOIN_CLI_CALL getblockhash $(( GENERATE_AT_HEIGHT + 1 )) )
+
+(>&2 echo "Rewinding chain back to height ${GENERATE_AT_HEIGHT} (by invalidating ${PIVOT_BLOCKHASH}); this may take a while")
+${BITCOIN_CLI_CALL} invalidateblock "${PIVOT_BLOCKHASH}"
+
+if [[ "${OUTPUT_PATH}" = "-" ]]; then
+  (>&2 echo "Generating txoutset info...")
+  ${BITCOIN_CLI_CALL} gettxoutsetinfo | grep hash_serialized_2 | sed 's/^.*: "\(.\+\)\+",/\1/g'
+else
+  (>&2 echo "Generating UTXO snapshot...")
+  ${BITCOIN_CLI_CALL} dumptxoutset "${OUTPUT_PATH}"
+fi
+
+(>&2 echo "Restoring chain to original height; this may take a while")
+${BITCOIN_CLI_CALL} reconsiderblock "${PIVOT_BLOCKHASH}"

src/Makefile.am

@@ -238,6 +238,7 @@ BITCOIN_CORE_H = \
   node/coinstats.h \
   node/context.h \
   node/transaction.h \
+  node/utxo_snapshot.h \
   noui.h \
   optional.h \
   policy/feerate.h \

src/node/coinstats.cpp

@@ -87,6 +87,7 @@ static void ApplyStats(CCoinsStats& stats, std::nullptr_t, const uint256& hash,
 template <typename T>
 static bool GetUTXOStats(CCoinsView* view, CCoinsStats& stats, T hash_obj)
 {
+    stats = CCoinsStats();
     std::unique_ptr<CCoinsViewCursor> pcursor(view->Cursor());
     assert(pcursor);
 
@@ -111,6 +112,7 @@ static bool GetUTXOStats(CCoinsView* view, CCoinsStats& stats, T hash_obj)
             }
             prevkey = key.hash;
             outputs[key.n] = std::move(coin);
+            stats.coins_count++;
         } else {
             return error("%s: unable to read value", __func__);
         }

src/node/coinstats.h

@@ -21,16 +21,17 @@ enum class CoinStatsHashType {
 
 struct CCoinsStats
 {
-    int nHeight;
+    int nHeight{0};
-    uint256 hashBlock;
+    uint256 hashBlock{};
-    uint64_t nTransactions;
+    uint64_t nTransactions{0};
-    uint64_t nTransactionOutputs;
+    uint64_t nTransactionOutputs{0};
-    uint64_t nBogoSize;
+    uint64_t nBogoSize{0};
-    uint256 hashSerialized;
+    uint256 hashSerialized{};
-    uint64_t nDiskSize;
+    uint64_t nDiskSize{0};
-    CAmount nTotalAmount;
+    CAmount nTotalAmount{0};
 
-    CCoinsStats() : nHeight(0), nTransactions(0), nTransactionOutputs(0), nBogoSize(0), nDiskSize(0), nTotalAmount(0) {}
+    //! The number of coins contained.
+    uint64_t coins_count{0};
 };
 
 //! Calculate statistics about the unspent transaction output set

src/node/utxo_snapshot.h

@@ -0,0 +1,41 @@
+// Copyright (c) 2009-2010 Satoshi Nakamoto
+// Copyright (c) 2009-2020 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#ifndef BITCOIN_NODE_UTXO_SNAPSHOT_H
+#define BITCOIN_NODE_UTXO_SNAPSHOT_H
+
+#include <uint256.h>
+#include <serialize.h>
+
+//! Metadata describing a serialized version of a UTXO set from which an
+//! assumeutxo CChainState can be constructed.
+class SnapshotMetadata
+{
+public:
+    //! The hash of the block that reflects the tip of the chain for the
+    //! UTXO set contained in this snapshot.
+    uint256 m_base_blockhash;
+
+    //! The number of coins in the UTXO set contained in this snapshot. Used
+    //! during snapshot load to estimate progress of UTXO set reconstruction.
+    uint64_t m_coins_count = 0;
+
+    //! Necessary to "fake" the base nChainTx so that we can estimate progress during
+    //! initial block download for the assumeutxo chainstate.
+    unsigned int m_nchaintx = 0;
+
+    SnapshotMetadata() { }
+    SnapshotMetadata(
+        const uint256& base_blockhash,
+        uint64_t coins_count,
+        unsigned int nchaintx) :
+            m_base_blockhash(base_blockhash),
+            m_coins_count(coins_count),
+            m_nchaintx(nchaintx) { }
+
+    SERIALIZE_METHODS(SnapshotMetadata, obj) { READWRITE(obj.m_base_blockhash, obj.m_coins_count, obj.m_nchaintx); }
+};
+
+#endif // BITCOIN_NODE_UTXO_SNAPSHOT_H

src/rpc/blockchain.cpp

@@ -18,6 +18,7 @@
 #include <key_io.h>
 #include <node/coinstats.h>
 #include <node/context.h>
+#include <node/utxo_snapshot.h>
 #include <policy/policy.h>
 #include <primitives/transaction.h>
 #include <rpc/server.h>
@@ -2673,6 +2674,114 @@ static UniValue getblockfilter(const JSONRPCRequest& request)
     return ret;
 }
 
+/**
+ * Serialize the UTXO set to a file for loading elsewhere.
+ *
+ * @see SnapshotMetadata
+ */
+UniValue dumptxoutset(const JSONRPCRequest& request)
+{
+    RPCHelpMan{
+        "dumptxoutset",
+        "\nWrite the serialized UTXO set to disk.\n"
+        "Incidentally flushes the latest coinsdb (leveldb) to disk.\n",
+        {
+            {"path",
+                RPCArg::Type::STR,
+                RPCArg::Optional::NO,
+                /* default_val */ "",
+                "path to the output file. If relative, will be prefixed by datadir."},
+        },
+        RPCResult{
+            RPCResult::Type::OBJ, "", "",
+                {
+                    {RPCResult::Type::NUM, "coins_written", "the number of coins written in the snapshot"},
+                    {RPCResult::Type::STR_HEX, "base_hash", "the hash of the base of the snapshot"},
+                    {RPCResult::Type::NUM, "base_height", "the height of the base of the snapshot"},
+                    {RPCResult::Type::STR, "path", "the absolute path that the snapshot was written to"},
+                }
+        },
+        RPCExamples{
+            HelpExampleCli("dumptxoutset", "utxo.dat")
+        }
+    }.Check(request);
+
+    fs::path path = fs::absolute(request.params[0].get_str(), GetDataDir());
+    // Write to a temporary path and then move into `path` on completion
+    // to avoid confusion due to an interruption.
+    fs::path temppath = fs::absolute(request.params[0].get_str() + ".incomplete", GetDataDir());
+
+    if (fs::exists(path)) {
+        throw JSONRPCError(
+            RPC_INVALID_PARAMETER,
+            path.string() + " already exists. If you are sure this is what you want, "
+            "move it out of the way first");
+    }
+
+    FILE* file{fsbridge::fopen(temppath, "wb")};
+    CAutoFile afile{file, SER_DISK, CLIENT_VERSION};
+    std::unique_ptr<CCoinsViewCursor> pcursor;
+    CCoinsStats stats;
+    CBlockIndex* tip;
+
+    {
+        // We need to lock cs_main to ensure that the coinsdb isn't written to
+        // between (i) flushing coins cache to disk (coinsdb), (ii) getting stats
+        // based upon the coinsdb, and (iii) constructing a cursor to the
+        // coinsdb for use below this block.
+        //
+        // Cursors returned by leveldb iterate over snapshots, so the contents
+        // of the pcursor will not be affected by simultaneous writes during
+        // use below this block.
+        //
+        // See discussion here:
+        //   https://github.com/bitcoin/bitcoin/pull/15606#discussion_r274479369
+        //
+        LOCK(::cs_main);
+
+        ::ChainstateActive().ForceFlushStateToDisk();
+
+        if (!GetUTXOStats(&::ChainstateActive().CoinsDB(), stats, CoinStatsHashType::NONE)) {
+            throw JSONRPCError(RPC_INTERNAL_ERROR, "Unable to read UTXO set");
+        }
+
+        pcursor = std::unique_ptr<CCoinsViewCursor>(::ChainstateActive().CoinsDB().Cursor());
+        tip = LookupBlockIndex(stats.hashBlock);
+        CHECK_NONFATAL(tip);
+    }
+
+    SnapshotMetadata metadata{tip->GetBlockHash(), stats.coins_count, tip->nChainTx};
+
+    afile << metadata;
+
+    COutPoint key;
+    Coin coin;
+    unsigned int iter{0};
+
+    while (pcursor->Valid()) {
+        if (iter % 5000 == 0 && !IsRPCRunning()) {
+            throw JSONRPCError(RPC_CLIENT_NOT_CONNECTED, "Shutting down");
+        }
+        ++iter;
+        if (pcursor->GetKey(key) && pcursor->GetValue(coin)) {
+            afile << key;
+            afile << coin;
+        }
+
+        pcursor->Next();
+    }
+
+    afile.fclose();
+    fs::rename(temppath, path);
+
+    UniValue result(UniValue::VOBJ);
+    result.pushKV("coins_written", stats.coins_count);
+    result.pushKV("base_hash", tip->GetBlockHash().ToString());
+    result.pushKV("base_height", tip->nHeight);
+    result.pushKV("path", path.string());
+    return result;
+}
+
 // clang-format off
 static const CRPCCommand commands[] =
 { //  category              name                      actor (function)         argNames
@@ -2714,6 +2823,7 @@ static const CRPCCommand commands[] =
     { "hidden",             "waitforblock",           &waitforblock,           {"blockhash","timeout"} },
     { "hidden",             "waitforblockheight",     &waitforblockheight,     {"height","timeout"} },
     { "hidden",             "syncwithvalidationinterfacequeue", &syncwithvalidationinterfacequeue, {} },
+    { "hidden",             "dumptxoutset",           &dumptxoutset,           {"path"} },
 };
 // clang-format on
 

src/validation.h

@@ -23,6 +23,7 @@
 #include <txdb.h>
 #include <txmempool.h> // For CTxMemPool::cs
 #include <versionbits.h>
+#include <serialize.h>
 #include <spentindex.h>
 
 #include <algorithm>

test/functional/rpc_dumptxoutset.py

@@ -0,0 +1,51 @@
+#!/usr/bin/env python3
+# Copyright (c) 2019 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+"""Test the generation of UTXO snapshots using `dumptxoutset`.
+"""
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.util import assert_equal, assert_raises_rpc_error
+
+import hashlib
+from pathlib import Path
+
+
+class DumptxoutsetTest(BitcoinTestFramework):
+    def set_test_params(self):
+        self.setup_clean_chain = True
+        self.num_nodes = 1
+
+    def run_test(self):
+        """Test a trivial usage of the dumptxoutset RPC command."""
+        node = self.nodes[0]
+        mocktime = node.getblockheader(node.getblockhash(0))['time'] + 1
+        node.setmocktime(mocktime)
+        node.generate(100)
+
+        FILENAME = 'txoutset.dat'
+        out = node.dumptxoutset(FILENAME)
+        expected_path = Path(node.datadir) / self.chain / FILENAME
+
+        assert expected_path.is_file()
+
+        assert_equal(out['coins_written'], 100)
+        assert_equal(out['base_height'], 100)
+        assert_equal(out['path'], str(expected_path))
+        # Blockhash should be deterministic based on mocked time.
+        assert_equal(
+            out['base_hash'],
+            '65a627bab5f50aea8e69acbf9fcbe6e5162bd556ce1c46f0ec4452afaedbb702')
+
+        with open(str(expected_path), 'rb') as f:
+            digest = hashlib.sha256(f.read()).hexdigest()
+            # UTXO snapshot hash should be deterministic based on mocked time.
+            assert_equal(
+                digest, '1d34e230e9d7d5691aaa03684492798ca8f92a9254c6cf80528241bf35939869')
+
+        # Specifying a path to an existing file will fail.
+        assert_raises_rpc_error(
+            -8, '{} already exists'.format(FILENAME),  node.dumptxoutset, FILENAME)
+
+if __name__ == '__main__':
+    DumptxoutsetTest().main()

test/functional/test_runner.py

@@ -217,6 +217,7 @@ BASE_SCRIPTS = [
     'rpc_uptime.py',
     'wallet_resendwallettransactions.py',
     'wallet_fallbackfee.py',
+    'rpc_dumptxoutset.py',
     'feature_minchainwork.py',
     'p2p_unrequested_blocks.py', # NOTE: needs dash_hash to pass
     'feature_shutdown.py',