merge bitcoin#24358: USDT tracepoint interface tests

configure.ac

@@ -1419,6 +1419,7 @@ if test x$use_usdt != xno; then
     [AC_MSG_RESULT(no); use_usdt=no;]
   )
 fi
+AM_CONDITIONAL([ENABLE_USDT_TRACEPOINTS], [test "$use_usdt" = "yes"])
 
 dnl Check for libminiupnpc (optional)
 if test x$use_upnp != xno; then

test/config.ini.in

@@ -23,3 +23,4 @@ RPCAUTH=@abs_top_srcdir@/share/rpcauth/rpcauth.py
 @BUILD_BITCOIND_TRUE@ENABLE_BITCOIND=true
 @ENABLE_FUZZ_TRUE@ENABLE_FUZZ=true
 @ENABLE_ZMQ_TRUE@ENABLE_ZMQ=true
+@ENABLE_USDT_TRACEPOINTS_TRUE@ENABLE_USDT_TRACEPOINTS=true

test/functional/interface_usdt_net.py

@@ -0,0 +1,171 @@
+#!/usr/bin/env python3
+# Copyright (c) 2022 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+"""  Tests the net:* tracepoint API interface.
+     See https://github.com/dashpay/dash/blob/develop/doc/tracing.md#context-net
+"""
+
+import ctypes
+from io import BytesIO
+# Test will be skipped if we don't have bcc installed
+try:
+    from bcc import BPF, USDT # type: ignore[import]
+except ImportError:
+    pass
+from test_framework.messages import msg_version
+from test_framework.p2p import P2PInterface
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.util import assert_equal
+
+# Tor v3 addresses are 62 chars + 6 chars for the port (':12345').
+MAX_PEER_ADDR_LENGTH = 68
+MAX_PEER_CONN_TYPE_LENGTH = 20
+MAX_MSG_TYPE_LENGTH = 20
+# We won't process messages larger than 150 byte in this test. For reading
+# larger messanges see contrib/tracing/log_raw_p2p_msgs.py
+MAX_MSG_DATA_LENGTH = 150
+
+net_tracepoints_program = """
+#include <uapi/linux/ptrace.h>
+
+#define MAX_PEER_ADDR_LENGTH {}
+#define MAX_PEER_CONN_TYPE_LENGTH {}
+#define MAX_MSG_TYPE_LENGTH {}
+#define MAX_MSG_DATA_LENGTH {}
+""".format(
+    MAX_PEER_ADDR_LENGTH,
+    MAX_PEER_CONN_TYPE_LENGTH,
+    MAX_MSG_TYPE_LENGTH,
+    MAX_MSG_DATA_LENGTH
+) + """
+#define MIN(a,b) ({ __typeof__ (a) _a = (a); __typeof__ (b) _b = (b); _a < _b ? _a : _b; })
+
+struct p2p_message
+{
+    u64     peer_id;
+    char    peer_addr[MAX_PEER_ADDR_LENGTH];
+    char    peer_conn_type[MAX_PEER_CONN_TYPE_LENGTH];
+    char    msg_type[MAX_MSG_TYPE_LENGTH];
+    u64     msg_size;
+    u8      msg[MAX_MSG_DATA_LENGTH];
+};
+
+BPF_PERF_OUTPUT(inbound_messages);
+int trace_inbound_message(struct pt_regs *ctx) {
+    struct p2p_message msg = {};
+    bpf_usdt_readarg(1, ctx, &msg.peer_id);
+    bpf_usdt_readarg_p(2, ctx, &msg.peer_addr, MAX_PEER_ADDR_LENGTH);
+    bpf_usdt_readarg_p(3, ctx, &msg.peer_conn_type, MAX_PEER_CONN_TYPE_LENGTH);
+    bpf_usdt_readarg_p(4, ctx, &msg.msg_type, MAX_MSG_TYPE_LENGTH);
+    bpf_usdt_readarg(5, ctx, &msg.msg_size);
+    bpf_usdt_readarg_p(6, ctx, &msg.msg, MIN(msg.msg_size, MAX_MSG_DATA_LENGTH));
+    inbound_messages.perf_submit(ctx, &msg, sizeof(msg));
+    return 0;
+}
+
+BPF_PERF_OUTPUT(outbound_messages);
+int trace_outbound_message(struct pt_regs *ctx) {
+    struct p2p_message msg = {};
+    bpf_usdt_readarg(1, ctx, &msg.peer_id);
+    bpf_usdt_readarg_p(2, ctx, &msg.peer_addr, MAX_PEER_ADDR_LENGTH);
+    bpf_usdt_readarg_p(3, ctx, &msg.peer_conn_type, MAX_PEER_CONN_TYPE_LENGTH);
+    bpf_usdt_readarg_p(4, ctx, &msg.msg_type, MAX_MSG_TYPE_LENGTH);
+    bpf_usdt_readarg(5, ctx, &msg.msg_size);
+    bpf_usdt_readarg_p(6, ctx, &msg.msg, MIN(msg.msg_size, MAX_MSG_DATA_LENGTH));
+    outbound_messages.perf_submit(ctx, &msg, sizeof(msg));
+    return 0;
+};
+"""
+
+
+class NetTracepointTest(BitcoinTestFramework):
+    def set_test_params(self):
+        self.num_nodes = 1
+
+    def skip_test_if_missing_module(self):
+        self.skip_if_platform_not_linux()
+        self.skip_if_no_bitcoind_tracepoints()
+        self.skip_if_no_python_bcc()
+        self.skip_if_no_bpf_permissions()
+
+    def run_test(self):
+        # Tests the net:inbound_message and net:outbound_message tracepoints
+        # See https://github.com/dashpay/dash/blob/develop/doc/tracing.md#context-net
+
+        class P2PMessage(ctypes.Structure):
+            _fields_ = [
+                ("peer_id", ctypes.c_uint64),
+                ("peer_addr", ctypes.c_char * MAX_PEER_ADDR_LENGTH),
+                ("peer_conn_type", ctypes.c_char * MAX_PEER_CONN_TYPE_LENGTH),
+                ("msg_type", ctypes.c_char * MAX_MSG_TYPE_LENGTH),
+                ("msg_size", ctypes.c_uint64),
+                ("msg", ctypes.c_ubyte * MAX_MSG_DATA_LENGTH),
+            ]
+
+            def __repr__(self):
+                return f"P2PMessage(peer={self.peer_id}, addr={self.peer_addr.decode('utf-8')}, conn_type={self.peer_conn_type.decode('utf-8')}, msg_type={self.msg_type.decode('utf-8')}, msg_size={self.msg_size})"
+
+        self.log.info(
+            "hook into the net:inbound_message and net:outbound_message tracepoints")
+        ctx = USDT(path=str(self.options.bitcoind))
+        ctx.enable_probe(probe="net:inbound_message",
+                         fn_name="trace_inbound_message")
+        ctx.enable_probe(probe="net:outbound_message",
+                         fn_name="trace_outbound_message")
+        bpf = BPF(text=net_tracepoints_program, usdt_contexts=[ctx], debug=0)
+
+        # The handle_* function is a ctypes callback function called from C. When
+        # we assert in the handle_* function, the AssertError doesn't propagate
+        # back to Python. The exception is ignored. We manually count and assert
+        # that the handle_* functions succeeded.
+        EXPECTED_INOUTBOUND_VERSION_MSG = 1
+        checked_inbound_version_msg = 0
+        checked_outbound_version_msg = 0
+
+        def check_p2p_message(event, inbound):
+            nonlocal checked_inbound_version_msg, checked_outbound_version_msg
+            if event.msg_type.decode("utf-8") == "version":
+                self.log.info(
+                    f"check_p2p_message(): {'inbound' if inbound else 'outbound'} {event}")
+                peer = self.nodes[0].getpeerinfo()[0]
+                msg = msg_version()
+                msg.deserialize(BytesIO(bytes(event.msg[:event.msg_size])))
+                assert_equal(peer["id"], event.peer_id, peer["id"])
+                assert_equal(peer["addr"], event.peer_addr.decode("utf-8"))
+                assert_equal(peer["connection_type"],
+                             event.peer_conn_type.decode("utf-8"))
+                if inbound:
+                    checked_inbound_version_msg += 1
+                else:
+                    checked_outbound_version_msg += 1
+
+        def handle_inbound(_, data, __):
+            event = ctypes.cast(data, ctypes.POINTER(P2PMessage)).contents
+            check_p2p_message(event, True)
+
+        def handle_outbound(_, data, __):
+            event = ctypes.cast(data, ctypes.POINTER(P2PMessage)).contents
+            check_p2p_message(event, False)
+
+        bpf["inbound_messages"].open_perf_buffer(handle_inbound)
+        bpf["outbound_messages"].open_perf_buffer(handle_outbound)
+
+        self.log.info("connect a P2P test node to our dashd node")
+        test_node = P2PInterface()
+        self.nodes[0].add_p2p_connection(test_node)
+        bpf.perf_buffer_poll(timeout=200)
+
+        self.log.info(
+            "check that we got both an inbound and outbound version message")
+        assert_equal(EXPECTED_INOUTBOUND_VERSION_MSG,
+                     checked_inbound_version_msg)
+        assert_equal(EXPECTED_INOUTBOUND_VERSION_MSG,
+                     checked_outbound_version_msg)
+
+        bpf.cleanup()
+
+
+if __name__ == '__main__':
+    NetTracepointTest().main()

test/functional/interface_usdt_utxocache.py

@@ -0,0 +1,408 @@
+#!/usr/bin/env python3
+# Copyright (c) 2022 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+""" Tests the utxocache:* tracepoint API interface.
+    See https://github.com/dashpay/dash/blob/develop/doc/tracing.md#context-utxocache
+"""
+
+import ctypes
+# Test will be skipped if we don't have bcc installed
+try:
+    from bcc import BPF, USDT # type: ignore[import]
+except ImportError:
+    pass
+from test_framework.messages import COIN
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.util import assert_equal
+from test_framework.wallet import MiniWallet
+
+utxocache_changes_program = """
+#include <uapi/linux/ptrace.h>
+
+typedef signed long long i64;
+
+struct utxocache_change
+{
+    char        txid[32];
+    u32         index;
+    u32         height;
+    i64         value;
+    bool        is_coinbase;
+};
+
+BPF_PERF_OUTPUT(utxocache_add);
+int trace_utxocache_add(struct pt_regs *ctx) {
+    struct utxocache_change add = {};
+    bpf_usdt_readarg_p(1, ctx, &add.txid, 32);
+    bpf_usdt_readarg(2, ctx, &add.index);
+    bpf_usdt_readarg(3, ctx, &add.height);
+    bpf_usdt_readarg(4, ctx, &add.value);
+    bpf_usdt_readarg(5, ctx, &add.is_coinbase);
+    utxocache_add.perf_submit(ctx, &add, sizeof(add));
+    return 0;
+}
+
+BPF_PERF_OUTPUT(utxocache_spent);
+int trace_utxocache_spent(struct pt_regs *ctx) {
+    struct utxocache_change spent = {};
+    bpf_usdt_readarg_p(1, ctx, &spent.txid, 32);
+    bpf_usdt_readarg(2, ctx, &spent.index);
+    bpf_usdt_readarg(3, ctx, &spent.height);
+    bpf_usdt_readarg(4, ctx, &spent.value);
+    bpf_usdt_readarg(5, ctx, &spent.is_coinbase);
+    utxocache_spent.perf_submit(ctx, &spent, sizeof(spent));
+    return 0;
+}
+
+BPF_PERF_OUTPUT(utxocache_uncache);
+int trace_utxocache_uncache(struct pt_regs *ctx) {
+    struct utxocache_change uncache = {};
+    bpf_usdt_readarg_p(1, ctx, &uncache.txid, 32);
+    bpf_usdt_readarg(2, ctx, &uncache.index);
+    bpf_usdt_readarg(3, ctx, &uncache.height);
+    bpf_usdt_readarg(4, ctx, &uncache.value);
+    bpf_usdt_readarg(5, ctx, &uncache.is_coinbase);
+    utxocache_uncache.perf_submit(ctx, &uncache, sizeof(uncache));
+    return 0;
+}
+"""
+
+utxocache_flushes_program = """
+#include <uapi/linux/ptrace.h>
+
+typedef signed long long i64;
+
+struct utxocache_flush
+{
+    i64         duration;
+    u32         mode;
+    u64         size;
+    u64         memory;
+    bool        for_prune;
+};
+
+BPF_PERF_OUTPUT(utxocache_flush);
+int trace_utxocache_flush(struct pt_regs *ctx) {
+    struct utxocache_flush flush = {};
+    bpf_usdt_readarg(1, ctx, &flush.duration);
+    bpf_usdt_readarg(2, ctx, &flush.mode);
+    bpf_usdt_readarg(3, ctx, &flush.size);
+    bpf_usdt_readarg(4, ctx, &flush.memory);
+    bpf_usdt_readarg(5, ctx, &flush.for_prune);
+    utxocache_flush.perf_submit(ctx, &flush, sizeof(flush));
+    return 0;
+}
+"""
+
+FLUSHMODE_NAME = {
+    0: "NONE",
+    1: "IF_NEEDED",
+    2: "PERIODIC",
+    3: "ALWAYS",
+}
+
+
+class UTXOCacheChange(ctypes.Structure):
+    _fields_ = [
+        ("txid", ctypes.c_ubyte * 32),
+        ("index", ctypes.c_uint32),
+        ("height", ctypes.c_uint32),
+        ("value", ctypes.c_uint64),
+        ("is_coinbase", ctypes.c_bool),
+    ]
+
+    def __repr__(self):
+        return f"UTXOCacheChange(outpoint={bytes(self.txid[::-1]).hex()}:{self.index}, height={self.height}, value={self.value}sat, is_coinbase={self.is_coinbase})"
+
+
+class UTXOCacheFlush(ctypes.Structure):
+    _fields_ = [
+        ("duration", ctypes.c_int64),
+        ("mode", ctypes.c_uint32),
+        ("size", ctypes.c_uint64),
+        ("memory", ctypes.c_uint64),
+        ("for_prune", ctypes.c_bool),
+    ]
+
+    def __repr__(self):
+        return f"UTXOCacheFlush(duration={self.duration}, mode={FLUSHMODE_NAME[self.mode]}, size={self.size}, memory={self.memory}, for_prune={self.for_prune})"
+
+
+class UTXOCacheTracepointTest(BitcoinTestFramework):
+    def set_test_params(self):
+        self.setup_clean_chain = False
+        self.num_nodes = 1
+        self.extra_args = [["-txindex"]]
+
+    def skip_test_if_missing_module(self):
+        self.skip_if_platform_not_linux()
+        self.skip_if_no_bitcoind_tracepoints()
+        self.skip_if_no_python_bcc()
+        self.skip_if_no_bpf_permissions()
+
+    def run_test(self):
+        self.wallet = MiniWallet(self.nodes[0])
+        self.wallet.generate(101)
+
+        self.test_uncache()
+        self.test_add_spent()
+        self.test_flush()
+
+    def test_uncache(self):
+        """ Tests the utxocache:uncache tracepoint API.
+        https://github.com/dashpay/dash/blob/develop/doc/tracing.md#tracepoint-utxocacheuncache
+        """
+        # To trigger an UTXO uncache from the cache, we create an invalid transaction
+        # spending a not-cached, but existing UTXO. During transaction validation, this
+        # the UTXO is added to the utxo cache, but as the transaction is invalid, it's
+        # uncached again.
+        self.log.info("testing the utxocache:uncache tracepoint API")
+
+        # Retrieve the txid for the UTXO created in the first block. This UTXO is not
+        # in our UTXO cache.
+        EARLY_BLOCK_HEIGHT = 1
+        block_1_hash = self.nodes[0].getblockhash(EARLY_BLOCK_HEIGHT)
+        block_1 = self.nodes[0].getblock(block_1_hash)
+        block_1_coinbase_txid = block_1["tx"][0]
+
+        # Create a transaction and invalidate it by changing the txid of the previous
+        # output to the coinbase txid of the block at height 1.
+        invalid_tx = self.wallet.create_self_transfer(
+            from_node=self.nodes[0])["tx"]
+        invalid_tx.vin[0].prevout.hash = int(block_1_coinbase_txid, 16)
+
+        self.log.info("hooking into the utxocache:uncache tracepoint")
+        ctx = USDT(path=str(self.options.bitcoind))
+        ctx.enable_probe(probe="utxocache:uncache",
+                         fn_name="trace_utxocache_uncache")
+        bpf = BPF(text=utxocache_changes_program, usdt_contexts=[ctx], debug=0)
+
+        # The handle_* function is a ctypes callback function called from C. When
+        # we assert in the handle_* function, the AssertError doesn't propagate
+        # back to Python. The exception is ignored. We manually count and assert
+        # that the handle_* functions succeeded.
+        EXPECTED_HANDLE_UNCACHE_SUCCESS = 1
+        handle_uncache_succeeds = 0
+
+        def handle_utxocache_uncache(_, data, __):
+            nonlocal handle_uncache_succeeds
+            event = ctypes.cast(data, ctypes.POINTER(UTXOCacheChange)).contents
+            self.log.info(f"handle_utxocache_uncache(): {event}")
+            assert_equal(block_1_coinbase_txid, bytes(event.txid[::-1]).hex())
+            assert_equal(0, event.index)  # prevout index
+            assert_equal(EARLY_BLOCK_HEIGHT, event.height)
+            assert_equal(500 * COIN, event.value)
+            assert_equal(True, event.is_coinbase)
+
+            handle_uncache_succeeds += 1
+
+        bpf["utxocache_uncache"].open_perf_buffer(handle_utxocache_uncache)
+
+        self.log.info(
+            "testmempoolaccept the invalid transaction to trigger an UTXO-cache uncache")
+        result = self.nodes[0].testmempoolaccept(
+            [invalid_tx.serialize().hex()])[0]
+        assert_equal(result["allowed"], False)
+
+        bpf.perf_buffer_poll(timeout=100)
+        bpf.cleanup()
+
+        self.log.info(
+            f"check that we successfully traced {EXPECTED_HANDLE_UNCACHE_SUCCESS} uncaches")
+        assert_equal(EXPECTED_HANDLE_UNCACHE_SUCCESS, handle_uncache_succeeds)
+
+    def test_add_spent(self):
+        """ Tests the utxocache:add utxocache:spent tracepoint API
+            See https://github.com/dashpay/dash/blob/develop/doc/tracing.md#tracepoint-utxocacheadd
+            and https://github.com/dashpay/dash/blob/develop/doc/tracing.md#tracepoint-utxocachespent
+        """
+
+        self.log.info(
+            "test the utxocache:add and utxocache:spent tracepoint API")
+
+        self.log.info("create an unconfirmed transaction")
+        self.wallet.send_self_transfer(from_node=self.nodes[0])
+
+        # We mine a block to trace changes (add/spent) to the active in-memory cache
+        # of the UTXO set (see CoinsTip() of CCoinsViewCache). However, in some cases
+        # temporary clones of the active cache are made. For example, during mining with
+        # the generate RPC call, the block is first tested in TestBlockValidity(). There,
+        # a clone of the active cache is modified during a test ConnectBlock() call.
+        # These are implementation details we don't want to test here. Thus, after
+        # mining, we invalidate the block, start the tracing, and then trace the cache
+        # changes to the active utxo cache.
+        self.log.info("mine and invalidate a block that is later reconsidered")
+        block_hash = self.wallet.generate(1)[0]
+        self.nodes[0].invalidateblock(block_hash)
+
+        self.log.info(
+            "hook into the utxocache:add and utxocache:spent tracepoints")
+        ctx = USDT(path=str(self.options.bitcoind))
+        ctx.enable_probe(probe="utxocache:add", fn_name="trace_utxocache_add")
+        ctx.enable_probe(probe="utxocache:spent",
+                         fn_name="trace_utxocache_spent")
+        bpf = BPF(text=utxocache_changes_program, usdt_contexts=[ctx], debug=0)
+
+        # The handle_* function is a ctypes callback function called from C. When
+        # we assert in the handle_* function, the AssertError doesn't propagate
+        # back to Python. The exception is ignored. We manually count and assert
+        # that the handle_* functions succeeded.
+        EXPECTED_HANDLE_ADD_SUCCESS = 2
+        EXPECTED_HANDLE_SPENT_SUCCESS = 1
+        handle_add_succeeds = 0
+        handle_spent_succeeds = 0
+
+        expected_utxocache_spents = []
+        expected_utxocache_adds = []
+
+        def handle_utxocache_add(_, data, __):
+            nonlocal handle_add_succeeds
+            event = ctypes.cast(data, ctypes.POINTER(UTXOCacheChange)).contents
+            self.log.info(f"handle_utxocache_add(): {event}")
+            add = expected_utxocache_adds.pop(0)
+            assert_equal(add["txid"], bytes(event.txid[::-1]).hex())
+            assert_equal(add["index"], event.index)
+            assert_equal(add["height"], event.height)
+            assert_equal(add["value"], event.value)
+            assert_equal(add["is_coinbase"], event.is_coinbase)
+            handle_add_succeeds += 1
+
+        def handle_utxocache_spent(_, data, __):
+            nonlocal handle_spent_succeeds
+            event = ctypes.cast(data, ctypes.POINTER(UTXOCacheChange)).contents
+            self.log.info(f"handle_utxocache_spent(): {event}")
+            spent = expected_utxocache_spents.pop(0)
+            assert_equal(spent["txid"], bytes(event.txid[::-1]).hex())
+            assert_equal(spent["index"], event.index)
+            assert_equal(spent["height"], event.height)
+            assert_equal(spent["value"], event.value)
+            assert_equal(spent["is_coinbase"], event.is_coinbase)
+            handle_spent_succeeds += 1
+
+        bpf["utxocache_add"].open_perf_buffer(handle_utxocache_add)
+        bpf["utxocache_spent"].open_perf_buffer(handle_utxocache_spent)
+
+        # We trigger a block re-connection. This causes changes (add/spent)
+        # to the UTXO-cache which in turn triggers the tracepoints.
+        self.log.info("reconsider the previously invalidated block")
+        self.nodes[0].reconsiderblock(block_hash)
+
+        block = self.nodes[0].getblock(block_hash, 2)
+        for (block_index, tx) in enumerate(block["tx"]):
+            for vin in tx["vin"]:
+                if "coinbase" not in vin:
+                    prevout_tx = self.nodes[0].getrawtransaction(
+                        vin["txid"], True)
+                    prevout_tx_block = self.nodes[0].getblockheader(
+                        prevout_tx["blockhash"])
+                    spends_coinbase = "coinbase" in prevout_tx["vin"][0]
+                    expected_utxocache_spents.append({
+                        "txid": vin["txid"],
+                        "index": vin["vout"],
+                        "height": prevout_tx_block["height"],
+                        "value": int(prevout_tx["vout"][vin["vout"]]["value"] * COIN),
+                        "is_coinbase": spends_coinbase,
+                    })
+            for (i, vout) in enumerate(tx["vout"]):
+                if vout["scriptPubKey"]["type"] != "nulldata":
+                    expected_utxocache_adds.append({
+                        "txid": tx["txid"],
+                        "index": i,
+                        "height": block["height"],
+                        "value": int(vout["value"] * COIN),
+                        "is_coinbase": block_index == 0,
+                    })
+
+        assert_equal(EXPECTED_HANDLE_ADD_SUCCESS, len(expected_utxocache_adds))
+        assert_equal(EXPECTED_HANDLE_SPENT_SUCCESS,
+                     len(expected_utxocache_spents))
+
+        bpf.perf_buffer_poll(timeout=200)
+        bpf.cleanup()
+
+        self.log.info(
+            f"check that we successfully traced {EXPECTED_HANDLE_ADD_SUCCESS} adds and {EXPECTED_HANDLE_SPENT_SUCCESS} spent")
+        assert_equal(0, len(expected_utxocache_adds))
+        assert_equal(0, len(expected_utxocache_spents))
+        assert_equal(EXPECTED_HANDLE_ADD_SUCCESS, handle_add_succeeds)
+        assert_equal(EXPECTED_HANDLE_SPENT_SUCCESS, handle_spent_succeeds)
+
+    def test_flush(self):
+        """ Tests the utxocache:flush tracepoint API.
+            See https://github.com/dashpay/dash/blob/develop/doc/tracing.md#tracepoint-utxocacheflush"""
+
+        self.log.info("test the utxocache:flush tracepoint API")
+        self.log.info("hook into the utxocache:flush tracepoint")
+        ctx = USDT(path=str(self.options.bitcoind))
+        ctx.enable_probe(probe="utxocache:flush",
+                         fn_name="trace_utxocache_flush")
+        bpf = BPF(text=utxocache_flushes_program, usdt_contexts=[ctx], debug=0)
+
+        # The handle_* function is a ctypes callback function called from C. When
+        # we assert in the handle_* function, the AssertError doesn't propagate
+        # back to Python. The exception is ignored. We manually count and assert
+        # that the handle_* functions succeeded.
+        EXPECTED_HANDLE_FLUSH_SUCCESS = 3
+        handle_flush_succeeds = 0
+        possible_cache_sizes = set()
+        expected_flushes = []
+
+        def handle_utxocache_flush(_, data, __):
+            nonlocal handle_flush_succeeds
+            event = ctypes.cast(data, ctypes.POINTER(UTXOCacheFlush)).contents
+            self.log.info(f"handle_utxocache_flush(): {event}")
+            expected = expected_flushes.pop(0)
+            assert_equal(expected["mode"], FLUSHMODE_NAME[event.mode])
+            possible_cache_sizes.remove(event.size)  # fails if size not in set
+            # sanity checks only
+            assert(event.memory > 0)
+            assert(event.duration > 0)
+            handle_flush_succeeds += 1
+
+        bpf["utxocache_flush"].open_perf_buffer(handle_utxocache_flush)
+
+        self.log.info("stop the node to flush the UTXO cache")
+        UTXOS_IN_CACHE = 104  # might need to be changed if the eariler tests are modified
+        # A node shutdown causes two flushes. One that flushes UTXOS_IN_CACHE
+        # UTXOs and one that flushes 0 UTXOs. Normally the 0-UTXO-flush is the
+        # second flush, however it can happen that the order changes.
+        possible_cache_sizes = {UTXOS_IN_CACHE, 0}
+        flush_for_shutdown = {"mode": "ALWAYS", "for_prune": False}
+        expected_flushes.extend([flush_for_shutdown, flush_for_shutdown])
+        self.stop_node(0)
+
+        bpf.perf_buffer_poll(timeout=200)
+
+        self.log.info("check that we don't expect additional flushes")
+        assert_equal(0, len(expected_flushes))
+        assert_equal(0, len(possible_cache_sizes))
+
+        self.log.info("restart the node with -prune")
+        self.start_node(0, ["-fastprune=1", "-prune=1"])
+
+        BLOCKS_TO_MINE = 450
+        self.log.info(f"mine {BLOCKS_TO_MINE} blocks to be able to prune")
+        self.wallet.generate(BLOCKS_TO_MINE)
+        # we added BLOCKS_TO_MINE coinbase UTXOs to the cache
+        possible_cache_sizes = {BLOCKS_TO_MINE}
+        expected_flushes.append(
+            {"mode": "NONE", "for_prune": True, "size_fn": lambda x: x == BLOCKS_TO_MINE})
+
+        self.log.info(f"prune blockchain to trigger a flush for pruning")
+        self.nodes[0].pruneblockchain(415)
+
+        bpf.perf_buffer_poll(timeout=500)
+        bpf.cleanup()
+
+        self.log.info(
+            f"check that we don't expect additional flushes and that the handle_* function succeeded")
+        assert_equal(0, len(expected_flushes))
+        assert_equal(0, len(possible_cache_sizes))
+        assert_equal(EXPECTED_HANDLE_FLUSH_SUCCESS, handle_flush_succeeds)
+
+        self.stop_node(0, expected_stderr='Warning: You are starting with governance validation disabled. This is expected because you are running a pruned node.')
+
+if __name__ == '__main__':
+    UTXOCacheTracepointTest().main()

test/functional/interface_usdt_validation.py

@@ -0,0 +1,136 @@
+#!/usr/bin/env python3
+# Copyright (c) 2022 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+""" Tests the validation:* tracepoint API interface.
+    See https://github.com/dashpay/dash/blob/develop/doc/tracing.md#context-validation
+"""
+
+import ctypes
+
+# Test will be skipped if we don't have bcc installed
+try:
+    from bcc import BPF, USDT # type: ignore[import]
+except ImportError:
+    pass
+
+from test_framework.address import ADDRESS_BCRT1_UNSPENDABLE
+from test_framework.test_framework import BitcoinTestFramework
+from test_framework.util import assert_equal
+
+
+validation_blockconnected_program = """
+#include <uapi/linux/ptrace.h>
+
+typedef signed long long i64;
+
+struct connected_block
+{
+    char        hash[32];
+    int         height;
+    i64         transactions;
+    int         inputs;
+    i64         sigops;
+    u64         duration;
+};
+
+BPF_PERF_OUTPUT(block_connected);
+int trace_block_connected(struct pt_regs *ctx) {
+    struct connected_block block = {};
+    bpf_usdt_readarg_p(1, ctx, &block.hash, 32);
+    bpf_usdt_readarg(2, ctx, &block.height);
+    bpf_usdt_readarg(3, ctx, &block.transactions);
+    bpf_usdt_readarg(4, ctx, &block.inputs);
+    bpf_usdt_readarg(5, ctx, &block.sigops);
+    bpf_usdt_readarg(6, ctx, &block.duration);
+    block_connected.perf_submit(ctx, &block, sizeof(block));
+    return 0;
+}
+"""
+
+
+class ValidationTracepointTest(BitcoinTestFramework):
+    def set_test_params(self):
+        self.num_nodes = 1
+
+    def skip_test_if_missing_module(self):
+        self.skip_if_platform_not_linux()
+        self.skip_if_no_bitcoind_tracepoints()
+        self.skip_if_no_python_bcc()
+        self.skip_if_no_bpf_permissions()
+
+    def run_test(self):
+        # Tests the validation:block_connected tracepoint by generating blocks
+        # and comparing the values passed in the tracepoint arguments with the
+        # blocks.
+        # See https://github.com/dashpay/dash/blob/develop/doc/tracing.md#tracepoint-validationblock_connected
+
+        class Block(ctypes.Structure):
+            _fields_ = [
+                ("hash", ctypes.c_ubyte * 32),
+                ("height", ctypes.c_int),
+                ("transactions", ctypes.c_int64),
+                ("inputs", ctypes.c_int),
+                ("sigops", ctypes.c_int64),
+                ("duration", ctypes.c_uint64),
+            ]
+
+            def __repr__(self):
+                return "ConnectedBlock(hash=%s height=%d, transactions=%d, inputs=%d, sigops=%d, duration=%d)" % (
+                    bytes(self.hash[::-1]).hex(),
+                    self.height,
+                    self.transactions,
+                    self.inputs,
+                    self.sigops,
+                    self.duration)
+
+        # The handle_* function is a ctypes callback function called from C. When
+        # we assert in the handle_* function, the AssertError doesn't propagate
+        # back to Python. The exception is ignored. We manually count and assert
+        # that the handle_* functions succeeded.
+        BLOCKS_EXPECTED = 2
+        blocks_checked = 0
+        expected_blocks = list()
+
+        self.log.info("hook into the validation:block_connected tracepoint")
+        ctx = USDT(path=str(self.options.bitcoind))
+        ctx.enable_probe(probe="validation:block_connected",
+                         fn_name="trace_block_connected")
+        bpf = BPF(text=validation_blockconnected_program,
+                  usdt_contexts=[ctx], debug=0)
+
+        def handle_blockconnected(_, data, __):
+            nonlocal expected_blocks, blocks_checked
+            event = ctypes.cast(data, ctypes.POINTER(Block)).contents
+            self.log.info(f"handle_blockconnected(): {event}")
+            block = expected_blocks.pop(0)
+            assert_equal(block["hash"], bytes(event.hash[::-1]).hex())
+            assert_equal(block["height"], event.height)
+            assert_equal(len(block["tx"]), event.transactions)
+            assert_equal(len([tx["vin"] for tx in block["tx"]]), event.inputs)
+            assert_equal(0, event.sigops)  # no sigops in coinbase tx
+            # only plausibility checks
+            assert(event.duration > 0)
+
+            blocks_checked += 1
+
+        bpf["block_connected"].open_perf_buffer(
+            handle_blockconnected)
+
+        self.log.info(f"mine {BLOCKS_EXPECTED} blocks")
+        block_hashes = self.nodes[0].generatetoaddress(
+            BLOCKS_EXPECTED, ADDRESS_BCRT1_UNSPENDABLE)
+        for block_hash in block_hashes:
+            expected_blocks.append(self.nodes[0].getblock(block_hash, 2))
+
+        bpf.perf_buffer_poll(timeout=200)
+        bpf.cleanup()
+
+        self.log.info(f"check that we traced {BLOCKS_EXPECTED} blocks")
+        assert_equal(BLOCKS_EXPECTED, blocks_checked)
+        assert_equal(0, len(expected_blocks))
+
+
+if __name__ == '__main__':
+    ValidationTracepointTest().main()

test/functional/test_framework/test_framework.py

@@ -12,6 +12,7 @@ from enum import Enum
 import argparse
 import logging
 import os
+import platform
 import pdb
 import random
 import re
@@ -955,6 +956,29 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass):
         except ImportError:
             raise SkipTest("python3-zmq module not available.")
 
+    def skip_if_no_python_bcc(self):
+        """Attempt to import the bcc package and skip the tests if the import fails."""
+        try:
+            import bcc  # type: ignore[import] # noqa: F401
+        except ImportError:
+            raise SkipTest("bcc python module not available")
+
+    def skip_if_no_bitcoind_tracepoints(self):
+        """Skip the running test if dashd has not been compiled with USDT tracepoint support."""
+        if not self.is_usdt_compiled():
+            raise SkipTest("dashd has not been built with USDT tracepoints enabled.")
+
+    def skip_if_no_bpf_permissions(self):
+        """Skip the running test if we don't have permissions to do BPF syscalls and load BPF maps."""
+        # check for 'root' permissions
+        if os.geteuid() != 0:
+            raise SkipTest("no permissions to use BPF (please review the tests carefully before running them with higher privileges)")
+
+    def skip_if_platform_not_linux(self):
+        """Skip the running test if we are not on a Linux platform"""
+        if platform.system() != "Linux":
+            raise SkipTest("not on a Linux system")
+
     def skip_if_no_bitcoind_zmq(self):
         """Skip the running test if dashd has not been compiled with zmq support."""
         if not self.is_zmq_compiled():
@@ -1019,6 +1043,10 @@ class BitcoinTestFramework(metaclass=BitcoinTestMetaClass):
         """Checks whether the zmq module was compiled."""
         return self.config["components"].getboolean("ENABLE_ZMQ")
 
+    def is_usdt_compiled(self):
+        """Checks whether the USDT tracepoints were compiled."""
+        return self.config["components"].getboolean("ENABLE_USDT_TRACEPOINTS")
+
     def is_sqlite_compiled(self):
         """Checks whether the wallet module was compiled with Sqlite support."""
         return self.config["components"].getboolean("USE_SQLITE")

test/functional/test_runner.py

@@ -186,6 +186,9 @@ BASE_SCRIPTS = [
     'wallet_watchonly.py --usecli --legacy-wallet',
     'interface_http.py',
     'interface_rpc.py',
+    'interface_usdt_net.py',
+    'interface_usdt_utxocache.py',
+    'interface_usdt_validation.py',
     'rpc_psbt.py --legacy-wallet',
     'rpc_psbt.py --descriptors',
     'rpc_users.py',