2016-05-06 11:23:48 +02:00
#!/usr/bin/env python3
2023-08-16 19:27:31 +02:00
# Copyright (c) 2014-2020 The Bitcoin Core developers
2015-08-26 12:05:36 +02:00
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
2019-05-19 22:20:34 +02:00
""" Run regression test suite.
2015-10-11 07:41:19 +02:00
This module calls down into individual test cases via subprocess . It will
2019-01-03 10:18:47 +01:00
forward all unrecognized arguments onto the individual test scripts .
2015-10-11 07:41:19 +02:00
For a description of arguments recognized by test scripts , see
2019-05-19 22:20:34 +02:00
` test / functional / test_framework / test_framework . py : BitcoinTestFramework . main ` .
2015-10-11 07:41:19 +02:00
"""
2015-08-26 12:05:36 +02:00
2019-01-03 10:18:47 +01:00
import argparse
2017-11-29 19:21:51 +01:00
from collections import deque
2019-01-03 10:18:47 +01:00
import configparser
2017-05-22 08:59:11 +02:00
import datetime
2015-08-26 12:05:36 +02:00
import os
2015-11-30 14:53:07 +01:00
import time
2015-10-11 07:41:19 +02:00
import shutil
2021-06-18 14:22:48 +02:00
import signal
2015-08-26 12:05:36 +02:00
import subprocess
2020-03-23 12:37:38 +01:00
import sys
2015-10-11 07:41:19 +02:00
import tempfile
2015-08-26 12:05:36 +02:00
import re
2017-03-28 11:24:14 +02:00
import logging
2020-03-26 21:57:53 +01:00
import unittest
2015-10-11 07:41:19 +02:00
2017-04-07 16:20:39 +02:00
# Formatting. Default colors to empty strings.
2022-05-10 13:12:05 +02:00
DEFAULT , BOLD , GREEN , RED = ( " " , " " ) , ( " " , " " ) , ( " " , " " ) , ( " " , " " )
2017-04-17 21:54:27 +02:00
try :
# Make sure python thinks it can write unicode to its stdout
" \u2713 " . encode ( " utf_8 " ) . decode ( sys . stdout . encoding )
TICK = " ✓ "
CROSS = " ✖ "
CIRCLE = " ○ "
except UnicodeDecodeError :
TICK = " P "
CROSS = " x "
CIRCLE = " o "
2018-09-24 22:10:13 +02:00
if os . name != ' nt ' or sys . getwindowsversion ( ) > = ( 10 , 0 , 14393 ) :
if os . name == ' nt ' :
import ctypes
2020-06-03 16:01:29 +02:00
kernel32 = ctypes . windll . kernel32 # type: ignore
2018-09-24 22:10:13 +02:00
ENABLE_VIRTUAL_TERMINAL_PROCESSING = 4
STD_OUTPUT_HANDLE = - 11
STD_ERROR_HANDLE = - 12
# Enable ascii color control to stdout
stdout = kernel32 . GetStdHandle ( STD_OUTPUT_HANDLE )
stdout_mode = ctypes . c_int32 ( )
kernel32 . GetConsoleMode ( stdout , ctypes . byref ( stdout_mode ) )
kernel32 . SetConsoleMode ( stdout , stdout_mode . value | ENABLE_VIRTUAL_TERMINAL_PROCESSING )
# Enable ascii color control to stderr
stderr = kernel32 . GetStdHandle ( STD_ERROR_HANDLE )
stderr_mode = ctypes . c_int32 ( )
kernel32 . GetConsoleMode ( stderr , ctypes . byref ( stderr_mode ) )
kernel32 . SetConsoleMode ( stderr , stderr_mode . value | ENABLE_VIRTUAL_TERMINAL_PROCESSING )
2017-04-07 16:20:39 +02:00
# primitive formatting on supported
# terminal via ANSI escape sequences:
2022-05-10 13:12:05 +02:00
DEFAULT = ( ' \033 [0m ' , ' \033 [0m ' )
2017-04-07 16:20:39 +02:00
BOLD = ( ' \033 [0m ' , ' \033 [1m ' )
2018-09-24 22:10:13 +02:00
GREEN = ( ' \033 [0m ' , ' \033 [0;32m ' )
2017-04-07 16:20:39 +02:00
RED = ( ' \033 [0m ' , ' \033 [0;31m ' )
2017-03-24 13:57:36 +01:00
TEST_EXIT_PASSED = 0
TEST_EXIT_SKIPPED = 77
2023-04-29 10:19:36 +02:00
# List of framework modules containing unit tests. Should be kept in sync with
# the output of `git grep unittest.TestCase ./test/functional/test_framework`
2020-03-26 21:57:53 +01:00
TEST_FRAMEWORK_MODULES = [
2020-05-30 18:33:34 +02:00
" address " ,
2020-06-04 17:26:47 +02:00
" blocktools " ,
2024-02-15 18:08:35 +01:00
" ellswift " ,
2024-02-14 11:48:41 +01:00
" key " ,
2024-02-15 18:08:35 +01:00
" muhash " ,
2023-04-29 10:19:36 +02:00
" ripemd160 " ,
2020-03-26 21:57:53 +01:00
" script " ,
]
2019-05-03 23:03:11 +02:00
EXTENDED_SCRIPTS = [
# These tests are not run by default.
# Longest test should go first, to favor running tests in parallel
' feature_pruning.py ' , # NOTE: Prune mode is incompatible with -txindex, should work with governance validation disabled though.
' feature_dbcrash.py ' ,
]
fix(tests): Fix retries for the non-deterministic test suite (#5307)
## Issue being fixed or feature implemented
the problem with retries implemented in #4793 is that they don't really
do anything besides fetching results of a failed job multiple times 🙈
## What was done?
partially reverted changes done in #4793, implemented actual job
restart. dropped `--sleep` and `--retries` and added `--attempts`
instead.
## How Has This Been Tested?
Pick any test, add some randomly failing expression somewhere and run it
with some high number of retries.
For example:
```diff
diff --git a/test/functional/feature_dip0020_activation.py b/test/functional/feature_dip0020_activation.py
index 471e4fdc66..b56a954b78 100755
--- a/test/functional/feature_dip0020_activation.py
+++ b/test/functional/feature_dip0020_activation.py
@@ -69,6 +69,7 @@ class DIP0020ActivationTest(BitcoinTestFramework):
# Should be spendable now
tx0id = self.node.sendrawtransaction(tx0_hex)
assert tx0id in set(self.node.getrawmempool())
+ assert int(tx0id[0], 16) < 4
if __name__ == '__main__':
```
On develop:
```
./test/functional/test_runner.py feature_dip0020_activation.py --retries=100 --sleep=0
```
if this fails on the first run, it keeps "failing" (simply fetching the
same results actually) till the end.
With this patch:
```
./test/functional/test_runner.py feature_dip0020_activation.py --attempts=100
```
if this fails on the first run, it can actually succeed after a few
attempts now, unless you are extremely unlucky ofc 😄
Also, check [ci results in my repo
](https://cdn.artifacts.gitlab-static.net/93/b4/93b4f8b17e5dcccab1afee165b4d74d90f05800caf65d6c48a83a1a78c979587/2023_04_08/4081291268/4478867166/job.log?response-content-type=text%2Fplain%3B%20charset%3Dutf-8&response-content-disposition=inline&Expires=1680945516&KeyName=gprd-artifacts-cdn&Signature=2d4mHCJBbgRaTDiSQ6kKIy1PdIM=).
Note:
```
...
feature_dip3_v19.py failed at attempt 1/3, Duration: 159s
...
4/179 - [1mfeature_dip3_v19.py[0m passed, Duration: 244 s
...
feature_llmq_hpmn.py failed at attempt 1/3, Duration: 284s
...
feature_llmq_hpmn.py failed at attempt 2/3, Duration: 296s
...
11/179 - [1mfeature_llmq_hpmn.py[0m failed, Duration: 233 s
...
```
An example with 2 tests failing initially and then passing:
https://gitlab.com/dashpay/dash/-/jobs/4089689970
## Breaking Changes
n/a
## Checklist:
- [x] I have performed a self-review of my own code
- [ ] I have commented my code, particularly in hard-to-understand areas
- [ ] I have added or updated relevant unit/integration/functional/e2e
tests
- [ ] I have made corresponding changes to the documentation
**For repository code-owners and collaborators only**
- [x] I have assigned this pull request to a milestone
2023-04-15 06:13:47 +02:00
BASE_SCRIPTS = [
2019-06-19 21:51:41 +02:00
# Scripts that are run by default.
2019-01-03 10:18:47 +01:00
# Longest test should go first, to favor running tests in parallel
2020-07-17 01:44:20 +02:00
' feature_dip3_deterministicmns.py ' , # NOTE: needs dash_hash to pass
2021-02-01 17:10:19 +01:00
' feature_llmq_data_recovery.py ' ,
2020-07-17 01:44:20 +02:00
' wallet_hd.py ' ,
' wallet_backup.py ' ,
2018-01-15 16:00:51 +01:00
# vv Tests less than 5m vv
2018-09-16 21:06:48 +02:00
' mining_getblocktemplate_longpoll.py ' , # FIXME: "socket.error: [Errno 54] Connection reset by peer" on my Mac, same as https://github.com/bitcoin/bitcoin/issues/6651
' feature_maxuploadtarget.py ' ,
2020-07-17 01:44:20 +02:00
' feature_block.py ' , # NOTE: needs dash_hash to pass
' rpc_fundrawtransaction.py ' ,
2023-05-24 19:38:33 +02:00
' rpc_fundrawtransaction.py --usehd ' ,
2020-07-28 22:51:09 +02:00
' wallet_multiwallet.py --usecli ' ,
2021-01-28 23:33:18 +01:00
' p2p_quorum_data.py ' ,
2018-01-15 16:00:51 +01:00
# vv Tests less than 2m vv
2020-07-17 01:44:20 +02:00
' p2p_instantsend.py ' ,
' wallet_basic.py ' ,
2017-10-20 19:27:55 +02:00
' wallet_labels.py ' ,
2018-09-16 21:06:48 +02:00
' p2p_timeouts.py ' ,
' feature_bip68_sequence.py ' ,
2020-04-29 18:07:05 +02:00
' mempool_updatefromblock.py ' ,
2019-08-05 14:01:21 +02:00
' p2p_tx_download.py ' ,
2020-07-17 01:44:20 +02:00
' wallet_dump.py ' ,
2018-04-24 14:27:55 +02:00
' wallet_listtransactions.py ' ,
2020-07-17 01:44:20 +02:00
' feature_multikeysporks.py ' ,
fix(tests): Fix retries for the non-deterministic test suite (#5307)
## Issue being fixed or feature implemented
the problem with retries implemented in #4793 is that they don't really
do anything besides fetching results of a failed job multiple times 🙈
## What was done?
partially reverted changes done in #4793, implemented actual job
restart. dropped `--sleep` and `--retries` and added `--attempts`
instead.
## How Has This Been Tested?
Pick any test, add some randomly failing expression somewhere and run it
with some high number of retries.
For example:
```diff
diff --git a/test/functional/feature_dip0020_activation.py b/test/functional/feature_dip0020_activation.py
index 471e4fdc66..b56a954b78 100755
--- a/test/functional/feature_dip0020_activation.py
+++ b/test/functional/feature_dip0020_activation.py
@@ -69,6 +69,7 @@ class DIP0020ActivationTest(BitcoinTestFramework):
# Should be spendable now
tx0id = self.node.sendrawtransaction(tx0_hex)
assert tx0id in set(self.node.getrawmempool())
+ assert int(tx0id[0], 16) < 4
if __name__ == '__main__':
```
On develop:
```
./test/functional/test_runner.py feature_dip0020_activation.py --retries=100 --sleep=0
```
if this fails on the first run, it keeps "failing" (simply fetching the
same results actually) till the end.
With this patch:
```
./test/functional/test_runner.py feature_dip0020_activation.py --attempts=100
```
if this fails on the first run, it can actually succeed after a few
attempts now, unless you are extremely unlucky ofc 😄
Also, check [ci results in my repo
](https://cdn.artifacts.gitlab-static.net/93/b4/93b4f8b17e5dcccab1afee165b4d74d90f05800caf65d6c48a83a1a78c979587/2023_04_08/4081291268/4478867166/job.log?response-content-type=text%2Fplain%3B%20charset%3Dutf-8&response-content-disposition=inline&Expires=1680945516&KeyName=gprd-artifacts-cdn&Signature=2d4mHCJBbgRaTDiSQ6kKIy1PdIM=).
Note:
```
...
feature_dip3_v19.py failed at attempt 1/3, Duration: 159s
...
4/179 - [1mfeature_dip3_v19.py[0m passed, Duration: 244 s
...
feature_llmq_hpmn.py failed at attempt 1/3, Duration: 284s
...
feature_llmq_hpmn.py failed at attempt 2/3, Duration: 296s
...
11/179 - [1mfeature_llmq_hpmn.py[0m failed, Duration: 233 s
...
```
An example with 2 tests failing initially and then passing:
https://gitlab.com/dashpay/dash/-/jobs/4089689970
## Breaking Changes
n/a
## Checklist:
- [x] I have performed a self-review of my own code
- [ ] I have commented my code, particularly in hard-to-understand areas
- [ ] I have added or updated relevant unit/integration/functional/e2e
tests
- [ ] I have made corresponding changes to the documentation
**For repository code-owners and collaborators only**
- [x] I have assigned this pull request to a milestone
2023-04-15 06:13:47 +02:00
' feature_dip3_v19.py ' ,
' feature_llmq_signing.py ' , # NOTE: needs dash_hash to pass
' feature_llmq_signing.py --spork21 ' , # NOTE: needs dash_hash to pass
' feature_llmq_chainlocks.py ' , # NOTE: needs dash_hash to pass
' feature_llmq_rotation.py ' , # NOTE: needs dash_hash to pass
' feature_llmq_connections.py ' , # NOTE: needs dash_hash to pass
2023-08-17 21:01:12 +02:00
' feature_llmq_evo.py ' , # NOTE: needs dash_hash to pass
fix(tests): Fix retries for the non-deterministic test suite (#5307)
## Issue being fixed or feature implemented
the problem with retries implemented in #4793 is that they don't really
do anything besides fetching results of a failed job multiple times 🙈
## What was done?
partially reverted changes done in #4793, implemented actual job
restart. dropped `--sleep` and `--retries` and added `--attempts`
instead.
## How Has This Been Tested?
Pick any test, add some randomly failing expression somewhere and run it
with some high number of retries.
For example:
```diff
diff --git a/test/functional/feature_dip0020_activation.py b/test/functional/feature_dip0020_activation.py
index 471e4fdc66..b56a954b78 100755
--- a/test/functional/feature_dip0020_activation.py
+++ b/test/functional/feature_dip0020_activation.py
@@ -69,6 +69,7 @@ class DIP0020ActivationTest(BitcoinTestFramework):
# Should be spendable now
tx0id = self.node.sendrawtransaction(tx0_hex)
assert tx0id in set(self.node.getrawmempool())
+ assert int(tx0id[0], 16) < 4
if __name__ == '__main__':
```
On develop:
```
./test/functional/test_runner.py feature_dip0020_activation.py --retries=100 --sleep=0
```
if this fails on the first run, it keeps "failing" (simply fetching the
same results actually) till the end.
With this patch:
```
./test/functional/test_runner.py feature_dip0020_activation.py --attempts=100
```
if this fails on the first run, it can actually succeed after a few
attempts now, unless you are extremely unlucky ofc 😄
Also, check [ci results in my repo
](https://cdn.artifacts.gitlab-static.net/93/b4/93b4f8b17e5dcccab1afee165b4d74d90f05800caf65d6c48a83a1a78c979587/2023_04_08/4081291268/4478867166/job.log?response-content-type=text%2Fplain%3B%20charset%3Dutf-8&response-content-disposition=inline&Expires=1680945516&KeyName=gprd-artifacts-cdn&Signature=2d4mHCJBbgRaTDiSQ6kKIy1PdIM=).
Note:
```
...
feature_dip3_v19.py failed at attempt 1/3, Duration: 159s
...
4/179 - [1mfeature_dip3_v19.py[0m passed, Duration: 244 s
...
feature_llmq_hpmn.py failed at attempt 1/3, Duration: 284s
...
feature_llmq_hpmn.py failed at attempt 2/3, Duration: 296s
...
11/179 - [1mfeature_llmq_hpmn.py[0m failed, Duration: 233 s
...
```
An example with 2 tests failing initially and then passing:
https://gitlab.com/dashpay/dash/-/jobs/4089689970
## Breaking Changes
n/a
## Checklist:
- [x] I have performed a self-review of my own code
- [ ] I have commented my code, particularly in hard-to-understand areas
- [ ] I have added or updated relevant unit/integration/functional/e2e
tests
- [ ] I have made corresponding changes to the documentation
**For repository code-owners and collaborators only**
- [x] I have assigned this pull request to a milestone
2023-04-15 06:13:47 +02:00
' feature_llmq_simplepose.py ' , # NOTE: needs dash_hash to pass
' feature_llmq_is_cl_conflicts.py ' , # NOTE: needs dash_hash to pass
' feature_llmq_is_retroactive.py ' , # NOTE: needs dash_hash to pass
' feature_llmq_dkgerrors.py ' , # NOTE: needs dash_hash to pass
' feature_dip4_coinbasemerkleroots.py ' , # NOTE: needs dash_hash to pass
2023-07-24 18:39:38 +02:00
' feature_asset_locks.py ' , # NOTE: needs dash_hash to pass
2023-09-23 20:45:42 +02:00
' feature_mnehf.py ' , # NOTE: needs dash_hash to pass
2018-01-15 16:00:51 +01:00
# vv Tests less than 60s vv
2020-07-17 01:44:20 +02:00
' p2p_sendheaders.py ' , # NOTE: needs dash_hash to pass
2022-03-11 20:39:12 +01:00
' p2p_sendheaders_compressed.py ' , # NOTE: needs dash_hash to pass
2020-07-17 01:44:20 +02:00
' wallet_importmulti.py ' ,
2018-01-15 16:00:51 +01:00
' mempool_limit.py ' ,
2020-07-17 01:44:20 +02:00
' rpc_txoutproof.py ' ,
fix(tests): Fix retries for the non-deterministic test suite (#5307)
## Issue being fixed or feature implemented
the problem with retries implemented in #4793 is that they don't really
do anything besides fetching results of a failed job multiple times 🙈
## What was done?
partially reverted changes done in #4793, implemented actual job
restart. dropped `--sleep` and `--retries` and added `--attempts`
instead.
## How Has This Been Tested?
Pick any test, add some randomly failing expression somewhere and run it
with some high number of retries.
For example:
```diff
diff --git a/test/functional/feature_dip0020_activation.py b/test/functional/feature_dip0020_activation.py
index 471e4fdc66..b56a954b78 100755
--- a/test/functional/feature_dip0020_activation.py
+++ b/test/functional/feature_dip0020_activation.py
@@ -69,6 +69,7 @@ class DIP0020ActivationTest(BitcoinTestFramework):
# Should be spendable now
tx0id = self.node.sendrawtransaction(tx0_hex)
assert tx0id in set(self.node.getrawmempool())
+ assert int(tx0id[0], 16) < 4
if __name__ == '__main__':
```
On develop:
```
./test/functional/test_runner.py feature_dip0020_activation.py --retries=100 --sleep=0
```
if this fails on the first run, it keeps "failing" (simply fetching the
same results actually) till the end.
With this patch:
```
./test/functional/test_runner.py feature_dip0020_activation.py --attempts=100
```
if this fails on the first run, it can actually succeed after a few
attempts now, unless you are extremely unlucky ofc 😄
Also, check [ci results in my repo
](https://cdn.artifacts.gitlab-static.net/93/b4/93b4f8b17e5dcccab1afee165b4d74d90f05800caf65d6c48a83a1a78c979587/2023_04_08/4081291268/4478867166/job.log?response-content-type=text%2Fplain%3B%20charset%3Dutf-8&response-content-disposition=inline&Expires=1680945516&KeyName=gprd-artifacts-cdn&Signature=2d4mHCJBbgRaTDiSQ6kKIy1PdIM=).
Note:
```
...
feature_dip3_v19.py failed at attempt 1/3, Duration: 159s
...
4/179 - [1mfeature_dip3_v19.py[0m passed, Duration: 244 s
...
feature_llmq_hpmn.py failed at attempt 1/3, Duration: 284s
...
feature_llmq_hpmn.py failed at attempt 2/3, Duration: 296s
...
11/179 - [1mfeature_llmq_hpmn.py[0m failed, Duration: 233 s
...
```
An example with 2 tests failing initially and then passing:
https://gitlab.com/dashpay/dash/-/jobs/4089689970
## Breaking Changes
n/a
## Checklist:
- [x] I have performed a self-review of my own code
- [ ] I have commented my code, particularly in hard-to-understand areas
- [ ] I have added or updated relevant unit/integration/functional/e2e
tests
- [ ] I have made corresponding changes to the documentation
**For repository code-owners and collaborators only**
- [x] I have assigned this pull request to a milestone
2023-04-15 06:13:47 +02:00
' wallet_listreceivedby.py ' ,
2020-07-17 01:44:20 +02:00
' wallet_abandonconflict.py ' ,
' feature_csv_activation.py ' ,
' rpc_rawtransaction.py ' ,
' feature_reindex.py ' ,
2019-07-25 02:30:18 +02:00
' feature_abortnode.py ' ,
2018-01-15 16:00:51 +01:00
# vv Tests less than 30s vv
2022-08-22 21:43:07 +02:00
' rpc_quorum.py ' ,
2020-07-17 01:44:20 +02:00
' wallet_keypool_topup.py ' ,
2019-08-06 14:33:51 +02:00
' feature_fee_estimation.py ' ,
fix(tests): Fix retries for the non-deterministic test suite (#5307)
## Issue being fixed or feature implemented
the problem with retries implemented in #4793 is that they don't really
do anything besides fetching results of a failed job multiple times 🙈
## What was done?
partially reverted changes done in #4793, implemented actual job
restart. dropped `--sleep` and `--retries` and added `--attempts`
instead.
## How Has This Been Tested?
Pick any test, add some randomly failing expression somewhere and run it
with some high number of retries.
For example:
```diff
diff --git a/test/functional/feature_dip0020_activation.py b/test/functional/feature_dip0020_activation.py
index 471e4fdc66..b56a954b78 100755
--- a/test/functional/feature_dip0020_activation.py
+++ b/test/functional/feature_dip0020_activation.py
@@ -69,6 +69,7 @@ class DIP0020ActivationTest(BitcoinTestFramework):
# Should be spendable now
tx0id = self.node.sendrawtransaction(tx0_hex)
assert tx0id in set(self.node.getrawmempool())
+ assert int(tx0id[0], 16) < 4
if __name__ == '__main__':
```
On develop:
```
./test/functional/test_runner.py feature_dip0020_activation.py --retries=100 --sleep=0
```
if this fails on the first run, it keeps "failing" (simply fetching the
same results actually) till the end.
With this patch:
```
./test/functional/test_runner.py feature_dip0020_activation.py --attempts=100
```
if this fails on the first run, it can actually succeed after a few
attempts now, unless you are extremely unlucky ofc 😄
Also, check [ci results in my repo
](https://cdn.artifacts.gitlab-static.net/93/b4/93b4f8b17e5dcccab1afee165b4d74d90f05800caf65d6c48a83a1a78c979587/2023_04_08/4081291268/4478867166/job.log?response-content-type=text%2Fplain%3B%20charset%3Dutf-8&response-content-disposition=inline&Expires=1680945516&KeyName=gprd-artifacts-cdn&Signature=2d4mHCJBbgRaTDiSQ6kKIy1PdIM=).
Note:
```
...
feature_dip3_v19.py failed at attempt 1/3, Duration: 159s
...
4/179 - [1mfeature_dip3_v19.py[0m passed, Duration: 244 s
...
feature_llmq_hpmn.py failed at attempt 1/3, Duration: 284s
...
feature_llmq_hpmn.py failed at attempt 2/3, Duration: 296s
...
11/179 - [1mfeature_llmq_hpmn.py[0m failed, Duration: 233 s
...
```
An example with 2 tests failing initially and then passing:
https://gitlab.com/dashpay/dash/-/jobs/4089689970
## Breaking Changes
n/a
## Checklist:
- [x] I have performed a self-review of my own code
- [ ] I have commented my code, particularly in hard-to-understand areas
- [ ] I have added or updated relevant unit/integration/functional/e2e
tests
- [ ] I have made corresponding changes to the documentation
**For repository code-owners and collaborators only**
- [x] I have assigned this pull request to a milestone
2023-04-15 06:13:47 +02:00
' interface_zmq_dash.py ' ,
2020-07-17 01:44:20 +02:00
' interface_zmq.py ' ,
2021-01-26 00:38:38 +01:00
' rpc_invalid_address_message.py ' ,
2020-07-17 01:44:20 +02:00
' interface_bitcoin_cli.py ' ,
' mempool_resurrect.py ' ,
' wallet_txn_doublespend.py --mineblock ' ,
2019-01-31 17:07:45 +01:00
' tool_wallet.py ' ,
2020-07-17 01:44:20 +02:00
' wallet_txn_clone.py ' ,
' rpc_getchaintips.py ' ,
2019-03-01 19:54:15 +01:00
' rpc_misc.py ' ,
2020-07-17 01:44:20 +02:00
' interface_rest.py ' ,
' mempool_spend_coinbase.py ' ,
2019-06-19 01:32:02 +02:00
' wallet_avoidreuse.py ' ,
2015-08-27 03:15:04 +02:00
' mempool_reorg.py ' ,
2017-05-03 09:23:39 +02:00
' mempool_persist.py ' ,
2020-07-17 01:44:20 +02:00
' wallet_multiwallet.py ' ,
2019-02-10 20:07:17 +01:00
' wallet_createwallet.py ' ,
' wallet_createwallet.py --usecli ' ,
2022-04-03 14:21:17 +02:00
' wallet_reorgsrestore.py ' ,
2019-08-16 04:55:26 +02:00
' wallet_watchonly.py ' ,
' wallet_watchonly.py --usecli ' ,
2020-07-17 01:44:20 +02:00
' interface_http.py ' ,
2018-11-22 18:14:44 +01:00
' interface_rpc.py ' ,
2021-07-13 18:30:17 +02:00
' rpc_psbt.py ' ,
2020-07-17 01:44:20 +02:00
' rpc_users.py ' ,
Merge #12763: Add RPC Whitelist Feature from #12248
2081442c421cc4376e5d7839f68fbe7630e89103 test: Add test for rpc_whitelist (Emil Engler)
7414d3820c833566b4f48c6c120a18bf53978c55 Add RPC Whitelist Feature from #12248 (Jeremy Rubin)
Pull request description:
Summary
====
This patch adds the RPC whitelisting feature requested in #12248. RPC Whitelists help enforce application policies for services being built on top of Bitcoin Core (e.g., your Lightning Node maybe shouldn't be adding new peers). The aim of this PR is not to make it advisable to connect your Bitcoin node to arbitrary services, but to reduce risk and prevent unintended access.
Using RPC Whitelists
====
The way it works is you specify (in your bitcoin.conf) configurations such as
```
rpcauth=user1:4cc74397d6e9972e5ee7671fd241$11849357f26a5be7809c68a032bc2b16ab5dcf6348ef3ed1cf30dae47b8bcc71
rpcauth=user2:181b4a25317bff60f3749adee7d6bca0$d9c331474f1322975fa170a2ffbcb176ba11644211746b27c1d317f265dd4ada
rpcauth=user3:a6c8a511b53b1edcf69c36984985e$13cfba0e626db19061c9d61fa58e712d0319c11db97ad845fa84517f454f6675
rpcwhitelist=user1:getnetworkinfo
rpcwhitelist=user2:getnetworkinfo,getwalletinfo, getbestblockhash
rpcwhitelistdefault=0
```
Now user1 can only call getnetworkinfo, user2 can only call getnetworkinfo or getwalletinfo, while user3 can still call all RPCs.
If any rpcwhitelist is set, act as if all users are subject to whitelists unless rpcwhitelistdefault is set to 0. If rpcwhitelistdefault is set to 1 and no rpcwhitelist is set, act as if all users are subject to whitelists.
Review Request
=====
In addition to normal review, would love specific review from someone working on LN (e.g., @ roasbeef) and someone working on an infrastructure team at an exchange (e.g., @ jimpo) to check that this works well with their system.
Notes
=====
The rpc list is spelling sensitive -- whitespace is stripped though. Spelling errors fail towards the RPC call being blocked, which is safer.
It was unclear to me if HTTPReq_JSONRPC is the best function to patch this functionality into, or if it would be better to place it in exec or somewhere else.
It was also unclear to me if it would be preferred to cache the whitelists on startup or parse them on every RPC as is done with multiUserAuthorized. I opted for the cached approach as I thought it was a bit cleaner.
Future Work
=====
In a future PR, I would like to add an inheritance scheme. This seemed more controversial so I didn't want to include that here. Inheritance semantics are tricky, but it would also make these whitelists easier to read.
It also might be good to add a `getrpcwhitelist` command to facilitate permission discovery.
Tests
=====
Thanks to @ emilengler for adding tests for this feature. The tests cover all cases except for where `rpcwhitelistdefault=1` is used, given difficulties around testing with the current test framework.
ACKs for top commit:
laanwj:
ACK 2081442c421cc4376e5d7839f68fbe7630e89103
Tree-SHA512: 0dc1ac6a6f2f4b0be9c9054d495dd17752fe7b3589aeab2c6ac4e1f91cf4e7e355deedcb5d76d707cbb5a949c2f989c871b74d6bf129351f429569a701adbcbf
2019-12-13 11:25:39 +01:00
' rpc_whitelist.py ' ,
2020-07-17 01:44:20 +02:00
' feature_proxy.py ' ,
' rpc_signrawtransaction.py ' ,
fix(tests): Fix retries for the non-deterministic test suite (#5307)
## Issue being fixed or feature implemented
the problem with retries implemented in #4793 is that they don't really
do anything besides fetching results of a failed job multiple times 🙈
## What was done?
partially reverted changes done in #4793, implemented actual job
restart. dropped `--sleep` and `--retries` and added `--attempts`
instead.
## How Has This Been Tested?
Pick any test, add some randomly failing expression somewhere and run it
with some high number of retries.
For example:
```diff
diff --git a/test/functional/feature_dip0020_activation.py b/test/functional/feature_dip0020_activation.py
index 471e4fdc66..b56a954b78 100755
--- a/test/functional/feature_dip0020_activation.py
+++ b/test/functional/feature_dip0020_activation.py
@@ -69,6 +69,7 @@ class DIP0020ActivationTest(BitcoinTestFramework):
# Should be spendable now
tx0id = self.node.sendrawtransaction(tx0_hex)
assert tx0id in set(self.node.getrawmempool())
+ assert int(tx0id[0], 16) < 4
if __name__ == '__main__':
```
On develop:
```
./test/functional/test_runner.py feature_dip0020_activation.py --retries=100 --sleep=0
```
if this fails on the first run, it keeps "failing" (simply fetching the
same results actually) till the end.
With this patch:
```
./test/functional/test_runner.py feature_dip0020_activation.py --attempts=100
```
if this fails on the first run, it can actually succeed after a few
attempts now, unless you are extremely unlucky ofc 😄
Also, check [ci results in my repo
](https://cdn.artifacts.gitlab-static.net/93/b4/93b4f8b17e5dcccab1afee165b4d74d90f05800caf65d6c48a83a1a78c979587/2023_04_08/4081291268/4478867166/job.log?response-content-type=text%2Fplain%3B%20charset%3Dutf-8&response-content-disposition=inline&Expires=1680945516&KeyName=gprd-artifacts-cdn&Signature=2d4mHCJBbgRaTDiSQ6kKIy1PdIM=).
Note:
```
...
feature_dip3_v19.py failed at attempt 1/3, Duration: 159s
...
4/179 - [1mfeature_dip3_v19.py[0m passed, Duration: 244 s
...
feature_llmq_hpmn.py failed at attempt 1/3, Duration: 284s
...
feature_llmq_hpmn.py failed at attempt 2/3, Duration: 296s
...
11/179 - [1mfeature_llmq_hpmn.py[0m failed, Duration: 233 s
...
```
An example with 2 tests failing initially and then passing:
https://gitlab.com/dashpay/dash/-/jobs/4089689970
## Breaking Changes
n/a
## Checklist:
- [x] I have performed a self-review of my own code
- [ ] I have commented my code, particularly in hard-to-understand areas
- [ ] I have added or updated relevant unit/integration/functional/e2e
tests
- [ ] I have made corresponding changes to the documentation
**For repository code-owners and collaborators only**
- [x] I have assigned this pull request to a milestone
2023-04-15 06:13:47 +02:00
' p2p_addrv2_relay.py ' ,
Merge #12257: [wallet] Use destination groups instead of coins in coin select
232f96f5c8a3920c09db92f4dbac2ad7d10ce8cf doc: Add release notes for -avoidpartialspends (Karl-Johan Alm)
e00b4699cc6d2ee5697d38dd6607eb2631c9b77a clean-up: Remove no longer used ivars from CInputCoin (Karl-Johan Alm)
43e04d13b1ffc02b1082176e87f420198b40c7b1 wallet: Remove deprecated OutputEligibleForSpending (Karl-Johan Alm)
0128121101fb3ee82f3abd3973a967a4226ffe0e test: Add basic testing for wallet groups (Karl-Johan Alm)
59d6f7b4e2f847ec1f2ff46c84e6157655984f85 wallet: Switch to using output groups instead of coins in coin selection (Karl-Johan Alm)
87ebce25d66952f5ce565bb5130dcf5e24049872 wallet: Add output grouping (Karl-Johan Alm)
bb629cb9dc567cc819724d9f4852652926e60cbf Add -avoidpartialspends and m_avoid_partial_spends (Karl-Johan Alm)
65b3eda458221644616d0fdd6ba0fe01bdbce893 wallet: Add input bytes to CInputCoin (Karl-Johan Alm)
a443d7a0ca333b0bae63e04b5d476f9ad9c7aeac moveonly: CoinElegibilityFilter into coinselection.h (Karl-Johan Alm)
173e18a289088c6087ba6fac708e322aa63b7a94 utils: Add insert() convenience templates (Karl-Johan Alm)
Pull request description:
This PR adds an optional (off by default) `-avoidpartialspends` flag, which changes coin select to use output groups rather than outputs, where each output group corresponds to all outputs with the same destination.
It is a privacy improvement, as each time you spend some output, any other output that is publicly associated with the destination (address) will also be spent at the same time, at the cost of fee increase for cases where coin select without group restriction would find a more optimal set of coins (see example below).
For regular use without address reuse, this PR should have no effect on the user experience whatsoever; it only affects users who, for some reason, have multiple outputs with the same destination (i.e. address reuse).
Nodes with this turned off will still try to avoid partial spending, if the fee of the resulting transaction is not greater than the fee of the original transaction.
Example: a node has four outputs linked to two addresses `A` and `B`:
* 1.0 btc to `A`
* 0.5 btc to `A`
* 1.0 btc to `B`
* 0.5 btc to `B`
The node sends 0.2 btc to `C`. Without `-avoidpartialspends`, the following coin selection will occur:
* 0.5 btc to `A` or `B` is picked
* 0.2 btc is output to `C`
* 0.3 - fee is output to (unique change address)
With `-avoidpartialspends`, the following will instead happen:
* Both of (0.5, 1.0) btc to `A` or `B` is picked (one or the other pair)
* 0.2 btc is output to `C`
* 1.3 - fee is output to (unique change address)
As noted, the pro here is that, assuming nobody sends to the address after you spend from it, you will only ever use one address once. The con is that the transaction becomes slightly larger in this case, because it is overpicking outputs to adhere to the no partial spending rule.
This complements #10386, in particular it addresses @luke-jr and @gmaxwell's concerns in https://github.com/bitcoin/bitcoin/pull/10386#issuecomment-300667926 and https://github.com/bitcoin/bitcoin/pull/10386#issuecomment-302361381.
Together with `-avoidreuse`, this fully addresses the concerns in #10065 I believe.
Tree-SHA512: 24687a4490ba59cf4198ed90052944ff4996653a4257833bb52ed24d058b3e924800c9b3790aeb6be6385b653b49e304453e5d7ff960e64c682fc23bfc447621
# Conflicts:
# src/Makefile.am
# src/bench/coin_selection.cpp
# src/wallet/coincontrol.h
# src/wallet/coinselection.cpp
# src/wallet/coinselection.h
# src/wallet/init.cpp
# src/wallet/test/coinselector_tests.cpp
# src/wallet/wallet.cpp
# src/wallet/wallet.h
# test/functional/test_runner.py
2018-07-24 15:06:21 +02:00
' wallet_groups.py ' ,
2020-07-17 01:44:20 +02:00
' p2p_disconnect_ban.py ' ,
' feature_addressindex.py ' ,
' feature_timestampindex.py ' ,
' feature_spentindex.py ' ,
' rpc_decodescript.py ' ,
' rpc_blockchain.py ' ,
' rpc_deprecated.py ' ,
' wallet_disable.py ' ,
2020-04-10 16:10:23 +02:00
' p2p_addr_relay.py ' ,
2023-06-02 11:02:59 +02:00
' p2p_getaddr_caching.py ' ,
2020-05-12 02:34:46 +02:00
' p2p_getdata.py ' ,
fix(tests): Fix retries for the non-deterministic test suite (#5307)
## Issue being fixed or feature implemented
the problem with retries implemented in #4793 is that they don't really
do anything besides fetching results of a failed job multiple times 🙈
## What was done?
partially reverted changes done in #4793, implemented actual job
restart. dropped `--sleep` and `--retries` and added `--attempts`
instead.
## How Has This Been Tested?
Pick any test, add some randomly failing expression somewhere and run it
with some high number of retries.
For example:
```diff
diff --git a/test/functional/feature_dip0020_activation.py b/test/functional/feature_dip0020_activation.py
index 471e4fdc66..b56a954b78 100755
--- a/test/functional/feature_dip0020_activation.py
+++ b/test/functional/feature_dip0020_activation.py
@@ -69,6 +69,7 @@ class DIP0020ActivationTest(BitcoinTestFramework):
# Should be spendable now
tx0id = self.node.sendrawtransaction(tx0_hex)
assert tx0id in set(self.node.getrawmempool())
+ assert int(tx0id[0], 16) < 4
if __name__ == '__main__':
```
On develop:
```
./test/functional/test_runner.py feature_dip0020_activation.py --retries=100 --sleep=0
```
if this fails on the first run, it keeps "failing" (simply fetching the
same results actually) till the end.
With this patch:
```
./test/functional/test_runner.py feature_dip0020_activation.py --attempts=100
```
if this fails on the first run, it can actually succeed after a few
attempts now, unless you are extremely unlucky ofc 😄
Also, check [ci results in my repo
](https://cdn.artifacts.gitlab-static.net/93/b4/93b4f8b17e5dcccab1afee165b4d74d90f05800caf65d6c48a83a1a78c979587/2023_04_08/4081291268/4478867166/job.log?response-content-type=text%2Fplain%3B%20charset%3Dutf-8&response-content-disposition=inline&Expires=1680945516&KeyName=gprd-artifacts-cdn&Signature=2d4mHCJBbgRaTDiSQ6kKIy1PdIM=).
Note:
```
...
feature_dip3_v19.py failed at attempt 1/3, Duration: 159s
...
4/179 - [1mfeature_dip3_v19.py[0m passed, Duration: 244 s
...
feature_llmq_hpmn.py failed at attempt 1/3, Duration: 284s
...
feature_llmq_hpmn.py failed at attempt 2/3, Duration: 296s
...
11/179 - [1mfeature_llmq_hpmn.py[0m failed, Duration: 233 s
...
```
An example with 2 tests failing initially and then passing:
https://gitlab.com/dashpay/dash/-/jobs/4089689970
## Breaking Changes
n/a
## Checklist:
- [x] I have performed a self-review of my own code
- [ ] I have commented my code, particularly in hard-to-understand areas
- [ ] I have added or updated relevant unit/integration/functional/e2e
tests
- [ ] I have made corresponding changes to the documentation
**For repository code-owners and collaborators only**
- [x] I have assigned this pull request to a milestone
2023-04-15 06:13:47 +02:00
' rpc_net.py ' ,
2020-07-17 01:44:20 +02:00
' wallet_keypool.py ' ,
' wallet_keypool_hd.py ' ,
2020-06-11 20:34:39 +02:00
' p2p_nobloomfilter_messages.py ' ,
2020-03-30 21:27:54 +02:00
' p2p_filter.py ' ,
2019-05-16 18:44:54 +02:00
' p2p_blocksonly.py ' ,
2019-08-16 16:17:20 +02:00
' rpc_setban.py ' ,
2020-07-17 01:44:20 +02:00
' mining_prioritisetransaction.py ' ,
2018-08-11 12:59:08 +02:00
' p2p_invalid_locator.py ' ,
2020-07-17 01:44:20 +02:00
' p2p_invalid_block.py ' ,
2018-11-06 11:08:40 +01:00
' p2p_invalid_messages.py ' ,
2020-07-17 01:44:20 +02:00
' p2p_invalid_tx.py ' ,
2018-09-16 21:06:48 +02:00
' feature_assumevalid.py ' ,
' example_test.py ' ,
' wallet_txn_doublespend.py ' ,
2020-02-12 15:19:50 +01:00
' feature_backwards_compatibility.py ' ,
2018-09-16 21:06:48 +02:00
' wallet_txn_clone.py --mineblock ' ,
' feature_notifications.py ' ,
2021-08-12 09:04:28 +02:00
' rpc_getblockfilter.py ' ,
2018-09-16 21:06:48 +02:00
' rpc_invalidateblock.py ' ,
' feature_txindex.py ' ,
2022-02-26 11:54:28 +01:00
' feature_utxo_set_hash.py ' ,
2018-09-16 21:06:48 +02:00
' mempool_packages.py ' ,
Merge #15681: [mempool] Allow one extra single-ancestor transaction per package
50cede3f5a4d4fbfbb7c420b94e661a6a159bced [mempool] Allow one extra single-ancestor transaction per package (Matt Corallo)
Pull request description:
This implements the proposed policy change from [1], which allows
certain classes of contract protocols involving revocation
punishments to use CPFP. Note that some such use-cases may still
want some form of one-deep package relay, though even this alone
may greatly simplify some lightning fee negotiation.
[1] https://lists.linuxfoundation.org/pipermail/bitcoin-dev/2018-November/016518.html
ACKs for top commit:
ajtowns:
ACK 50cede3f5a4d4fbfbb7c420b94e661a6a159bced -- looked over code again, compared with previous commit, compiles, etc.
sdaftuar:
ACK 50cede3f5a4d4fbfbb7c420b94e661a6a159bced
ryanofsky:
utACK 50cede3f5a4d4fbfbb7c420b94e661a6a159bced. Changes since last review: adding EXTRA_DESCENDANT_TX_SIZE_LIMIT constant, changing max ancestor size from 1,000,000 to nLimitAncestorSize constant (101,000), fixing test comment and getting rid of unused test node.
Tree-SHA512: b052c2a0f384855572b4579310131897b612201214b5abbb225167224e4f550049e300b471dbf320928652571e92ca2d650050b7cf39ac92b3bc1d2bcd386c1c
2019-07-19 19:56:12 +02:00
' mempool_package_onemore.py ' ,
2018-07-14 05:26:42 +02:00
' rpc_createmultisig.py ' ,
2024-01-29 16:01:14 +01:00
' rpc_packages.py ' ,
2020-07-17 01:44:20 +02:00
' feature_versionbits_warning.py ' ,
' rpc_preciousblock.py ' ,
' wallet_importprunedfunds.py ' ,
2018-10-27 13:17:59 +02:00
' p2p_leak_tx.py ' ,
2020-06-12 20:48:14 +02:00
' p2p_eviction.py ' ,
2020-07-17 01:44:20 +02:00
' rpc_signmessage.py ' ,
2022-03-25 01:30:06 +01:00
' rpc_generateblock.py ' ,
2021-12-12 14:38:12 +01:00
' wallet_balance.py ' ,
2020-07-17 01:44:20 +02:00
' feature_nulldummy.py ' ,
2021-04-08 21:35:16 +02:00
' mempool_accept.py ' ,
2020-02-21 19:23:08 +01:00
' mempool_expiry.py ' ,
2020-07-17 01:44:20 +02:00
' wallet_import_rescan.py ' ,
2018-11-13 08:51:34 +01:00
' wallet_import_with_label.py ' ,
2020-04-29 17:08:32 +02:00
' wallet_upgradewallet.py ' ,
2023-06-28 18:01:24 +02:00
' wallet_mnemonicbits.py ' ,
2020-07-17 01:44:20 +02:00
' rpc_bind.py --ipv4 ' ,
' rpc_bind.py --ipv6 ' ,
' rpc_bind.py --nonloopback ' ,
' mining_basic.py ' ,
' rpc_named_arguments.py ' ,
' wallet_listsinceblock.py ' ,
' p2p_leak.py ' ,
' p2p_compactblocks.py ' ,
2021-01-22 15:58:07 +01:00
' p2p_connect_to_devnet.py ' ,
2020-07-17 01:44:20 +02:00
' feature_sporks.py ' ,
Merge #10757: RPC: Introduce getblockstats to plot things (#3058)
* Merge #10757: RPC: Introduce getblockstats to plot things
41d0476f62269027ec2193a5f80d508d789de8aa Tests: Add data file (Anthony Towns)
4cbfb6aad9ba8fa17b5e7ed3e9a36dc8a24f1fcf Tests: Test new getblockstats RPC (Jorge Timón)
35e77a0288bcac5594ff25c10c9679a161cb730b RPC: Introduce getblockstats (Jorge Timón)
cda8e36f019dd181e5c3774961b4f1335e5602cb Refactor: RPC: Separate GetBlockChecked() from getblock() (Jorge Timón)
Pull request description:
It returns per block statistics about several things. It should be easy to add more if people think of other things to add or remove some if I went too far (but once written, why not keep it? EDIT: answer: not to test or maintain them).
The currently available options are: minfee,maxfee,totalfee,minfeerate,maxfeerate,avgfee,avgfeerate,txs,ins,outs (EDIT: see updated list in the rpc call documentation)
For the x axis, one can use height or block.nTime (I guess I could add mediantime if there's interest [EDIT: nobody showed interest but I implemented mediantime nonetheless, in fact there's no distinction between x or y axis anymore, that's for the caller to judge]).
To calculate fees, -txindex is required.
Tree-SHA512: 2b2787a3c7dc4a11df1fce62c8a4c748f5347d7f7104205d5f0962ffec1e0370c825b49fd4d58ce8ce86bf39d8453f698bcd46206eea505f077541ca7d59b18c
* Replace get_mocktime() usage with self.mocktime
2019-08-28 13:50:29 +02:00
' rpc_getblockstats.py ' ,
2020-07-17 01:44:20 +02:00
' wallet_encryption.py ' ,
2021-01-25 04:03:33 +01:00
' wallet_upgradetohd.py ' ,
2020-07-17 01:44:20 +02:00
' feature_dersig.py ' ,
' feature_cltv.py ' ,
2020-12-10 00:08:05 +01:00
' feature_new_quorum_type_activation.py ' ,
2020-12-01 04:55:29 +01:00
' feature_governance_objects.py ' ,
2023-08-29 17:31:59 +02:00
' feature_governance.py ' ,
2020-07-17 01:44:20 +02:00
' rpc_uptime.py ' ,
' wallet_resendwallettransactions.py ' ,
2018-03-01 18:27:08 +01:00
' wallet_fallbackfee.py ' ,
2022-05-21 12:08:48 +02:00
' rpc_dumptxoutset.py ' ,
2020-07-17 01:44:20 +02:00
' feature_minchainwork.py ' ,
2020-03-11 20:54:42 +01:00
' rpc_estimatefee.py ' ,
2020-07-17 01:44:20 +02:00
' p2p_unrequested_blocks.py ' , # NOTE: needs dash_hash to pass
Merge #14670: http: Fix HTTP server shutdown
28479f926f21f2a91bec5a06671c60e5b0c55532 qa: Test bitcond shutdown (João Barbosa)
8d3f46ec3938e2ba17654fecacd1d2629f9915fd http: Remove timeout to exit event loop (João Barbosa)
e98a9eede2fb48ff33a020acc888cbcd83e24bbf http: Remove unnecessary event_base_loopexit call (João Barbosa)
6b13580f4e3842c11abd9b8bee7255fb2472b6fe http: Unlisten sockets after all workers quit (João Barbosa)
18e968581697078c36a3c3818f8906cf134ccadd http: Send "Connection: close" header if shutdown is requested (João Barbosa)
02e1e4eff6cda0bfc24b455a7c1583394cbff6eb rpc: Add wait argument to stop (João Barbosa)
Pull request description:
Fixes #11777. Reverts #11006. Replaces #13501.
With this change the HTTP server will exit gracefully, meaning that all requests will finish processing and sending the response, even if this means to wait more than 2 seconds (current time allowed to exit the event loop).
Another small change is that connections are accepted even when the server is stopping, but HTTP requests are rejected. This can be improved later, especially if chunked replies are implemented.
Briefly, before this PR, this is the order or events when a request arrives (RPC `stop`):
1. `bufferevent_disable(..., EV_READ)`
2. `StartShutdown()`
3. `evhttp_del_accept_socket(...)`
4. `ThreadHTTP` terminates (event loop exits) because there are no active or pending events thanks to 1. and 3.
5. client doesn't get the response thanks to 4.
This can be verified by applying
```diff
// Event loop will exit after current HTTP requests have been handled, so
// this reply will get back to the client.
StartShutdown();
+ MilliSleep(2000);
return "Bitcoin server stopping";
}
```
and checking the log output:
```
Received a POST request for / from 127.0.0.1:62443
ThreadRPCServer method=stop user=__cookie__
Interrupting HTTP server
** Exited http event loop
Interrupting HTTP RPC server
Interrupting RPC
tor: Thread interrupt
Shutdown: In progress...
torcontrol thread exit
Stopping HTTP RPC server
addcon thread exit
opencon thread exit
Unregistering HTTP handler for / (exactmatch 1)
Unregistering HTTP handler for /wallet/ (exactmatch 0)
Stopping RPC
RPC stopped.
Stopping HTTP server
Waiting for HTTP worker threads to exit
msghand thread exit
net thread exit
... sleep 2 seconds ...
Waiting for HTTP event thread to exit
Stopped HTTP server
```
For this reason point 3. is moved right after all HTTP workers quit. In that moment HTTP replies are queued in the event loop which keeps spinning util all connections are closed. In order to trigger the server side close with keep alive connections (implicit in HTTP/1.1) the header `Connection: close` is sent if shutdown was requested. This can be tested by
```
bitcoind -regtest
nc localhost 18443
POST / HTTP/1.1
Authorization: Basic ...
Content-Type: application/json
Content-Length: 44
{"jsonrpc": "2.0","method":"stop","id":123}
```
Summing up, this PR:
- removes explicit event loop exit — event loop exits once there are no active or pending events
- changes the moment the listening sockets are removed — explained above
- sends header `Connection: close` on active requests when shutdown was requested which is relevant when it's a persistent connection (default in HTTP 1.1) — libevent is aware of this header and closes the connection gracefully
- removes event loop explicit break after 2 seconds timeout
Tree-SHA512: 4dac1e86abe388697c1e2dedbf31fb36a394cfafe5e64eadbf6ed01d829542785a8c3b91d1ab680d3f03f912d14fc87176428041141441d25dcb6c98a1e069d8
2018-12-06 17:42:52 +01:00
' feature_shutdown.py ' ,
2021-03-17 23:36:11 +01:00
' rpc_coinjoin.py ' ,
2020-12-15 03:15:09 +01:00
' rpc_masternode.py ' ,
2021-01-25 03:50:16 +01:00
' rpc_mnauth.py ' ,
2021-01-11 18:13:50 +01:00
' rpc_verifyislock.py ' ,
2021-04-26 13:51:15 +02:00
' rpc_verifychainlock.py ' ,
2021-11-01 16:30:13 +01:00
' wallet_create_tx.py ' ,
2020-07-17 01:44:20 +02:00
' p2p_fingerprint.py ' ,
2020-11-24 02:39:50 +01:00
' rpc_platform_filter.py ' ,
2023-06-27 20:51:40 +02:00
' rpc_wipewallettxes.py ' ,
fix(tests): Fix retries for the non-deterministic test suite (#5307)
## Issue being fixed or feature implemented
the problem with retries implemented in #4793 is that they don't really
do anything besides fetching results of a failed job multiple times 🙈
## What was done?
partially reverted changes done in #4793, implemented actual job
restart. dropped `--sleep` and `--retries` and added `--attempts`
instead.
## How Has This Been Tested?
Pick any test, add some randomly failing expression somewhere and run it
with some high number of retries.
For example:
```diff
diff --git a/test/functional/feature_dip0020_activation.py b/test/functional/feature_dip0020_activation.py
index 471e4fdc66..b56a954b78 100755
--- a/test/functional/feature_dip0020_activation.py
+++ b/test/functional/feature_dip0020_activation.py
@@ -69,6 +69,7 @@ class DIP0020ActivationTest(BitcoinTestFramework):
# Should be spendable now
tx0id = self.node.sendrawtransaction(tx0_hex)
assert tx0id in set(self.node.getrawmempool())
+ assert int(tx0id[0], 16) < 4
if __name__ == '__main__':
```
On develop:
```
./test/functional/test_runner.py feature_dip0020_activation.py --retries=100 --sleep=0
```
if this fails on the first run, it keeps "failing" (simply fetching the
same results actually) till the end.
With this patch:
```
./test/functional/test_runner.py feature_dip0020_activation.py --attempts=100
```
if this fails on the first run, it can actually succeed after a few
attempts now, unless you are extremely unlucky ofc 😄
Also, check [ci results in my repo
](https://cdn.artifacts.gitlab-static.net/93/b4/93b4f8b17e5dcccab1afee165b4d74d90f05800caf65d6c48a83a1a78c979587/2023_04_08/4081291268/4478867166/job.log?response-content-type=text%2Fplain%3B%20charset%3Dutf-8&response-content-disposition=inline&Expires=1680945516&KeyName=gprd-artifacts-cdn&Signature=2d4mHCJBbgRaTDiSQ6kKIy1PdIM=).
Note:
```
...
feature_dip3_v19.py failed at attempt 1/3, Duration: 159s
...
4/179 - [1mfeature_dip3_v19.py[0m passed, Duration: 244 s
...
feature_llmq_hpmn.py failed at attempt 1/3, Duration: 284s
...
feature_llmq_hpmn.py failed at attempt 2/3, Duration: 296s
...
11/179 - [1mfeature_llmq_hpmn.py[0m failed, Duration: 233 s
...
```
An example with 2 tests failing initially and then passing:
https://gitlab.com/dashpay/dash/-/jobs/4089689970
## Breaking Changes
n/a
## Checklist:
- [x] I have performed a self-review of my own code
- [ ] I have commented my code, particularly in hard-to-understand areas
- [ ] I have added or updated relevant unit/integration/functional/e2e
tests
- [ ] I have made corresponding changes to the documentation
**For repository code-owners and collaborators only**
- [x] I have assigned this pull request to a milestone
2023-04-15 06:13:47 +02:00
' feature_dip0020_activation.py ' ,
2020-07-17 01:44:20 +02:00
' feature_uacomment.py ' ,
2018-12-20 17:56:46 +01:00
' wallet_coinbase_category.py ' ,
2018-10-20 03:59:28 +02:00
' feature_filelock.py ' ,
2019-10-23 10:57:30 +02:00
' feature_loadblock.py ' ,
2021-09-19 06:31:43 +02:00
' p2p_blockfilters.py ' ,
Merge #19509: Per-Peer Message Capture
bff7c66e67aa2f18ef70139338643656a54444fe Add documentation to contrib folder (Troy Giorshev)
381f77be858d7417209b6de0b7cd23cb7eb99261 Add Message Capture Test (Troy Giorshev)
e4f378a505922c0f544b4cfbfdb169e884e02be9 Add capture parser (Troy Giorshev)
4d1a582549bc982d55e24585b0ba06f92f21e9da Call CaptureMessage at appropriate locations (Troy Giorshev)
f2a77ff97bec09dd5fcc043d8659d8ec5dfb87c2 Add CaptureMessage (Troy Giorshev)
dbf779d5deb04f55c6e8493ce4e12ed4628638f3 Clean PushMessage and ProcessMessages (Troy Giorshev)
Pull request description:
This PR introduces per-peer message capture into Bitcoin Core. 📓
## Purpose
The purpose and scope of this feature is intentionally limited. It answers a question anyone new to Bitcoin's P2P protocol has had: "Can I see what messages my node is sending and receiving?".
## Functionality
When a new debug-only command line argument `capturemessages` is set, any message that the node receives or sends is captured. The capture occurs in the MessageHandler thread. When receiving a message, it is captured as soon as the MessageHandler thread takes the message off of the vProcessMsg queue. When sending, the message is captured just before the message is pushed onto the vSendMsg queue.
The message capture is as minimal as possible to reduce the performance impact on the node. Messages are captured to a new `message_capture` folder in the datadir. Each node has their own subfolder named with their IP address and port. Inside, received and sent messages are captured into two binary files, msgs_recv.dat and msgs_sent.dat, like so:
```
message_capture/203.0.113.7:56072/msgs_recv.dat
message_capture/203.0.113.7:56072/msgs_sent.dat
```
Because the messages are raw binary dumps, included in this PR is a Python parsing tool to convert the binary files into human-readable JSON. This script has been placed on its own and out of the way in the new `contrib/message-capture` folder. Its usage is simple and easily discovered by the autogenerated `-h` option.
## Future Maintenance
I sympathize greatly with anyone who says "the best code is no code".
The future maintenance of this feature will be minimal. The logic to deserialize the payload of the p2p messages exists in our testing framework. As long as our testing framework works, so will this tool.
Additionally, I hope that the simplicity of this tool will mean that it gets used frequently, so that problems will be discovered and solved when they are small.
## FAQ
"Why not just use Wireshark"
Yes, Wireshark has the ability to filter and decode Bitcoin messages. However, the purpose of the message capture added in this PR is to assist with debugging, primarily for new developers looking to improve their knowledge of the Bitcoin Protocol. This drives the design in a different direction than Wireshark, in two different ways. First, this tool must be convenient and simple to use. Using an external tool, like Wireshark, requires setup and interpretation of the results. To a new user who doesn't necessarily know what to expect, this is unnecessary difficulty. This tool, on the other hand, "just works". Turn on the command line flag, run your node, run the script, read the JSON. Second, because this tool is being used for debugging, we want it to be as close to the true behavior of the node as possible. A lot can happen in the SocketHandler thread that would be missed by Wireshark.
Additionally, if we are to use Wireshark, we are at the mercy of whoever it maintaining the protocol in Wireshark, both as to it being accurate and recent. As can be seen by the **many** previous attempts to include Bitcoin in Wireshark (google "bitcoin dissector") this is easier said than done.
Lastly, I truly believe that this tool will be used significantly more by being included in the codebase. It's just that much more discoverable.
ACKs for top commit:
MarcoFalke:
re-ACK bff7c66e67aa2f18ef70139338643656a54444fe only some minor changes: 👚
jnewbery:
utACK bff7c66e67aa2f18ef70139338643656a54444fe
theStack:
re-ACK bff7c66e67aa2f18ef70139338643656a54444fe
Tree-SHA512: e59e3160422269221f70f98720b47842775781c247c064071d546c24fa7a35a0e5534e8baa4b4591a750d7eb16de6b4ecf54cbee6d193b261f4f104e28c15f47
2021-02-02 13:11:14 +01:00
' p2p_message_capture.py ' ,
2021-05-18 18:30:48 +02:00
' feature_asmap.py ' ,
2018-05-09 06:31:11 +02:00
' feature_includeconf.py ' ,
fix(tests): Fix retries for the non-deterministic test suite (#5307)
## Issue being fixed or feature implemented
the problem with retries implemented in #4793 is that they don't really
do anything besides fetching results of a failed job multiple times 🙈
## What was done?
partially reverted changes done in #4793, implemented actual job
restart. dropped `--sleep` and `--retries` and added `--attempts`
instead.
## How Has This Been Tested?
Pick any test, add some randomly failing expression somewhere and run it
with some high number of retries.
For example:
```diff
diff --git a/test/functional/feature_dip0020_activation.py b/test/functional/feature_dip0020_activation.py
index 471e4fdc66..b56a954b78 100755
--- a/test/functional/feature_dip0020_activation.py
+++ b/test/functional/feature_dip0020_activation.py
@@ -69,6 +69,7 @@ class DIP0020ActivationTest(BitcoinTestFramework):
# Should be spendable now
tx0id = self.node.sendrawtransaction(tx0_hex)
assert tx0id in set(self.node.getrawmempool())
+ assert int(tx0id[0], 16) < 4
if __name__ == '__main__':
```
On develop:
```
./test/functional/test_runner.py feature_dip0020_activation.py --retries=100 --sleep=0
```
if this fails on the first run, it keeps "failing" (simply fetching the
same results actually) till the end.
With this patch:
```
./test/functional/test_runner.py feature_dip0020_activation.py --attempts=100
```
if this fails on the first run, it can actually succeed after a few
attempts now, unless you are extremely unlucky ofc 😄
Also, check [ci results in my repo
](https://cdn.artifacts.gitlab-static.net/93/b4/93b4f8b17e5dcccab1afee165b4d74d90f05800caf65d6c48a83a1a78c979587/2023_04_08/4081291268/4478867166/job.log?response-content-type=text%2Fplain%3B%20charset%3Dutf-8&response-content-disposition=inline&Expires=1680945516&KeyName=gprd-artifacts-cdn&Signature=2d4mHCJBbgRaTDiSQ6kKIy1PdIM=).
Note:
```
...
feature_dip3_v19.py failed at attempt 1/3, Duration: 159s
...
4/179 - [1mfeature_dip3_v19.py[0m passed, Duration: 244 s
...
feature_llmq_hpmn.py failed at attempt 1/3, Duration: 284s
...
feature_llmq_hpmn.py failed at attempt 2/3, Duration: 296s
...
11/179 - [1mfeature_llmq_hpmn.py[0m failed, Duration: 233 s
...
```
An example with 2 tests failing initially and then passing:
https://gitlab.com/dashpay/dash/-/jobs/4089689970
## Breaking Changes
n/a
## Checklist:
- [x] I have performed a self-review of my own code
- [ ] I have commented my code, particularly in hard-to-understand areas
- [ ] I have added or updated relevant unit/integration/functional/e2e
tests
- [ ] I have made corresponding changes to the documentation
**For repository code-owners and collaborators only**
- [x] I have assigned this pull request to a milestone
2023-04-15 06:13:47 +02:00
' mempool_unbroadcast.py ' ,
2020-06-16 13:21:18 +02:00
' mempool_compatibility.py ' ,
2021-10-12 09:16:07 +02:00
' rpc_deriveaddresses.py ' ,
' rpc_deriveaddresses.py --usecli ' ,
2018-01-15 22:23:44 +01:00
' rpc_scantxoutset.py ' ,
2017-12-04 18:30:25 +01:00
' feature_logging.py ' ,
2023-07-30 13:49:32 +02:00
' feature_coinstatsindex.py ' ,
2021-06-04 13:53:11 +02:00
' wallet_orphanedreward.py ' ,
2020-07-17 01:44:20 +02:00
' p2p_node_network_limited.py ' ,
Merge #16248: Make whitebind/whitelist permissions more flexible
c5b404e8f1973afe071a07c63ba1038eefe13f0f Add functional tests for flexible whitebind/list (nicolas.dorier)
d541fa391844f658bd7035659b5b16695733dd56 Replace the use of fWhitelisted by permission checks (nicolas.dorier)
ecd5cf7ea4c3644a30092100ffc399e30e193275 Do not disconnect peer for asking mempool if it has NO_BAN permission (nicolas.dorier)
e5b26deaaa6842f7dd7c4537ede000f965ea0189 Make whitebind/whitelist permissions more flexible (nicolas.dorier)
Pull request description:
# Motivation
In 0.19, bloom filter will be disabled by default. I tried to make [a PR](https://github.com/bitcoin/bitcoin/pull/16176) to enable bloom filter for whitelisted peers regardless of `-peerbloomfilters`.
Bloom filter have non existent privacy and server can omit filter's matches. However, both problems are completely irrelevant when you connect to your own node. If you connect to your own node, bloom filters are the most bandwidth efficient way to synchronize your light client without the need of some middleware like Electrum.
It is also a superior alternative to BIP157 as it does not require to maintain an additional index and it would work well on pruned nodes.
When I attempted to allow bloom filters for whitelisted peer, my proposal has been NACKed in favor of [a more flexible approach](https://github.com/bitcoin/bitcoin/pull/16176#issuecomment-500762907) which should allow node operator to set fine grained permissions instead of a global `whitelisted` attribute.
Doing so will also make follow up idea very easy to implement in a backward compatible way.
# Implementation details
The PR propose a new format for `--white{list,bind}`. I added a way to specify permissions granted to inbound connection matching `white{list,bind}`.
The following permissions exists:
* ForceRelay
* Relay
* NoBan
* BloomFilter
* Mempool
Example:
* `-whitelist=bloomfilter@127.0.0.1/32`.
* `-whitebind=bloomfilter,relay,noban@127.0.0.1:10020`.
If no permissions are specified, `NoBan | Mempool` is assumed. (making this PR backward compatible)
When we receive an inbound connection, we calculate the effective permissions for this peer by fetching the permissions granted from `whitelist` and add to it the permissions granted from `whitebind`.
To keep backward compatibility, if no permissions are specified in `white{list,bind}` (e.g. `--whitelist=127.0.0.1`) then parameters `-whitelistforcerelay` and `-whiterelay` will add the permissions `ForceRelay` and `Relay` to the inbound node.
`-whitelistforcerelay` and `-whiterelay` are ignored if the permissions flags are explicitly set in `white{bind,list}`.
# Follow up idea
Based on this PR, other changes become quite easy to code in a trivially review-able, backward compatible way:
* Changing `connect` at rpc and config file level to understand the permissions flags.
* Changing the permissions of a peer at RPC level.
ACKs for top commit:
laanwj:
re-ACK c5b404e8f1973afe071a07c63ba1038eefe13f0f
Tree-SHA512: adfefb373d09e68cae401247c8fc64034e305694cdef104bdcdacb9f1704277bd53b18f52a2427a5cffdbc77bda410d221aed252bc2ece698ffbb9cf1b830577
2019-08-14 16:35:54 +02:00
' p2p_permissions.py ' ,
2018-03-27 21:04:22 +02:00
' feature_blocksdir.py ' ,
2022-06-21 15:30:55 +02:00
' wallet_startup.py ' ,
2023-07-14 17:22:44 +02:00
' p2p_i2p_ports.py ' ,
2020-07-17 01:44:20 +02:00
' feature_config_args.py ' ,
2020-07-23 18:39:18 +02:00
' feature_settings.py ' ,
2020-03-10 13:17:18 +01:00
' rpc_getdescriptorinfo.py ' ,
2020-01-07 23:23:58 +01:00
' rpc_getaddressinfo_labels_purpose_deprecation.py ' ,
2020-02-02 09:35:24 +01:00
' rpc_getaddressinfo_label_deprecation.py ' ,
2018-08-29 15:33:14 +02:00
' rpc_help.py ' ,
Merge #12843: [tests] Test starting bitcoind with -h and -version
63048ec73d [tests] Test starting bitcoind with -h and -version (John Newbery)
Pull request description:
Test that starting bitcoind/bitcoin-qt with `-h` and `-version` works as expected.
Prompted by https://github.com/bitcoin/bitcoin/pull/10762#commitcomment-28345993, which is a nullpointer dereference triggered by starting bitcoin-qt with `-h`.
On master, this test passes when run over bitcoind, but fails when running over bitcoin-qt. I used xvfb as a virtual frame buffer to test:
```
BITCOIND=/home/ubuntu/bitcoin/src/qt/bitcoin-qt xvfb-run ./feature_help.py --nocleanup
2018-03-30T17:09:37.767000Z TestFramework (INFO): Initializing test directory /tmp/user/1000/testdi4dre13
2018-03-30T17:09:37.767000Z TestFramework (INFO): Start bitcoin with -h for help text
2018-03-30T17:09:37.841000Z TestFramework (ERROR): Assertion failed
Traceback (most recent call last):
File "/home/ubuntu/bitcoin/test/functional/test_framework/test_framework.py", line 126, in main
self.run_test()
File "./feature_help.py", line 25, in run_test
assert_equal(ret_code, 0)
File "/home/ubuntu/bitcoin/test/functional/test_framework/util.py", line 39, in assert_equal
raise AssertionError("not(%s)" % " == ".join(str(arg) for arg in (thing1, thing2) + args))
AssertionError: not(-11 == 0)
2018-03-30T17:09:37.842000Z TestFramework (INFO): Stopping nodes
Traceback (most recent call last):
File "./feature_help.py", line 42, in <module>
HelpTest().main()
File "/home/ubuntu/bitcoin/test/functional/test_framework/test_framework.py", line 149, in main
self.stop_nodes()
File "/home/ubuntu/bitcoin/test/functional/test_framework/test_framework.py", line 273, in stop_nodes
node.stop_node()
File "/home/ubuntu/bitcoin/test/functional/test_framework/test_node.py", line 141, in stop_node
self.stop()
File "/home/ubuntu/bitcoin/test/functional/test_framework/test_node.py", line 87, in __getattr__
assert self.rpc_connected and self.rpc is not None, "Error: no RPC connection"
AssertionError: Error: no RPC connection
```
Passes for bitcoind and bitcoin-qt when run on #12836.
Longer term, we should consider running functional tests over bitcoin-qt in one of the Travis jobs.
Tree-SHA512: 0c2f40f3d5f0e78c3a1b07dbee8fd383eebab27ed0bf2a98a5b9cc66613dbd7b70e363c56163a37e02f68ae7ff7b3ae1769705d0e110ca68a00f8693315730a4
2018-04-01 21:27:44 +02:00
' feature_help.py ' ,
2023-07-17 15:57:30 +02:00
' feature_blockfilterindex_prune.py '
2018-01-03 11:06:04 +01:00
# Don't append tests at the end to avoid merge conflicts
# Put them in a random line within the section that fits their approximate run-time
2015-08-26 12:05:36 +02:00
]
2016-04-09 21:14:18 +02:00
fix(tests): Fix retries for the non-deterministic test suite (#5307)
## Issue being fixed or feature implemented
the problem with retries implemented in #4793 is that they don't really
do anything besides fetching results of a failed job multiple times 🙈
## What was done?
partially reverted changes done in #4793, implemented actual job
restart. dropped `--sleep` and `--retries` and added `--attempts`
instead.
## How Has This Been Tested?
Pick any test, add some randomly failing expression somewhere and run it
with some high number of retries.
For example:
```diff
diff --git a/test/functional/feature_dip0020_activation.py b/test/functional/feature_dip0020_activation.py
index 471e4fdc66..b56a954b78 100755
--- a/test/functional/feature_dip0020_activation.py
+++ b/test/functional/feature_dip0020_activation.py
@@ -69,6 +69,7 @@ class DIP0020ActivationTest(BitcoinTestFramework):
# Should be spendable now
tx0id = self.node.sendrawtransaction(tx0_hex)
assert tx0id in set(self.node.getrawmempool())
+ assert int(tx0id[0], 16) < 4
if __name__ == '__main__':
```
On develop:
```
./test/functional/test_runner.py feature_dip0020_activation.py --retries=100 --sleep=0
```
if this fails on the first run, it keeps "failing" (simply fetching the
same results actually) till the end.
With this patch:
```
./test/functional/test_runner.py feature_dip0020_activation.py --attempts=100
```
if this fails on the first run, it can actually succeed after a few
attempts now, unless you are extremely unlucky ofc 😄
Also, check [ci results in my repo
](https://cdn.artifacts.gitlab-static.net/93/b4/93b4f8b17e5dcccab1afee165b4d74d90f05800caf65d6c48a83a1a78c979587/2023_04_08/4081291268/4478867166/job.log?response-content-type=text%2Fplain%3B%20charset%3Dutf-8&response-content-disposition=inline&Expires=1680945516&KeyName=gprd-artifacts-cdn&Signature=2d4mHCJBbgRaTDiSQ6kKIy1PdIM=).
Note:
```
...
feature_dip3_v19.py failed at attempt 1/3, Duration: 159s
...
4/179 - [1mfeature_dip3_v19.py[0m passed, Duration: 244 s
...
feature_llmq_hpmn.py failed at attempt 1/3, Duration: 284s
...
feature_llmq_hpmn.py failed at attempt 2/3, Duration: 296s
...
11/179 - [1mfeature_llmq_hpmn.py[0m failed, Duration: 233 s
...
```
An example with 2 tests failing initially and then passing:
https://gitlab.com/dashpay/dash/-/jobs/4089689970
## Breaking Changes
n/a
## Checklist:
- [x] I have performed a self-review of my own code
- [ ] I have commented my code, particularly in hard-to-understand areas
- [ ] I have added or updated relevant unit/integration/functional/e2e
tests
- [ ] I have made corresponding changes to the documentation
**For repository code-owners and collaborators only**
- [x] I have assigned this pull request to a milestone
2023-04-15 06:13:47 +02:00
# Place EXTENDED_SCRIPTS first since it has the 3 longest running tests
ALL_SCRIPTS = EXTENDED_SCRIPTS + BASE_SCRIPTS
2019-01-03 10:18:47 +01:00
2017-03-28 11:36:29 +02:00
NON_SCRIPTS = [
# These are python files that live in the functional tests directory, but are not test scripts.
" combine_logs.py " ,
" create_cache.py " ,
" test_runner.py " ,
]
2019-01-03 10:18:47 +01:00
def main ( ) :
# Parse arguments and pass through unrecognised args
parser = argparse . ArgumentParser ( add_help = False ,
2019-05-19 22:20:34 +02:00
usage = ' %(prog)s [test_runner.py options] [script options] [scripts] ' ,
2019-01-03 10:18:47 +01:00
description = __doc__ ,
epilog = '''
Help text and arguments for individual test script : ''' ,
formatter_class = argparse . RawTextHelpFormatter )
2019-08-15 13:42:20 +02:00
parser . add_argument ( ' --ansi ' , action = ' store_true ' , default = sys . stdout . isatty ( ) , help = " Use ANSI colors and dots in output (enabled by default when standard output is a TTY) " )
fix(tests): Fix retries for the non-deterministic test suite (#5307)
## Issue being fixed or feature implemented
the problem with retries implemented in #4793 is that they don't really
do anything besides fetching results of a failed job multiple times 🙈
## What was done?
partially reverted changes done in #4793, implemented actual job
restart. dropped `--sleep` and `--retries` and added `--attempts`
instead.
## How Has This Been Tested?
Pick any test, add some randomly failing expression somewhere and run it
with some high number of retries.
For example:
```diff
diff --git a/test/functional/feature_dip0020_activation.py b/test/functional/feature_dip0020_activation.py
index 471e4fdc66..b56a954b78 100755
--- a/test/functional/feature_dip0020_activation.py
+++ b/test/functional/feature_dip0020_activation.py
@@ -69,6 +69,7 @@ class DIP0020ActivationTest(BitcoinTestFramework):
# Should be spendable now
tx0id = self.node.sendrawtransaction(tx0_hex)
assert tx0id in set(self.node.getrawmempool())
+ assert int(tx0id[0], 16) < 4
if __name__ == '__main__':
```
On develop:
```
./test/functional/test_runner.py feature_dip0020_activation.py --retries=100 --sleep=0
```
if this fails on the first run, it keeps "failing" (simply fetching the
same results actually) till the end.
With this patch:
```
./test/functional/test_runner.py feature_dip0020_activation.py --attempts=100
```
if this fails on the first run, it can actually succeed after a few
attempts now, unless you are extremely unlucky ofc 😄
Also, check [ci results in my repo
](https://cdn.artifacts.gitlab-static.net/93/b4/93b4f8b17e5dcccab1afee165b4d74d90f05800caf65d6c48a83a1a78c979587/2023_04_08/4081291268/4478867166/job.log?response-content-type=text%2Fplain%3B%20charset%3Dutf-8&response-content-disposition=inline&Expires=1680945516&KeyName=gprd-artifacts-cdn&Signature=2d4mHCJBbgRaTDiSQ6kKIy1PdIM=).
Note:
```
...
feature_dip3_v19.py failed at attempt 1/3, Duration: 159s
...
4/179 - [1mfeature_dip3_v19.py[0m passed, Duration: 244 s
...
feature_llmq_hpmn.py failed at attempt 1/3, Duration: 284s
...
feature_llmq_hpmn.py failed at attempt 2/3, Duration: 296s
...
11/179 - [1mfeature_llmq_hpmn.py[0m failed, Duration: 233 s
...
```
An example with 2 tests failing initially and then passing:
https://gitlab.com/dashpay/dash/-/jobs/4089689970
## Breaking Changes
n/a
## Checklist:
- [x] I have performed a self-review of my own code
- [ ] I have commented my code, particularly in hard-to-understand areas
- [ ] I have added or updated relevant unit/integration/functional/e2e
tests
- [ ] I have made corresponding changes to the documentation
**For repository code-owners and collaborators only**
- [x] I have assigned this pull request to a milestone
2023-04-15 06:13:47 +02:00
parser . add_argument ( ' --attempts ' , ' -a ' , type = int , default = 1 , help = ' how many attempts should be allowed for the non-deterministic test suite. Default=1. ' )
2018-11-06 16:44:39 +01:00
parser . add_argument ( ' --combinedlogslen ' , ' -c ' , type = int , default = 0 , metavar = ' n ' , help = ' On failure, print a log (of length n lines) to the console, combined from the test framework and all test nodes. ' )
2019-01-03 10:18:47 +01:00
parser . add_argument ( ' --coverage ' , action = ' store_true ' , help = ' generate a basic coverage report for the RPC interface ' )
2018-11-02 20:04:55 +01:00
parser . add_argument ( ' --ci ' , action = ' store_true ' , help = ' Run checks and code that are usually only enabled in a continuous integration environment ' )
2017-08-16 00:28:57 +02:00
parser . add_argument ( ' --exclude ' , ' -x ' , help = ' specify a comma-separated-list of scripts to exclude. ' )
2019-01-03 10:18:47 +01:00
parser . add_argument ( ' --extended ' , action = ' store_true ' , help = ' run the extended test suite in addition to the basic tests ' )
parser . add_argument ( ' --help ' , ' -h ' , ' -? ' , action = ' store_true ' , help = ' print help text and exit ' )
parser . add_argument ( ' --jobs ' , ' -j ' , type = int , default = 4 , help = ' how many test scripts to run in parallel. Default=4. ' )
2017-04-17 22:15:12 +02:00
parser . add_argument ( ' --keepcache ' , ' -k ' , action = ' store_true ' , help = ' the default behavior is to flush the cache directory on startup. --keepcache retains the cache from the previous testrun. ' )
2018-11-01 15:42:10 +01:00
parser . add_argument ( ' --quiet ' , ' -q ' , action = ' store_true ' , help = ' only print dots, results summary and failure logs ' )
2017-05-22 08:59:11 +02:00
parser . add_argument ( ' --tmpdirprefix ' , ' -t ' , default = tempfile . gettempdir ( ) , help = " Root directory for datadirs " )
2021-11-15 09:32:01 +01:00
parser . add_argument ( ' --failfast ' , ' -F ' , action = ' store_true ' , help = ' stop execution after the first test failure ' )
2019-07-16 15:01:20 +02:00
parser . add_argument ( ' --filter ' , help = ' filter scripts to run by regular expression ' )
2019-08-15 13:42:20 +02:00
2019-01-03 10:18:47 +01:00
args , unknown_args = parser . parse_known_args ( )
2019-08-15 13:42:20 +02:00
if not args . ansi :
2022-05-10 13:12:05 +02:00
global DEFAULT , BOLD , GREEN , RED
DEFAULT = ( " " , " " )
2019-08-15 13:42:20 +02:00
BOLD = ( " " , " " )
GREEN = ( " " , " " )
RED = ( " " , " " )
2019-01-03 10:18:47 +01:00
2017-05-17 08:11:46 +02:00
# args to be passed on always start with two dashes; tests are the remaining unknown args
tests = [ arg for arg in unknown_args if arg [ : 2 ] != " -- " ]
2019-01-03 10:18:47 +01:00
passon_args = [ arg for arg in unknown_args if arg [ : 2 ] == " -- " ]
# Read config generated by configure.
config = configparser . ConfigParser ( )
2017-06-06 23:54:23 +02:00
configfile = os . path . abspath ( os . path . dirname ( __file__ ) ) + " /../config.ini "
2018-06-16 15:21:01 +02:00
config . read_file ( open ( configfile , encoding = " utf8 " ) )
2017-05-03 08:37:14 +02:00
passon_args . append ( " --configfile= %s " % configfile )
2019-01-03 10:18:47 +01:00
2017-03-28 11:24:14 +02:00
# Set up logging
logging_level = logging . INFO if args . quiet else logging . DEBUG
logging . basicConfig ( format = ' %(message)s ' , level = logging_level )
2017-05-22 08:59:11 +02:00
# Create base test directory
2021-06-29 23:59:35 +02:00
tmpdir = " %s /test_runner_∋_🏃_ %s " % ( args . tmpdirprefix , datetime . datetime . now ( ) . strftime ( " % Y % m %d _ % H % M % S " ) )
2018-09-24 22:10:13 +02:00
2017-05-22 08:59:11 +02:00
os . makedirs ( tmpdir )
logging . debug ( " Temporary test directory at %s " % tmpdir )
2019-01-03 10:18:47 +01:00
enable_bitcoind = config [ " components " ] . getboolean ( " ENABLE_BITCOIND " )
2018-09-13 12:33:15 +02:00
if not enable_bitcoind :
print ( " No functional tests to run. " )
print ( " Rerun ./configure with --with-daemon and then make " )
2019-01-03 10:18:47 +01:00
sys . exit ( 0 )
# Build list of tests
2018-03-18 00:28:02 +01:00
test_list = [ ]
2019-01-03 10:18:47 +01:00
if tests :
# Individual tests have been specified. Run specified tests that exist
2019-07-18 03:39:24 +02:00
# in the ALL_SCRIPTS list. Accept names with or without a .py extension.
# Specified tests can contain wildcards, but in that case the supplied
# paths should be coherent, e.g. the same path as that provided to call
# test_runner.py. Examples:
# `test/functional/test_runner.py test/functional/wallet*`
# `test/functional/test_runner.py ./test/functional/wallet*`
# `test_runner.py wallet*`
# but not:
# `test/functional/test_runner.py wallet*`
# Multiple wildcards can be passed:
# `test_runner.py tool* mempool*`
2018-03-17 16:05:37 +01:00
for test in tests :
2019-07-18 03:39:24 +02:00
script = test . split ( " / " ) [ - 1 ]
script = script + " .py " if " .py " not in script else script
2021-09-28 15:26:37 +02:00
matching_scripts = [ s for s in ALL_SCRIPTS if s . startswith ( script ) ]
if matching_scripts :
test_list . extend ( matching_scripts )
2017-05-17 08:11:46 +02:00
else :
2018-03-17 16:05:37 +01:00
print ( " {} WARNING! {} Test ' {} ' not found in full test list. " . format ( BOLD [ 1 ] , BOLD [ 0 ] , test ) )
elif args . extended :
# Include extended tests
2018-03-18 00:28:02 +01:00
test_list + = ALL_SCRIPTS
2019-01-03 10:18:47 +01:00
else :
2018-03-17 16:05:37 +01:00
# Run base tests only
2018-03-18 00:28:02 +01:00
test_list + = BASE_SCRIPTS
2019-01-03 10:18:47 +01:00
2019-01-07 10:55:35 +01:00
# Remove the test cases that the user has explicitly asked to exclude.
if args . exclude :
2018-09-27 18:16:13 +02:00
exclude_tests = [ test . split ( ' .py ' ) [ 0 ] for test in args . exclude . split ( ' , ' ) ]
2018-03-17 16:05:37 +01:00
for exclude_test in exclude_tests :
2018-09-27 18:16:13 +02:00
# Remove <test_name>.py and <test_name>.py --arg from the test list
exclude_list = [ test for test in test_list if test . split ( ' .py ' ) [ 0 ] == exclude_test ]
for exclude_item in exclude_list :
test_list . remove ( exclude_item )
if not exclude_list :
2017-05-17 08:11:46 +02:00
print ( " {} WARNING! {} Test ' {} ' not found in current test list. " . format ( BOLD [ 1 ] , BOLD [ 0 ] , exclude_test ) )
2019-01-07 10:55:35 +01:00
2019-07-16 15:01:20 +02:00
if args . filter :
test_list = list ( filter ( re . compile ( args . filter ) . search , test_list ) )
2019-01-07 10:55:35 +01:00
if not test_list :
print ( " No valid test scripts specified. Check that your test is in one "
2019-05-19 22:20:34 +02:00
" of the test lists in test_runner.py, or run test_runner.py with no arguments to run all tests " )
2019-01-07 10:55:35 +01:00
sys . exit ( 0 )
2019-01-03 10:18:47 +01:00
if args . help :
2017-03-25 12:15:01 +01:00
# Print help for test_runner.py, then print help of the first script (with args removed) and exit.
2019-01-03 10:18:47 +01:00
parser . print_help ( )
2018-02-12 11:30:17 +01:00
subprocess . check_call ( [ sys . executable , os . path . join ( config [ " environment " ] [ " SRCDIR " ] , ' test ' , ' functional ' , test_list [ 0 ] . split ( ) [ 0 ] ) , ' -h ' ] )
2016-05-09 17:01:55 +02:00
sys . exit ( 0 )
2018-11-02 20:04:55 +01:00
check_script_list ( src_dir = config [ " environment " ] [ " SRCDIR " ] , fail_on_warn = args . ci )
2020-07-17 01:44:20 +02:00
check_script_prefixes ( )
2017-03-28 11:36:29 +02:00
2017-04-17 22:15:12 +02:00
if not args . keepcache :
shutil . rmtree ( " %s /test/cache " % config [ " environment " ] [ " BUILDDIR " ] , ignore_errors = True )
2018-04-30 15:19:26 +02:00
run_tests (
2018-11-02 20:04:55 +01:00
test_list = test_list ,
src_dir = config [ " environment " ] [ " SRCDIR " ] ,
build_dir = config [ " environment " ] [ " BUILDDIR " ] ,
tmpdir = tmpdir ,
2018-04-30 15:19:26 +02:00
jobs = args . jobs ,
fix(tests): Fix retries for the non-deterministic test suite (#5307)
## Issue being fixed or feature implemented
the problem with retries implemented in #4793 is that they don't really
do anything besides fetching results of a failed job multiple times 🙈
## What was done?
partially reverted changes done in #4793, implemented actual job
restart. dropped `--sleep` and `--retries` and added `--attempts`
instead.
## How Has This Been Tested?
Pick any test, add some randomly failing expression somewhere and run it
with some high number of retries.
For example:
```diff
diff --git a/test/functional/feature_dip0020_activation.py b/test/functional/feature_dip0020_activation.py
index 471e4fdc66..b56a954b78 100755
--- a/test/functional/feature_dip0020_activation.py
+++ b/test/functional/feature_dip0020_activation.py
@@ -69,6 +69,7 @@ class DIP0020ActivationTest(BitcoinTestFramework):
# Should be spendable now
tx0id = self.node.sendrawtransaction(tx0_hex)
assert tx0id in set(self.node.getrawmempool())
+ assert int(tx0id[0], 16) < 4
if __name__ == '__main__':
```
On develop:
```
./test/functional/test_runner.py feature_dip0020_activation.py --retries=100 --sleep=0
```
if this fails on the first run, it keeps "failing" (simply fetching the
same results actually) till the end.
With this patch:
```
./test/functional/test_runner.py feature_dip0020_activation.py --attempts=100
```
if this fails on the first run, it can actually succeed after a few
attempts now, unless you are extremely unlucky ofc 😄
Also, check [ci results in my repo
](https://cdn.artifacts.gitlab-static.net/93/b4/93b4f8b17e5dcccab1afee165b4d74d90f05800caf65d6c48a83a1a78c979587/2023_04_08/4081291268/4478867166/job.log?response-content-type=text%2Fplain%3B%20charset%3Dutf-8&response-content-disposition=inline&Expires=1680945516&KeyName=gprd-artifacts-cdn&Signature=2d4mHCJBbgRaTDiSQ6kKIy1PdIM=).
Note:
```
...
feature_dip3_v19.py failed at attempt 1/3, Duration: 159s
...
4/179 - [1mfeature_dip3_v19.py[0m passed, Duration: 244 s
...
feature_llmq_hpmn.py failed at attempt 1/3, Duration: 284s
...
feature_llmq_hpmn.py failed at attempt 2/3, Duration: 296s
...
11/179 - [1mfeature_llmq_hpmn.py[0m failed, Duration: 233 s
...
```
An example with 2 tests failing initially and then passing:
https://gitlab.com/dashpay/dash/-/jobs/4089689970
## Breaking Changes
n/a
## Checklist:
- [x] I have performed a self-review of my own code
- [ ] I have commented my code, particularly in hard-to-understand areas
- [ ] I have added or updated relevant unit/integration/functional/e2e
tests
- [ ] I have made corresponding changes to the documentation
**For repository code-owners and collaborators only**
- [x] I have assigned this pull request to a milestone
2023-04-15 06:13:47 +02:00
attempts = args . attempts ,
2018-04-30 15:19:26 +02:00
enable_coverage = args . coverage ,
args = passon_args ,
2020-06-14 03:58:03 +02:00
combined_logs_len = args . combinedlogslen ,
2018-11-02 20:04:55 +01:00
failfast = args . failfast ,
2019-08-15 13:42:20 +02:00
use_term_control = args . ansi ,
2018-04-30 15:19:26 +02:00
)
2020-03-23 12:37:38 +01:00
def run_tests ( * , test_list , src_dir , build_dir , tmpdir , jobs = 1 , attempts = 1 , enable_coverage = False , args = None , combined_logs_len = 0 , failfast = False , use_term_control ) :
2018-04-30 15:19:26 +02:00
args = args or [ ]
2019-01-03 10:18:47 +01:00
2019-11-18 14:23:33 +01:00
# Warn if dashd is already running
2017-04-17 22:15:12 +02:00
try :
Merge #19368: test: improve functional tests compatibility with BSD/macOS
3a7e79478ab41af7c53ce14d9fca9815bffe1f73 test: retry when write to a socket fails on macOS (Ivan Metlushko)
8cf9d15b823d91d2a74fc83832fccca2219342c9 test: use pgrep for better compatibility (Ivan Metlushko)
Pull request description:
Rationale: a few minor changes to make experience of running tests on macOS a bit better
1.`pidof` is not available on BSD/macOS, while `pgrep` is present on BSD, Linux and macOS
2. Add retry as a workaround for a weird behavior when writing to a socket (https://bugs.python.org/issue33450). Stacktrace attached
Man pages:
https://www.freebsd.org/cgi/man.cgi?query=pgrep&apropos=0&sektion=1&manpath=FreeBSD+6.0-RELEASE&arch=default&format=html
https://man7.org/linux/man-pages/man1/pgrep.1.html
Related to #19281
Stacktrace example:
```
...
33/161 - feature_abortnode.py failed, Duration: 63 s
stdout:
2020-06-11T10:46:43.947000Z TestFramework (INFO): Initializing test directory /var/folders/2q/d5w9zh614r7g5c8r74ln3g400000gq/T/test_runner_₿_🏃_20200611_174102/feature_abortnode_128
2020-06-11T10:46:45.199000Z TestFramework (INFO): Waiting for crash
2020-06-11T10:47:15.921000Z TestFramework (INFO): Node crashed - now verifying restart fails
2020-06-11T10:47:47.068000Z TestFramework (INFO): Stopping nodes
[node 1] Cleaning up leftover process
stderr:
Traceback (most recent call last):
File "/Users/xxx/Projects/bitcoin/test/functional/feature_abortnode.py", line 50, in <module>
AbortNodeTest().main()
File "/Users/xxx/Projects/bitcoin/test/functional/test_framework/test_framework.py", line 142, in main
exit_code = self.shutdown()
File "/Users/xxx/Projects/bitcoin/test/functional/test_framework/test_framework.py", line 266, in shutdown
self.stop_nodes()
File "/Users/xxx/Projects/bitcoin/test/functional/test_framework/test_framework.py", line 515, in stop_nodes
node.stop_node(wait=wait)
File "/Users/xxx/Projects/bitcoin/test/functional/test_framework/test_node.py", line 318, in stop_node
self.stop(wait=wait)
File "/Users/xxx/Projects/bitcoin/test/functional/test_framework/coverage.py", line 47, in __call__
return_val = self.auth_service_proxy_instance.__call__(*args, **kwargs)
File "/Users/xxx/Projects/bitcoin/test/functional/test_framework/authproxy.py", line 142, in __call__
response, status = self._request('POST', self.__url.path, postdata.encode('utf-8'))
File "/Users/xxx/Projects/bitcoin/test/functional/test_framework/authproxy.py", line 107, in _request
self.__conn.request(method, path, postdata, headers)
File "/Users/xxx/.pyenv/versions/3.5.6/lib/python3.5/http/client.py", line 1107, in request
self._send_request(method, url, body, headers)
File "/Users/xxx/.pyenv/versions/3.5.6/lib/python3.5/http/client.py", line 1152, in _send_request
self.endheaders(body)
File "/Users/xxx/.pyenv/versions/3.5.6/lib/python3.5/http/client.py", line 1103, in endheaders
self._send_output(message_body)
File "/Users/xxx/.pyenv/versions/3.5.6/lib/python3.5/http/client.py", line 936, in _send_output
self.send(message_body)
File "/Users/xxx/.pyenv/versions/3.5.6/lib/python3.5/http/client.py", line 908, in send
self.sock.sendall(data)
OSError: [Errno 41] Protocol wrong type for socket
```
ACKs for top commit:
laanwj:
ACK 3a7e79478ab41af7c53ce14d9fca9815bffe1f73
Tree-SHA512: fefbe40ce94ab29f18bbbed2a434194b1384ffa5279b1d04db7a3708e3dd422bd9e450f1db3f95a1a851fac5a626ab533c6ebcfd7ede96f8ccae9e6f3e9fff92
2020-07-01 15:06:29 +02:00
# pgrep exits with code zero when one or more matching processes found
if subprocess . run ( [ " pgrep " , " -x " , " dashd " ] , stdout = subprocess . DEVNULL ) . returncode == 0 :
2017-04-17 22:15:12 +02:00
print ( " %s WARNING! %s There is already a dashd process running on this system. Tests may fail unexpectedly due to resource contention! " % ( BOLD [ 1 ] , BOLD [ 0 ] ) )
Merge #19368: test: improve functional tests compatibility with BSD/macOS
3a7e79478ab41af7c53ce14d9fca9815bffe1f73 test: retry when write to a socket fails on macOS (Ivan Metlushko)
8cf9d15b823d91d2a74fc83832fccca2219342c9 test: use pgrep for better compatibility (Ivan Metlushko)
Pull request description:
Rationale: a few minor changes to make experience of running tests on macOS a bit better
1.`pidof` is not available on BSD/macOS, while `pgrep` is present on BSD, Linux and macOS
2. Add retry as a workaround for a weird behavior when writing to a socket (https://bugs.python.org/issue33450). Stacktrace attached
Man pages:
https://www.freebsd.org/cgi/man.cgi?query=pgrep&apropos=0&sektion=1&manpath=FreeBSD+6.0-RELEASE&arch=default&format=html
https://man7.org/linux/man-pages/man1/pgrep.1.html
Related to #19281
Stacktrace example:
```
...
33/161 - feature_abortnode.py failed, Duration: 63 s
stdout:
2020-06-11T10:46:43.947000Z TestFramework (INFO): Initializing test directory /var/folders/2q/d5w9zh614r7g5c8r74ln3g400000gq/T/test_runner_₿_🏃_20200611_174102/feature_abortnode_128
2020-06-11T10:46:45.199000Z TestFramework (INFO): Waiting for crash
2020-06-11T10:47:15.921000Z TestFramework (INFO): Node crashed - now verifying restart fails
2020-06-11T10:47:47.068000Z TestFramework (INFO): Stopping nodes
[node 1] Cleaning up leftover process
stderr:
Traceback (most recent call last):
File "/Users/xxx/Projects/bitcoin/test/functional/feature_abortnode.py", line 50, in <module>
AbortNodeTest().main()
File "/Users/xxx/Projects/bitcoin/test/functional/test_framework/test_framework.py", line 142, in main
exit_code = self.shutdown()
File "/Users/xxx/Projects/bitcoin/test/functional/test_framework/test_framework.py", line 266, in shutdown
self.stop_nodes()
File "/Users/xxx/Projects/bitcoin/test/functional/test_framework/test_framework.py", line 515, in stop_nodes
node.stop_node(wait=wait)
File "/Users/xxx/Projects/bitcoin/test/functional/test_framework/test_node.py", line 318, in stop_node
self.stop(wait=wait)
File "/Users/xxx/Projects/bitcoin/test/functional/test_framework/coverage.py", line 47, in __call__
return_val = self.auth_service_proxy_instance.__call__(*args, **kwargs)
File "/Users/xxx/Projects/bitcoin/test/functional/test_framework/authproxy.py", line 142, in __call__
response, status = self._request('POST', self.__url.path, postdata.encode('utf-8'))
File "/Users/xxx/Projects/bitcoin/test/functional/test_framework/authproxy.py", line 107, in _request
self.__conn.request(method, path, postdata, headers)
File "/Users/xxx/.pyenv/versions/3.5.6/lib/python3.5/http/client.py", line 1107, in request
self._send_request(method, url, body, headers)
File "/Users/xxx/.pyenv/versions/3.5.6/lib/python3.5/http/client.py", line 1152, in _send_request
self.endheaders(body)
File "/Users/xxx/.pyenv/versions/3.5.6/lib/python3.5/http/client.py", line 1103, in endheaders
self._send_output(message_body)
File "/Users/xxx/.pyenv/versions/3.5.6/lib/python3.5/http/client.py", line 936, in _send_output
self.send(message_body)
File "/Users/xxx/.pyenv/versions/3.5.6/lib/python3.5/http/client.py", line 908, in send
self.sock.sendall(data)
OSError: [Errno 41] Protocol wrong type for socket
```
ACKs for top commit:
laanwj:
ACK 3a7e79478ab41af7c53ce14d9fca9815bffe1f73
Tree-SHA512: fefbe40ce94ab29f18bbbed2a434194b1384ffa5279b1d04db7a3708e3dd422bd9e450f1db3f95a1a851fac5a626ab533c6ebcfd7ede96f8ccae9e6f3e9fff92
2020-07-01 15:06:29 +02:00
except OSError :
# pgrep not supported
2017-04-17 22:15:12 +02:00
pass
# Warn if there is a cache directory
cache_dir = " %s /test/cache " % build_dir
if os . path . isdir ( cache_dir ) :
print ( " %s WARNING! %s There is a cache directory here: %s . If tests fail unexpectedly, try deleting the cache directory. " % ( BOLD [ 1 ] , BOLD [ 0 ] , cache_dir ) )
2020-03-26 21:57:53 +01:00
# Test Framework Tests
print ( " Running Unit Tests for Test Framework Modules " )
test_framework_tests = unittest . TestSuite ( )
for module in TEST_FRAMEWORK_MODULES :
test_framework_tests . addTest ( unittest . TestLoader ( ) . loadTestsFromName ( " test_framework. {} " . format ( module ) ) )
result = unittest . TextTestRunner ( verbosity = 1 , failfast = True ) . run ( test_framework_tests )
if not result . wasSuccessful ( ) :
logging . debug ( " Early exiting after failure in TestFramework unit tests " )
sys . exit ( False )
2019-05-19 22:20:34 +02:00
tests_dir = src_dir + ' /test/functional/ '
2015-10-11 07:41:19 +02:00
2018-05-09 10:41:40 +02:00
flags = [ ' --cachedir= {} ' . format ( cache_dir ) ] + args
2019-01-03 10:18:47 +01:00
if enable_coverage :
2015-10-11 07:41:19 +02:00
coverage = RPCCoverage ( )
2016-05-10 18:27:31 +02:00
flags . append ( coverage . flag )
2017-03-28 11:24:14 +02:00
logging . debug ( " Initializing coverage directory at %s " % coverage . dir )
2019-01-03 10:18:47 +01:00
else :
coverage = None
2016-05-10 18:27:31 +02:00
2019-01-03 10:18:47 +01:00
if len ( test_list ) > 1 and jobs > 1 :
2016-05-10 18:27:31 +02:00
# Populate cache
2017-11-18 14:32:50 +01:00
try :
2018-02-12 11:30:17 +01:00
subprocess . check_output ( [ sys . executable , tests_dir + ' create_cache.py ' ] + flags + [ " --tmpdir= %s /cache " % tmpdir ] )
2018-01-22 14:28:07 +01:00
except subprocess . CalledProcessError as e :
sys . stdout . buffer . write ( e . output )
raise
2016-04-09 21:14:18 +02:00
#Run Tests
2018-11-02 20:04:55 +01:00
job_queue = TestHandler (
num_tests_parallel = jobs ,
tests_dir = tests_dir ,
tmpdir = tmpdir ,
test_list = test_list ,
flags = flags ,
2019-08-15 13:42:20 +02:00
use_term_control = use_term_control ,
fix(tests): Fix retries for the non-deterministic test suite (#5307)
## Issue being fixed or feature implemented
the problem with retries implemented in #4793 is that they don't really
do anything besides fetching results of a failed job multiple times 🙈
## What was done?
partially reverted changes done in #4793, implemented actual job
restart. dropped `--sleep` and `--retries` and added `--attempts`
instead.
## How Has This Been Tested?
Pick any test, add some randomly failing expression somewhere and run it
with some high number of retries.
For example:
```diff
diff --git a/test/functional/feature_dip0020_activation.py b/test/functional/feature_dip0020_activation.py
index 471e4fdc66..b56a954b78 100755
--- a/test/functional/feature_dip0020_activation.py
+++ b/test/functional/feature_dip0020_activation.py
@@ -69,6 +69,7 @@ class DIP0020ActivationTest(BitcoinTestFramework):
# Should be spendable now
tx0id = self.node.sendrawtransaction(tx0_hex)
assert tx0id in set(self.node.getrawmempool())
+ assert int(tx0id[0], 16) < 4
if __name__ == '__main__':
```
On develop:
```
./test/functional/test_runner.py feature_dip0020_activation.py --retries=100 --sleep=0
```
if this fails on the first run, it keeps "failing" (simply fetching the
same results actually) till the end.
With this patch:
```
./test/functional/test_runner.py feature_dip0020_activation.py --attempts=100
```
if this fails on the first run, it can actually succeed after a few
attempts now, unless you are extremely unlucky ofc 😄
Also, check [ci results in my repo
](https://cdn.artifacts.gitlab-static.net/93/b4/93b4f8b17e5dcccab1afee165b4d74d90f05800caf65d6c48a83a1a78c979587/2023_04_08/4081291268/4478867166/job.log?response-content-type=text%2Fplain%3B%20charset%3Dutf-8&response-content-disposition=inline&Expires=1680945516&KeyName=gprd-artifacts-cdn&Signature=2d4mHCJBbgRaTDiSQ6kKIy1PdIM=).
Note:
```
...
feature_dip3_v19.py failed at attempt 1/3, Duration: 159s
...
4/179 - [1mfeature_dip3_v19.py[0m passed, Duration: 244 s
...
feature_llmq_hpmn.py failed at attempt 1/3, Duration: 284s
...
feature_llmq_hpmn.py failed at attempt 2/3, Duration: 296s
...
11/179 - [1mfeature_llmq_hpmn.py[0m failed, Duration: 233 s
...
```
An example with 2 tests failing initially and then passing:
https://gitlab.com/dashpay/dash/-/jobs/4089689970
## Breaking Changes
n/a
## Checklist:
- [x] I have performed a self-review of my own code
- [ ] I have commented my code, particularly in hard-to-understand areas
- [ ] I have added or updated relevant unit/integration/functional/e2e
tests
- [ ] I have made corresponding changes to the documentation
**For repository code-owners and collaborators only**
- [x] I have assigned this pull request to a milestone
2023-04-15 06:13:47 +02:00
attempts = attempts ,
2018-11-02 20:04:55 +01:00
)
2018-03-17 16:05:37 +01:00
start_time = time . time ( )
2017-04-07 16:20:39 +02:00
test_results = [ ]
2019-01-03 10:18:47 +01:00
max_len_name = len ( max ( test_list , key = len ) )
2021-06-25 00:19:53 +02:00
test_count = len ( test_list )
fix(tests): Fix retries for the non-deterministic test suite (#5307)
## Issue being fixed or feature implemented
the problem with retries implemented in #4793 is that they don't really
do anything besides fetching results of a failed job multiple times 🙈
## What was done?
partially reverted changes done in #4793, implemented actual job
restart. dropped `--sleep` and `--retries` and added `--attempts`
instead.
## How Has This Been Tested?
Pick any test, add some randomly failing expression somewhere and run it
with some high number of retries.
For example:
```diff
diff --git a/test/functional/feature_dip0020_activation.py b/test/functional/feature_dip0020_activation.py
index 471e4fdc66..b56a954b78 100755
--- a/test/functional/feature_dip0020_activation.py
+++ b/test/functional/feature_dip0020_activation.py
@@ -69,6 +69,7 @@ class DIP0020ActivationTest(BitcoinTestFramework):
# Should be spendable now
tx0id = self.node.sendrawtransaction(tx0_hex)
assert tx0id in set(self.node.getrawmempool())
+ assert int(tx0id[0], 16) < 4
if __name__ == '__main__':
```
On develop:
```
./test/functional/test_runner.py feature_dip0020_activation.py --retries=100 --sleep=0
```
if this fails on the first run, it keeps "failing" (simply fetching the
same results actually) till the end.
With this patch:
```
./test/functional/test_runner.py feature_dip0020_activation.py --attempts=100
```
if this fails on the first run, it can actually succeed after a few
attempts now, unless you are extremely unlucky ofc 😄
Also, check [ci results in my repo
](https://cdn.artifacts.gitlab-static.net/93/b4/93b4f8b17e5dcccab1afee165b4d74d90f05800caf65d6c48a83a1a78c979587/2023_04_08/4081291268/4478867166/job.log?response-content-type=text%2Fplain%3B%20charset%3Dutf-8&response-content-disposition=inline&Expires=1680945516&KeyName=gprd-artifacts-cdn&Signature=2d4mHCJBbgRaTDiSQ6kKIy1PdIM=).
Note:
```
...
feature_dip3_v19.py failed at attempt 1/3, Duration: 159s
...
4/179 - [1mfeature_dip3_v19.py[0m passed, Duration: 244 s
...
feature_llmq_hpmn.py failed at attempt 1/3, Duration: 284s
...
feature_llmq_hpmn.py failed at attempt 2/3, Duration: 296s
...
11/179 - [1mfeature_llmq_hpmn.py[0m failed, Duration: 233 s
...
```
An example with 2 tests failing initially and then passing:
https://gitlab.com/dashpay/dash/-/jobs/4089689970
## Breaking Changes
n/a
## Checklist:
- [x] I have performed a self-review of my own code
- [ ] I have commented my code, particularly in hard-to-understand areas
- [ ] I have added or updated relevant unit/integration/functional/e2e
tests
- [ ] I have made corresponding changes to the documentation
**For repository code-owners and collaborators only**
- [x] I have assigned this pull request to a milestone
2023-04-15 06:13:47 +02:00
for i in range ( test_count ) :
test_result , testdir , stdout , stderr = job_queue . get_next ( )
test_results . append ( test_result )
done_str = " {} / {} - {} {} {} " . format ( i + 1 , test_count , BOLD [ 1 ] , test_result . name , BOLD [ 0 ] )
2017-04-07 16:20:39 +02:00
if test_result . status == " Passed " :
2021-06-25 00:19:53 +02:00
logging . debug ( " %s passed, Duration: %s s " % ( done_str , test_result . time ) )
2017-04-07 16:20:39 +02:00
elif test_result . status == " Skipped " :
2021-06-25 00:19:53 +02:00
logging . debug ( " %s skipped " % ( done_str ) )
2017-03-28 11:24:14 +02:00
else :
fix(tests): Fix retries for the non-deterministic test suite (#5307)
## Issue being fixed or feature implemented
the problem with retries implemented in #4793 is that they don't really
do anything besides fetching results of a failed job multiple times 🙈
## What was done?
partially reverted changes done in #4793, implemented actual job
restart. dropped `--sleep` and `--retries` and added `--attempts`
instead.
## How Has This Been Tested?
Pick any test, add some randomly failing expression somewhere and run it
with some high number of retries.
For example:
```diff
diff --git a/test/functional/feature_dip0020_activation.py b/test/functional/feature_dip0020_activation.py
index 471e4fdc66..b56a954b78 100755
--- a/test/functional/feature_dip0020_activation.py
+++ b/test/functional/feature_dip0020_activation.py
@@ -69,6 +69,7 @@ class DIP0020ActivationTest(BitcoinTestFramework):
# Should be spendable now
tx0id = self.node.sendrawtransaction(tx0_hex)
assert tx0id in set(self.node.getrawmempool())
+ assert int(tx0id[0], 16) < 4
if __name__ == '__main__':
```
On develop:
```
./test/functional/test_runner.py feature_dip0020_activation.py --retries=100 --sleep=0
```
if this fails on the first run, it keeps "failing" (simply fetching the
same results actually) till the end.
With this patch:
```
./test/functional/test_runner.py feature_dip0020_activation.py --attempts=100
```
if this fails on the first run, it can actually succeed after a few
attempts now, unless you are extremely unlucky ofc 😄
Also, check [ci results in my repo
](https://cdn.artifacts.gitlab-static.net/93/b4/93b4f8b17e5dcccab1afee165b4d74d90f05800caf65d6c48a83a1a78c979587/2023_04_08/4081291268/4478867166/job.log?response-content-type=text%2Fplain%3B%20charset%3Dutf-8&response-content-disposition=inline&Expires=1680945516&KeyName=gprd-artifacts-cdn&Signature=2d4mHCJBbgRaTDiSQ6kKIy1PdIM=).
Note:
```
...
feature_dip3_v19.py failed at attempt 1/3, Duration: 159s
...
4/179 - [1mfeature_dip3_v19.py[0m passed, Duration: 244 s
...
feature_llmq_hpmn.py failed at attempt 1/3, Duration: 284s
...
feature_llmq_hpmn.py failed at attempt 2/3, Duration: 296s
...
11/179 - [1mfeature_llmq_hpmn.py[0m failed, Duration: 233 s
...
```
An example with 2 tests failing initially and then passing:
https://gitlab.com/dashpay/dash/-/jobs/4089689970
## Breaking Changes
n/a
## Checklist:
- [x] I have performed a self-review of my own code
- [ ] I have commented my code, particularly in hard-to-understand areas
- [ ] I have added or updated relevant unit/integration/functional/e2e
tests
- [ ] I have made corresponding changes to the documentation
**For repository code-owners and collaborators only**
- [x] I have assigned this pull request to a milestone
2023-04-15 06:13:47 +02:00
print ( " %s failed, Duration: %s s \n " % ( done_str , test_result . time ) )
print ( BOLD [ 1 ] + ' stdout: \n ' + BOLD [ 0 ] + stdout + ' \n ' )
print ( BOLD [ 1 ] + ' stderr: \n ' + BOLD [ 0 ] + stderr + ' \n ' )
if combined_logs_len and os . path . isdir ( testdir ) :
# Print the final `combinedlogslen` lines of the combined logs
print ( ' {} Combine the logs and print the last {} lines ... {} ' . format ( BOLD [ 1 ] , combined_logs_len , BOLD [ 0 ] ) )
print ( ' \n ============ ' )
print ( ' {} Combined log for {} : {} ' . format ( BOLD [ 1 ] , testdir , BOLD [ 0 ] ) )
print ( ' ============ \n ' )
combined_logs_args = [ sys . executable , os . path . join ( tests_dir , ' combine_logs.py ' ) , testdir ]
if BOLD [ 0 ] :
combined_logs_args + = [ ' --color ' ]
combined_logs , _ = subprocess . Popen ( combined_logs_args , universal_newlines = True , stdout = subprocess . PIPE ) . communicate ( )
print ( " \n " . join ( deque ( combined_logs . splitlines ( ) , combined_logs_len ) ) )
if failfast :
logging . debug ( " Early exiting after test failure " )
2018-04-30 15:19:26 +02:00
break
2018-03-17 16:05:37 +01:00
print_results ( test_results , max_len_name , ( int ( time . time ( ) - start_time ) ) )
2016-04-09 21:14:18 +02:00
if coverage :
2019-05-15 21:49:48 +02:00
coverage_passed = coverage . report_rpc_coverage ( )
2016-04-09 21:14:18 +02:00
2017-03-28 11:24:14 +02:00
logging . debug ( " Cleaning up coverage data " )
2016-04-09 21:14:18 +02:00
coverage . cleanup ( )
2019-05-15 21:49:48 +02:00
else :
coverage_passed = True
2015-10-11 07:41:19 +02:00
2017-05-22 08:59:11 +02:00
# Clear up the temp directory if all subdirectories are gone
if not os . listdir ( tmpdir ) :
os . rmdir ( tmpdir )
2019-05-15 21:49:48 +02:00
all_passed = all ( map ( lambda test_result : test_result . was_successful , test_results ) ) and coverage_passed
2017-04-07 16:20:39 +02:00
2021-06-18 14:22:48 +02:00
# Clean up dangling processes if any. This may only happen with --failfast option.
# Killing the process group will also terminate the current process but that is
# not an issue
if len ( job_queue . jobs ) :
os . killpg ( os . getpgid ( 0 ) , signal . SIGKILL )
2018-04-30 15:19:26 +02:00
2016-05-10 18:27:31 +02:00
sys . exit ( not all_passed )
2017-04-07 16:20:39 +02:00
def print_results ( test_results , max_len_name , runtime ) :
results = " \n " + BOLD [ 1 ] + " %s | %s | %s \n \n " % ( " TEST " . ljust ( max_len_name ) , " STATUS " , " DURATION " ) + BOLD [ 0 ]
2018-03-28 16:00:24 +02:00
test_results . sort ( key = TestResult . sort_key )
2017-04-07 16:20:39 +02:00
all_passed = True
time_sum = 0
for test_result in test_results :
2017-04-12 00:57:20 +02:00
all_passed = all_passed and test_result . was_successful
2017-04-07 16:20:39 +02:00
time_sum + = test_result . time
test_result . padding = max_len_name
results + = str ( test_result )
status = TICK + " Passed " if all_passed else CROSS + " Failed "
2018-03-28 16:00:24 +02:00
if not all_passed :
results + = RED [ 1 ]
2017-04-07 16:20:39 +02:00
results + = BOLD [ 1 ] + " \n %s | %s | %s s (accumulated) \n " % ( " ALL " . ljust ( max_len_name ) , status . ljust ( 9 ) , time_sum ) + BOLD [ 0 ]
2018-03-28 16:00:24 +02:00
if not all_passed :
results + = RED [ 0 ]
2017-04-07 16:20:39 +02:00
results + = " Runtime: %s s \n " % ( runtime )
print ( results )
2019-05-19 22:20:34 +02:00
class TestHandler :
2016-05-10 18:27:31 +02:00
"""
2017-08-16 00:45:58 +02:00
Trigger the test scripts passed in via the list .
2016-05-10 18:27:31 +02:00
"""
2020-03-23 12:37:38 +01:00
def __init__ ( self , * , num_tests_parallel , tests_dir , tmpdir , test_list , flags , use_term_control , attempts ) :
2018-11-02 20:04:55 +01:00
assert num_tests_parallel > = 1
2016-05-10 18:27:31 +02:00
self . num_jobs = num_tests_parallel
2019-01-03 10:18:47 +01:00
self . tests_dir = tests_dir
2017-05-22 08:59:11 +02:00
self . tmpdir = tmpdir
2016-05-10 18:27:31 +02:00
self . test_list = test_list
self . flags = flags
self . num_running = 0
self . jobs = [ ]
2019-08-15 13:42:20 +02:00
self . use_term_control = use_term_control
fix(tests): Fix retries for the non-deterministic test suite (#5307)
## Issue being fixed or feature implemented
the problem with retries implemented in #4793 is that they don't really
do anything besides fetching results of a failed job multiple times 🙈
## What was done?
partially reverted changes done in #4793, implemented actual job
restart. dropped `--sleep` and `--retries` and added `--attempts`
instead.
## How Has This Been Tested?
Pick any test, add some randomly failing expression somewhere and run it
with some high number of retries.
For example:
```diff
diff --git a/test/functional/feature_dip0020_activation.py b/test/functional/feature_dip0020_activation.py
index 471e4fdc66..b56a954b78 100755
--- a/test/functional/feature_dip0020_activation.py
+++ b/test/functional/feature_dip0020_activation.py
@@ -69,6 +69,7 @@ class DIP0020ActivationTest(BitcoinTestFramework):
# Should be spendable now
tx0id = self.node.sendrawtransaction(tx0_hex)
assert tx0id in set(self.node.getrawmempool())
+ assert int(tx0id[0], 16) < 4
if __name__ == '__main__':
```
On develop:
```
./test/functional/test_runner.py feature_dip0020_activation.py --retries=100 --sleep=0
```
if this fails on the first run, it keeps "failing" (simply fetching the
same results actually) till the end.
With this patch:
```
./test/functional/test_runner.py feature_dip0020_activation.py --attempts=100
```
if this fails on the first run, it can actually succeed after a few
attempts now, unless you are extremely unlucky ofc 😄
Also, check [ci results in my repo
](https://cdn.artifacts.gitlab-static.net/93/b4/93b4f8b17e5dcccab1afee165b4d74d90f05800caf65d6c48a83a1a78c979587/2023_04_08/4081291268/4478867166/job.log?response-content-type=text%2Fplain%3B%20charset%3Dutf-8&response-content-disposition=inline&Expires=1680945516&KeyName=gprd-artifacts-cdn&Signature=2d4mHCJBbgRaTDiSQ6kKIy1PdIM=).
Note:
```
...
feature_dip3_v19.py failed at attempt 1/3, Duration: 159s
...
4/179 - [1mfeature_dip3_v19.py[0m passed, Duration: 244 s
...
feature_llmq_hpmn.py failed at attempt 1/3, Duration: 284s
...
feature_llmq_hpmn.py failed at attempt 2/3, Duration: 296s
...
11/179 - [1mfeature_llmq_hpmn.py[0m failed, Duration: 233 s
...
```
An example with 2 tests failing initially and then passing:
https://gitlab.com/dashpay/dash/-/jobs/4089689970
## Breaking Changes
n/a
## Checklist:
- [x] I have performed a self-review of my own code
- [ ] I have commented my code, particularly in hard-to-understand areas
- [ ] I have added or updated relevant unit/integration/functional/e2e
tests
- [ ] I have made corresponding changes to the documentation
**For repository code-owners and collaborators only**
- [x] I have assigned this pull request to a milestone
2023-04-15 06:13:47 +02:00
self . attempts = attempts
2016-05-10 18:27:31 +02:00
def get_next ( self ) :
while self . num_running < self . num_jobs and self . test_list :
# Add tests
self . num_running + = 1
2018-03-17 16:05:37 +01:00
test = self . test_list . pop ( 0 )
2018-06-11 14:33:34 +02:00
portseed = len ( self . test_list )
2017-05-22 08:59:11 +02:00
portseed_arg = [ " --portseed= {} " . format ( portseed ) ]
2016-09-19 16:51:35 +02:00
log_stdout = tempfile . SpooledTemporaryFile ( max_size = 2 * * 16 )
log_stderr = tempfile . SpooledTemporaryFile ( max_size = 2 * * 16 )
2018-03-17 16:05:37 +01:00
test_argv = test . split ( )
2017-11-29 19:21:51 +01:00
testdir = " {} / {} _ {} " . format ( self . tmpdir , re . sub ( " .py$ " , " " , test_argv [ 0 ] ) , portseed )
tmpdir_arg = [ " --tmpdir= {} " . format ( testdir ) ]
2018-03-17 16:05:37 +01:00
self . jobs . append ( ( test ,
2016-05-10 18:27:31 +02:00
time . time ( ) ,
2018-02-12 11:30:17 +01:00
subprocess . Popen ( [ sys . executable , self . tests_dir + test_argv [ 0 ] ] + test_argv [ 1 : ] + self . flags + portseed_arg + tmpdir_arg ,
2016-05-10 18:27:31 +02:00
universal_newlines = True ,
2016-09-19 16:51:35 +02:00
stdout = log_stdout ,
stderr = log_stderr ) ,
2017-11-29 19:21:51 +01:00
testdir ,
2016-09-19 16:51:35 +02:00
log_stdout ,
fix(tests): Fix retries for the non-deterministic test suite (#5307)
## Issue being fixed or feature implemented
the problem with retries implemented in #4793 is that they don't really
do anything besides fetching results of a failed job multiple times 🙈
## What was done?
partially reverted changes done in #4793, implemented actual job
restart. dropped `--sleep` and `--retries` and added `--attempts`
instead.
## How Has This Been Tested?
Pick any test, add some randomly failing expression somewhere and run it
with some high number of retries.
For example:
```diff
diff --git a/test/functional/feature_dip0020_activation.py b/test/functional/feature_dip0020_activation.py
index 471e4fdc66..b56a954b78 100755
--- a/test/functional/feature_dip0020_activation.py
+++ b/test/functional/feature_dip0020_activation.py
@@ -69,6 +69,7 @@ class DIP0020ActivationTest(BitcoinTestFramework):
# Should be spendable now
tx0id = self.node.sendrawtransaction(tx0_hex)
assert tx0id in set(self.node.getrawmempool())
+ assert int(tx0id[0], 16) < 4
if __name__ == '__main__':
```
On develop:
```
./test/functional/test_runner.py feature_dip0020_activation.py --retries=100 --sleep=0
```
if this fails on the first run, it keeps "failing" (simply fetching the
same results actually) till the end.
With this patch:
```
./test/functional/test_runner.py feature_dip0020_activation.py --attempts=100
```
if this fails on the first run, it can actually succeed after a few
attempts now, unless you are extremely unlucky ofc 😄
Also, check [ci results in my repo
](https://cdn.artifacts.gitlab-static.net/93/b4/93b4f8b17e5dcccab1afee165b4d74d90f05800caf65d6c48a83a1a78c979587/2023_04_08/4081291268/4478867166/job.log?response-content-type=text%2Fplain%3B%20charset%3Dutf-8&response-content-disposition=inline&Expires=1680945516&KeyName=gprd-artifacts-cdn&Signature=2d4mHCJBbgRaTDiSQ6kKIy1PdIM=).
Note:
```
...
feature_dip3_v19.py failed at attempt 1/3, Duration: 159s
...
4/179 - [1mfeature_dip3_v19.py[0m passed, Duration: 244 s
...
feature_llmq_hpmn.py failed at attempt 1/3, Duration: 284s
...
feature_llmq_hpmn.py failed at attempt 2/3, Duration: 296s
...
11/179 - [1mfeature_llmq_hpmn.py[0m failed, Duration: 233 s
...
```
An example with 2 tests failing initially and then passing:
https://gitlab.com/dashpay/dash/-/jobs/4089689970
## Breaking Changes
n/a
## Checklist:
- [x] I have performed a self-review of my own code
- [ ] I have commented my code, particularly in hard-to-understand areas
- [ ] I have added or updated relevant unit/integration/functional/e2e
tests
- [ ] I have made corresponding changes to the documentation
**For repository code-owners and collaborators only**
- [x] I have assigned this pull request to a milestone
2023-04-15 06:13:47 +02:00
log_stderr ,
portseed ,
1 ) ) # attempt
2016-05-10 18:27:31 +02:00
if not self . jobs :
2016-06-21 10:24:09 +02:00
raise IndexError ( ' pop from empty list ' )
2019-03-18 16:26:03 +01:00
# Print remaining running jobs when all jobs have been started.
if not self . test_list :
print ( " Remaining jobs: [ {} ] " . format ( " , " . join ( j [ 0 ] for j in self . jobs ) ) )
2021-06-25 00:19:53 +02:00
dot_count = 0
2016-05-10 18:27:31 +02:00
while True :
# Return first proc that finishes
time . sleep ( .5 )
2018-03-17 16:05:37 +01:00
for job in self . jobs :
fix(tests): Fix retries for the non-deterministic test suite (#5307)
## Issue being fixed or feature implemented
the problem with retries implemented in #4793 is that they don't really
do anything besides fetching results of a failed job multiple times 🙈
## What was done?
partially reverted changes done in #4793, implemented actual job
restart. dropped `--sleep` and `--retries` and added `--attempts`
instead.
## How Has This Been Tested?
Pick any test, add some randomly failing expression somewhere and run it
with some high number of retries.
For example:
```diff
diff --git a/test/functional/feature_dip0020_activation.py b/test/functional/feature_dip0020_activation.py
index 471e4fdc66..b56a954b78 100755
--- a/test/functional/feature_dip0020_activation.py
+++ b/test/functional/feature_dip0020_activation.py
@@ -69,6 +69,7 @@ class DIP0020ActivationTest(BitcoinTestFramework):
# Should be spendable now
tx0id = self.node.sendrawtransaction(tx0_hex)
assert tx0id in set(self.node.getrawmempool())
+ assert int(tx0id[0], 16) < 4
if __name__ == '__main__':
```
On develop:
```
./test/functional/test_runner.py feature_dip0020_activation.py --retries=100 --sleep=0
```
if this fails on the first run, it keeps "failing" (simply fetching the
same results actually) till the end.
With this patch:
```
./test/functional/test_runner.py feature_dip0020_activation.py --attempts=100
```
if this fails on the first run, it can actually succeed after a few
attempts now, unless you are extremely unlucky ofc 😄
Also, check [ci results in my repo
](https://cdn.artifacts.gitlab-static.net/93/b4/93b4f8b17e5dcccab1afee165b4d74d90f05800caf65d6c48a83a1a78c979587/2023_04_08/4081291268/4478867166/job.log?response-content-type=text%2Fplain%3B%20charset%3Dutf-8&response-content-disposition=inline&Expires=1680945516&KeyName=gprd-artifacts-cdn&Signature=2d4mHCJBbgRaTDiSQ6kKIy1PdIM=).
Note:
```
...
feature_dip3_v19.py failed at attempt 1/3, Duration: 159s
...
4/179 - [1mfeature_dip3_v19.py[0m passed, Duration: 244 s
...
feature_llmq_hpmn.py failed at attempt 1/3, Duration: 284s
...
feature_llmq_hpmn.py failed at attempt 2/3, Duration: 296s
...
11/179 - [1mfeature_llmq_hpmn.py[0m failed, Duration: 233 s
...
```
An example with 2 tests failing initially and then passing:
https://gitlab.com/dashpay/dash/-/jobs/4089689970
## Breaking Changes
n/a
## Checklist:
- [x] I have performed a self-review of my own code
- [ ] I have commented my code, particularly in hard-to-understand areas
- [ ] I have added or updated relevant unit/integration/functional/e2e
tests
- [ ] I have made corresponding changes to the documentation
**For repository code-owners and collaborators only**
- [x] I have assigned this pull request to a milestone
2023-04-15 06:13:47 +02:00
( name , start_time , proc , testdir , log_out , log_err , portseed , attempt ) = job
2016-05-10 18:27:31 +02:00
if proc . poll ( ) is not None :
2016-09-19 16:51:35 +02:00
log_out . seek ( 0 ) , log_err . seek ( 0 )
2018-03-19 14:54:15 +01:00
[ stdout , stderr ] = [ log_file . read ( ) . decode ( ' utf-8 ' ) for log_file in ( log_out , log_err ) ]
2016-09-19 16:51:35 +02:00
log_out . close ( ) , log_err . close ( )
2017-03-24 13:57:36 +01:00
if proc . returncode == TEST_EXIT_PASSED and stderr == " " :
status = " Passed "
elif proc . returncode == TEST_EXIT_SKIPPED :
status = " Skipped "
fix(tests): Fix retries for the non-deterministic test suite (#5307)
## Issue being fixed or feature implemented
the problem with retries implemented in #4793 is that they don't really
do anything besides fetching results of a failed job multiple times 🙈
## What was done?
partially reverted changes done in #4793, implemented actual job
restart. dropped `--sleep` and `--retries` and added `--attempts`
instead.
## How Has This Been Tested?
Pick any test, add some randomly failing expression somewhere and run it
with some high number of retries.
For example:
```diff
diff --git a/test/functional/feature_dip0020_activation.py b/test/functional/feature_dip0020_activation.py
index 471e4fdc66..b56a954b78 100755
--- a/test/functional/feature_dip0020_activation.py
+++ b/test/functional/feature_dip0020_activation.py
@@ -69,6 +69,7 @@ class DIP0020ActivationTest(BitcoinTestFramework):
# Should be spendable now
tx0id = self.node.sendrawtransaction(tx0_hex)
assert tx0id in set(self.node.getrawmempool())
+ assert int(tx0id[0], 16) < 4
if __name__ == '__main__':
```
On develop:
```
./test/functional/test_runner.py feature_dip0020_activation.py --retries=100 --sleep=0
```
if this fails on the first run, it keeps "failing" (simply fetching the
same results actually) till the end.
With this patch:
```
./test/functional/test_runner.py feature_dip0020_activation.py --attempts=100
```
if this fails on the first run, it can actually succeed after a few
attempts now, unless you are extremely unlucky ofc 😄
Also, check [ci results in my repo
](https://cdn.artifacts.gitlab-static.net/93/b4/93b4f8b17e5dcccab1afee165b4d74d90f05800caf65d6c48a83a1a78c979587/2023_04_08/4081291268/4478867166/job.log?response-content-type=text%2Fplain%3B%20charset%3Dutf-8&response-content-disposition=inline&Expires=1680945516&KeyName=gprd-artifacts-cdn&Signature=2d4mHCJBbgRaTDiSQ6kKIy1PdIM=).
Note:
```
...
feature_dip3_v19.py failed at attempt 1/3, Duration: 159s
...
4/179 - [1mfeature_dip3_v19.py[0m passed, Duration: 244 s
...
feature_llmq_hpmn.py failed at attempt 1/3, Duration: 284s
...
feature_llmq_hpmn.py failed at attempt 2/3, Duration: 296s
...
11/179 - [1mfeature_llmq_hpmn.py[0m failed, Duration: 233 s
...
```
An example with 2 tests failing initially and then passing:
https://gitlab.com/dashpay/dash/-/jobs/4089689970
## Breaking Changes
n/a
## Checklist:
- [x] I have performed a self-review of my own code
- [ ] I have commented my code, particularly in hard-to-understand areas
- [ ] I have added or updated relevant unit/integration/functional/e2e
tests
- [ ] I have made corresponding changes to the documentation
**For repository code-owners and collaborators only**
- [x] I have assigned this pull request to a milestone
2023-04-15 06:13:47 +02:00
elif attempt < self . attempts :
# cleanup
if self . use_term_control :
clearline = ' \r ' + ( ' ' * dot_count ) + ' \r '
print ( clearline , end = ' ' , flush = True )
dot_count = 0
shutil . rmtree ( testdir , ignore_errors = True )
self . jobs . remove ( job )
print ( f " { name } failed at attempt { attempt } / { self . attempts } , Duration: { int ( time . time ( ) - start_time ) } s " )
# start over
portseed_arg = [ " --portseed= {} " . format ( portseed ) ]
log_stdout = tempfile . SpooledTemporaryFile ( max_size = 2 * * 16 )
log_stderr = tempfile . SpooledTemporaryFile ( max_size = 2 * * 16 )
test_argv = name . split ( )
tmpdir_arg = [ " --tmpdir= {} " . format ( testdir ) ]
self . jobs . append ( ( name ,
time . time ( ) ,
subprocess . Popen ( [ sys . executable , self . tests_dir + test_argv [ 0 ] ] + test_argv [ 1 : ] + self . flags + portseed_arg + tmpdir_arg ,
universal_newlines = True ,
stdout = log_stdout ,
stderr = log_stderr ) ,
testdir ,
log_stdout ,
log_stderr ,
portseed ,
attempt + 1 ) ) # attempt
# no results for now, move to the next job
continue
2017-03-24 13:57:36 +01:00
else :
status = " Failed "
2016-05-10 18:27:31 +02:00
self . num_running - = 1
2018-03-17 16:05:37 +01:00
self . jobs . remove ( job )
2019-08-15 13:42:20 +02:00
if self . use_term_control :
clearline = ' \r ' + ( ' ' * dot_count ) + ' \r '
print ( clearline , end = ' ' , flush = True )
2018-11-01 15:42:10 +01:00
dot_count = 0
2018-03-17 16:05:37 +01:00
return TestResult ( name , status , int ( time . time ( ) - start_time ) ) , testdir , stdout , stderr
2019-08-15 13:42:20 +02:00
if self . use_term_control :
print ( ' . ' , end = ' ' , flush = True )
2018-11-01 15:42:10 +01:00
dot_count + = 1
2016-05-10 18:27:31 +02:00
2018-04-30 15:19:26 +02:00
2017-04-07 16:20:39 +02:00
class TestResult ( ) :
def __init__ ( self , name , status , time ) :
self . name = name
self . status = status
self . time = time
self . padding = 0
2018-03-28 16:00:24 +02:00
def sort_key ( self ) :
if self . status == " Passed " :
return 0 , self . name . lower ( )
elif self . status == " Failed " :
return 2 , self . name . lower ( )
elif self . status == " Skipped " :
return 1 , self . name . lower ( )
2017-04-07 16:20:39 +02:00
def __repr__ ( self ) :
if self . status == " Passed " :
2018-09-24 22:10:13 +02:00
color = GREEN
2017-04-07 16:20:39 +02:00
glyph = TICK
elif self . status == " Failed " :
color = RED
glyph = CROSS
elif self . status == " Skipped " :
2022-05-10 13:12:05 +02:00
color = DEFAULT
2017-04-07 16:20:39 +02:00
glyph = CIRCLE
return color [ 1 ] + " %s | %s %s | %s s \n " % ( self . name . ljust ( self . padding ) , glyph , self . status . ljust ( 7 ) , self . time ) + color [ 0 ]
2017-04-12 00:57:20 +02:00
@property
def was_successful ( self ) :
return self . status != " Failed "
2017-04-07 16:20:39 +02:00
2020-07-17 01:44:20 +02:00
def check_script_prefixes ( ) :
2018-01-30 14:01:44 +01:00
""" Check that test scripts start with one of the allowed name prefixes. """
2020-07-17 01:44:20 +02:00
2020-03-26 21:57:53 +01:00
good_prefixes_re = re . compile ( " ^(example|feature|interface|mempool|mining|p2p|rpc|wallet|tool)_ " )
2020-07-17 01:44:20 +02:00
bad_script_names = [ script for script in ALL_SCRIPTS if good_prefixes_re . match ( script ) is None ]
2018-01-30 14:01:44 +01:00
if bad_script_names :
print ( " %s ERROR: %s %d tests not meeting naming conventions: " % ( BOLD [ 1 ] , BOLD [ 0 ] , len ( bad_script_names ) ) )
2020-07-17 01:44:20 +02:00
print ( " %s " % ( " \n " . join ( sorted ( bad_script_names ) ) ) )
2018-01-30 14:01:44 +01:00
raise AssertionError ( " Some tests are not following naming convention! " )
2020-07-17 01:44:20 +02:00
2018-11-02 20:04:55 +01:00
def check_script_list ( * , src_dir , fail_on_warn ) :
2017-03-28 11:36:29 +02:00
""" Check scripts directory.
Check that there are no scripts in the functional tests directory which are
not being run by pull - tester . py . """
script_dir = src_dir + ' /test/functional/ '
2018-03-19 14:54:15 +01:00
python_files = set ( [ test_file for test_file in os . listdir ( script_dir ) if test_file . endswith ( " .py " ) ] )
2017-03-28 11:36:29 +02:00
missed_tests = list ( python_files - set ( map ( lambda x : x . split ( ) [ 0 ] , ALL_SCRIPTS + NON_SCRIPTS ) ) )
if len ( missed_tests ) != 0 :
2017-04-17 22:15:12 +02:00
print ( " %s WARNING! %s The following scripts are not being run: %s . Check the test lists in test_runner.py. " % ( BOLD [ 1 ] , BOLD [ 0 ] , str ( missed_tests ) ) )
2018-11-02 20:04:55 +01:00
if fail_on_warn :
2019-06-19 21:51:41 +02:00
# On CI this warning is an error to prevent merging incomplete commits into master
2017-04-17 22:15:12 +02:00
sys . exit ( 1 )
2015-10-11 07:41:19 +02:00
2018-11-02 20:04:55 +01:00
2017-10-17 21:03:09 +02:00
class RPCCoverage ( ) :
2015-10-11 07:41:19 +02:00
"""
2019-05-19 22:20:34 +02:00
Coverage reporting utilities for test_runner .
2015-10-11 07:41:19 +02:00
Coverage calculation works by having each test script subprocess write
coverage files into a particular directory . These files contain the RPC
commands invoked during testing , as well as a complete listing of RPC
2020-06-11 10:39:04 +02:00
commands per ` dash - cli help ` ( ` rpc_interface . txt ` ) .
2015-10-11 07:41:19 +02:00
After all tests complete , the commands run are combined and diff ' d against
the complete list to calculate uncovered RPC commands .
2019-05-19 22:20:34 +02:00
See also : test / functional / test_framework / coverage . py
2015-10-11 07:41:19 +02:00
"""
def __init__ ( self ) :
self . dir = tempfile . mkdtemp ( prefix = " coverage " )
2016-05-10 18:27:31 +02:00
self . flag = ' --coveragedir= %s ' % self . dir
2015-10-11 07:41:19 +02:00
def report_rpc_coverage ( self ) :
"""
Print out RPC commands that were unexercised by tests .
"""
uncovered = self . _get_uncovered_rpc_commands ( )
if uncovered :
print ( " Uncovered RPC commands: " )
2018-03-17 16:05:37 +01:00
print ( " " . join ( ( " - %s \n " % command ) for command in sorted ( uncovered ) ) )
2019-05-15 21:49:48 +02:00
return False
2015-10-11 07:41:19 +02:00
else :
print ( " All RPC commands covered. " )
2019-05-15 21:49:48 +02:00
return True
2015-10-11 07:41:19 +02:00
def cleanup ( self ) :
return shutil . rmtree ( self . dir )
def _get_uncovered_rpc_commands ( self ) :
"""
Return a set of currently untested RPC commands .
"""
2019-05-19 22:20:34 +02:00
# This is shared from `test/functional/test-framework/coverage.py`
2019-01-03 10:18:47 +01:00
reference_filename = ' rpc_interface.txt '
coverage_file_prefix = ' coverage. '
2015-10-11 07:41:19 +02:00
2019-01-03 10:18:47 +01:00
coverage_ref_filename = os . path . join ( self . dir , reference_filename )
2015-10-11 07:41:19 +02:00
coverage_filenames = set ( )
all_cmds = set ( )
covered_cmds = set ( )
if not os . path . isfile ( coverage_ref_filename ) :
raise RuntimeError ( " No coverage reference found " )
2018-03-19 14:54:15 +01:00
with open ( coverage_ref_filename , ' r ' , encoding = " utf8 " ) as coverage_ref_file :
all_cmds . update ( [ line . strip ( ) for line in coverage_ref_file . readlines ( ) ] )
2015-10-11 07:41:19 +02:00
for root , dirs , files in os . walk ( self . dir ) :
for filename in files :
2019-01-03 10:18:47 +01:00
if filename . startswith ( coverage_file_prefix ) :
2015-10-11 07:41:19 +02:00
coverage_filenames . add ( os . path . join ( root , filename ) )
for filename in coverage_filenames :
2018-03-19 14:54:15 +01:00
with open ( filename , ' r ' , encoding = " utf8 " ) as coverage_file :
covered_cmds . update ( [ line . strip ( ) for line in coverage_file . readlines ( ) ] )
2015-10-11 07:41:19 +02:00
return all_cmds - covered_cmds
if __name__ == ' __main__ ' :
2019-01-03 10:18:47 +01:00
main ( )