Merge bitcoin#9956: Reorganise qa directory (#2912)

* Merge #9956: Reorganise qa directory

63d66ba Move src/test/bitcoin-util-test.py to test/util/bitcoin-util-test.py (John Newbery)
5b0bff4 Rename --enable-extended-rpc-tests to --enable-extended-functional-tests (John Newbery)
a9bd622 Rename test/pull-tester/rpc-tests.py to test/functional/test_runner.py (John Newbery)
c28ee91 Rename rpc-tests directory to functional (John Newbery)
00902c4 Rename qa directory to test (John Newbery)

Tree-SHA512: ee7125c0c647d81590177beef2c8852c4ef76fdcf888096d9d4d360562a01d8d3b453345c3040487b2a043935bd1e7e80018f34462d6e02262bedbe23edcc576

resolve build errors

Signed-off-by: Pasta <Pasta@dash.org>

update test_runner.py in testintegrations.sh

Signed-off-by: Pasta <Pasta@dash.org>

* moved dash specific tests

Signed-off-by: Pasta <Pasta@dash.org>

* dashify README.md

Signed-off-by: Pasta <Pasta@dash.org>

* removed autoix*.py

Signed-off-by: Pasta <Pasta@dash.org>

* change back file perms

* dedashify

Signed-off-by: Pasta <Pasta@dash.org>
This commit is contained in:
PastaPastaPasta 2019-05-19 15:20:34 -05:00 committed by UdjinM6
parent 89d8f7a37d
commit 6edbc9cebd
160 changed files with 239 additions and 245 deletions

6
.gitignore vendored
View File

@ -108,9 +108,9 @@ coverage_percent.txt
linux-coverage-build
linux-build
win32-build
qa/pull-tester/tests_config.py
qa/pull-tester/tests_config.ini
qa/cache/*
test/functional/config.ini
test/util/buildenv.py
test/cache/*
!src/leveldb*/Makefile

View File

@ -61,7 +61,7 @@ OSX_PACKAGING = $(OSX_DEPLOY_SCRIPT) $(OSX_FANCY_PLIST) $(OSX_INSTALLER_ICONS) \
COVERAGE_INFO = baseline_filtered_combined.info baseline.info \
leveldb_baseline.info test_dash_filtered.info total_coverage.info \
baseline_filtered.info rpc_test.info rpc_test_filtered.info \
baseline_filtered.info functional_test.info functional_test_filtered.info \
leveldb_baseline_filtered.info test_dash_coverage.info test_dash.info
dist-hook:
@ -191,20 +191,20 @@ test_dash.info: baseline_filtered_combined.info
test_dash_filtered.info: test_dash.info
$(LCOV) -r $< "/usr/include/*" -o $@
rpc_test.info: test_dash_filtered.info
-@TIMEOUT=15 python qa/pull-tester/rpc-tests.py $(EXTENDED_RPC_TESTS)
$(LCOV) -c -d $(abs_builddir)/src --t rpc-tests -o $@
functional_test.info: test_dash_filtered.info
-@TIMEOUT=15 python test/functional/test_runner.py $(EXTENDED_FUNCTIONAL_TESTS)
$(LCOV) -c -d $(abs_builddir)/src --t functional-tests -o $@
$(LCOV) -z -d $(abs_builddir)/src
$(LCOV) -z -d $(abs_builddir)/src/leveldb
rpc_test_filtered.info: rpc_test.info
functional_test_filtered.info: functional_test.info
$(LCOV) -r $< "/usr/include/*" -o $@
test_dash_coverage.info: baseline_filtered_combined.info test_dash_filtered.info
$(LCOV) -a baseline_filtered.info -a leveldb_baseline_filtered.info -a test_dash_filtered.info -o $@
total_coverage.info: baseline_filtered_combined.info test_dash_filtered.info rpc_test_filtered.info
$(LCOV) -a baseline_filtered.info -a leveldb_baseline_filtered.info -a test_dash_filtered.info -a rpc_test_filtered.info -o $@ | $(GREP) "\%" | $(AWK) '{ print substr($$3,2,50) "/" $$5 }' > coverage_percent.txt
total_coverage.info: baseline_filtered_combined.info test_dash_filtered.info functional_test_filtered.info
$(LCOV) -a baseline_filtered.info -a leveldb_baseline_filtered.info -a test_dash_filtered.info -a functional_test_filtered.info -o $@ | $(GREP) "\%" | $(AWK) '{ print substr($$3,2,50) "/" $$5 }' > coverage_percent.txt
test_dash.coverage/.dirstamp: test_dash_coverage.info
$(GENHTML) -s $< -o $(@D)
@ -220,14 +220,58 @@ endif
dist_noinst_SCRIPTS = autogen.sh
EXTRA_DIST = $(top_srcdir)/share/genbuild.sh qa/pull-tester/rpc-tests.py qa/rpc-tests $(DIST_CONTRIB) $(DIST_DOCS) $(WINDOWS_PACKAGING) $(OSX_PACKAGING) $(BIN_CHECKS)
EXTRA_DIST = $(top_srcdir)/share/genbuild.sh test/functional/test_runner.py test/functional $(DIST_CONTRIB) $(DIST_DOCS) $(WINDOWS_PACKAGING) $(OSX_PACKAGING) $(BIN_CHECKS)
EXTRA_DIST += \
test/util/bctest.py \
test/util/bitcoin-util-test.py \
test/util/data/bitcoin-util-test.json \
test/util/data/blanktxv1.hex \
test/util/data/blanktxv1.json \
test/util/data/blanktxv2.hex \
test/util/data/blanktxv2.json \
test/util/data/tt-delin1-out.hex \
test/util/data/tt-delin1-out.json \
test/util/data/tt-delout1-out.hex \
test/util/data/tt-delout1-out.json \
test/util/data/tt-locktime317000-out.hex \
test/util/data/tt-locktime317000-out.json \
test/util/data/tx394b54bb.hex \
test/util/data/txcreate1.hex \
test/util/data/txcreate1.json \
test/util/data/txcreate2.hex \
test/util/data/txcreate2.json \
test/util/data/txcreatedata1.hex \
test/util/data/txcreatedata1.json \
test/util/data/txcreatedata2.hex \
test/util/data/txcreatedata2.json \
test/util/data/txcreatedata_seq0.hex \
test/util/data/txcreatedata_seq0.json \
test/util/data/txcreatedata_seq1.hex \
test/util/data/txcreatedata_seq1.json \
test/util/data/txcreatemultisig1.hex \
test/util/data/txcreatemultisig1.json \
test/util/data/txcreatemultisig2.hex \
test/util/data/txcreatemultisig2.json \
test/util/data/txcreateoutpubkey1.hex \
test/util/data/txcreateoutpubkey1.json \
test/util/data/txcreatescript1.hex \
test/util/data/txcreatescript1.json \
test/util/data/txcreatescript2.hex \
test/util/data/txcreatescript2.json \
test/util/data/txcreatesignv1.hex \
test/util/data/txcreatesignv1.json \
test/util/data/txcreatesignv2.hex
CLEANFILES = $(OSX_DMG) $(BITCOIN_WIN_INSTALLER)
# This file is problematic for out-of-tree builds if it exists.
DISTCLEANFILES = test/util/buildenv.pyc
.INTERMEDIATE: $(COVERAGE_INFO)
DISTCHECK_CONFIGURE_FLAGS = --enable-man
clean-local:
rm -rf coverage_percent.txt test_dash.coverage/ total.coverage/ qa/tmp/ cache/ $(OSX_APP)
rm -rf qa/pull-tester/__pycache__
rm -rf coverage_percent.txt test_dash.coverage/ total.coverage/ test/tmp/ cache/ $(OSX_APP)
rm -rf test/functional/__pycache__

View File

@ -49,9 +49,9 @@ submit new unit tests for old code. Unit tests can be compiled and run
(assuming they weren't disabled in configure) with: `make check`. Further details on running
and extending unit tests can be found in [/src/test/README.md](/src/test/README.md).
There are also [regression and integration tests](/qa) of the RPC interface, written
There are also [regression and integration tests](/test), written
in Python, that are run automatically on the build server.
These tests can be run (if the [test dependencies](/qa) are installed) with: `qa/pull-tester/rpc-tests.py`
These tests can be run (if the [test dependencies](/test) are installed) with: `test/functional/test_runner.py`
The Travis CI system makes sure that every pull request is built for Windows, Linux, and OS X, and that unit/sanity tests are run automatically.

View File

@ -17,4 +17,4 @@ export LD_LIBRARY_PATH=$BUILD_DIR/depends/$HOST/lib
cd build-ci/dashcore-$BUILD_TARGET
./qa/pull-tester/rpc-tests.py --coverage $PASS_ARGS
./test/functional/test_runner.py --coverage $PASS_ARGS

View File

@ -124,10 +124,10 @@ AC_ARG_ENABLE(bench,
[use_bench=$enableval],
[use_bench=yes])
AC_ARG_ENABLE([extended-rpc-tests],
AS_HELP_STRING([--enable-extended-rpc-tests],[enable expensive RPC tests when using lcov (default no)]),
[use_extended_rpc_tests=$enableval],
[use_extended_rpc_tests=no])
AC_ARG_ENABLE([extended-functional-tests],
AS_HELP_STRING([--enable-extended-functional-tests],[enable expensive functional tests when using lcov (default no)]),
[use_extended_functional_tests=$enableval],
[use_extended_functional_tests=no])
AC_ARG_WITH([qrencode],
[AS_HELP_STRING([--with-qrencode],
@ -459,8 +459,8 @@ if test x$use_pkgconfig = xyes; then
])
fi
if test x$use_extended_rpc_tests != xno; then
AC_SUBST(EXTENDED_RPC_TESTS, -extended)
if test x$use_extended_functional_tests != xno; then
AC_SUBST(EXTENDED_FUNCTIONAL_TESTS, --extended)
fi
if test x$use_lcov = xyes; then
@ -1200,10 +1200,12 @@ AC_SUBST(EVENT_PTHREADS_LIBS)
AC_SUBST(ZMQ_LIBS)
AC_SUBST(PROTOBUF_LIBS)
AC_SUBST(QR_LIBS)
AC_CONFIG_FILES([Makefile src/Makefile doc/man/Makefile share/setup.nsi share/qt/Info.plist src/test/buildenv.py])
AC_CONFIG_FILES([qa/pull-tester/tests_config.ini],[chmod +x qa/pull-tester/tests_config.ini])
AC_CONFIG_FILES([Makefile src/Makefile doc/man/Makefile share/setup.nsi share/qt/Info.plist test/functional/config.ini])
AC_CONFIG_FILES([test/util/buildenv.py],[chmod +x test/util/buildenv.py])
AC_CONFIG_FILES([contrib/devtools/split-debug.sh],[chmod +x contrib/devtools/split-debug.sh])
AC_CONFIG_LINKS([qa/pull-tester/rpc-tests.py:qa/pull-tester/rpc-tests.py])
AC_CONFIG_LINKS([test/functional/test_runner.py:test/functional/test_runner.py])
AC_CONFIG_LINKS([test/util/bitcoin-util-test.py:test/util/bitcoin-util-test.py])
AC_CONFIG_LINKS([test/util/bctest.py:test/util/bctest.py])
dnl boost's m4 checks do something really nasty: they export these vars. As a
dnl result, they leak into secp256k1's configure and crazy things happen.
@ -1251,8 +1253,8 @@ esac
dnl Replace the BUILDDIR path with the correct Windows path if compiling on Native Windows
case ${OS} in
*Windows*)
sed 's/BUILDDIR="\/\([[a-z]]\)/BUILDDIR="\1:/' qa/pull-tester/tests_config.py > qa/pull-tester/tests_config-2.py
mv qa/pull-tester/tests_config-2.py qa/pull-tester/tests_config.py
sed 's/BUILDDIR="\/\([[a-z]]\)/BUILDDIR="\1:/' test/functional/config.ini > test/functional/config-2.ini
mv test/functional/config-2.ini test/functional/config.ini
;;
esac

View File

@ -33,7 +33,7 @@ EXCLUDE = [
'src/tinyformat.h',
'src/leveldb/util/env_win.cc',
'src/crypto/ctaes/bench.c',
'qa/rpc-tests/test_framework/bignum.py',
'test/functional/test_framework/bignum.py',
# python init:
'*__init__.py',
]

View File

@ -133,7 +133,7 @@ Run with the -testnet option to run with "play coins" on the test network, if yo
are testing multi-machine code that needs to operate across the internet.
If you are testing something that can run on one machine, run with the -regtest option.
In regression test mode, blocks can be created on-demand; see qa/rpc-tests/ for tests
In regression test mode, blocks can be created on-demand; see test/functional/ for tests
that run in -regtest mode.
**DEBUG_LOCKORDER**
@ -255,7 +255,7 @@ Wallet
- *Rationale*: In RPC code that conditionally uses the wallet (such as
`validateaddress`) it is easy to forget that global pointer `pwalletMain`
can be NULL. See `qa/rpc-tests/disablewallet.py` for functional tests
can be NULL. See `test/functional/disablewallet.py` for functional tests
exercising the API with `-disablewallet`
- Include `db_cxx.h` (BerkeleyDB header) only when `ENABLE_WALLET` is set

View File

@ -1,87 +0,0 @@
The [pull-tester](/qa/pull-tester/) folder contains a script to call
multiple tests from the [rpc-tests](/qa/rpc-tests/) folder.
Every pull request to the Dash Core repository is built and run through
the regression test suite. You can also run all or only individual
tests locally.
Test dependencies
=================
Before running the tests, the following must be installed.
Unix
----
The python3-zmq library is required. On Ubuntu or Debian it can be installed via:
```
sudo apt-get install python3-zmq
```
OS X
------
```
pip3 install pyzmq
```
Running tests
=============
You can run any single test by calling
qa/pull-tester/rpc-tests.py <testname>
Or you can run any combination of tests by calling
qa/pull-tester/rpc-tests.py <testname1> <testname2> <testname3> ...
Run the regression test suite with
qa/pull-tester/rpc-tests.py
Run all possible tests with
qa/pull-tester/rpc-tests.py --extended
By default, tests will be run in parallel. To specify how many jobs to run,
append `--jobs=n` (default n=4).
If you want to create a basic coverage report for the RPC test suite, append `--coverage`.
Possible options, which apply to each individual test run:
```
-h, --help show this help message and exit
--nocleanup Leave dashds and test.* datadir on exit or error
--noshutdown Don't stop dashds after the test execution
--srcdir=SRCDIR Source directory containing dashd/dash-cli
(default: ../../src)
--tmpdir=TMPDIR Root directory for datadirs
--tracerpc Print out all RPC calls as they are made
--coveragedir=COVERAGEDIR
Write tested RPC commands into this directory
```
If you set the environment variable `PYTHON_DEBUG=1` you will get some debug
output (example: `PYTHON_DEBUG=1 qa/pull-tester/rpc-tests.py wallet`).
A 200-block -regtest blockchain and wallets for four nodes
is created the first time a regression test is run and
is stored in the cache/ directory. Each node has 25 mature
blocks (25*500=12500 DASH) in its wallet.
After the first run, the cache/ blockchain and wallets are
copied into a temporary directory and used as the initial
test state.
If you get into a bad state, you should be able
to recover with:
```bash
rm -rf cache
killall dashd
```
Writing tests
=============
You are encouraged to write tests for new or existing features.
Further information about the test framework and individual RPC
tests is found in [qa/rpc-tests](/qa/rpc-tests).

View File

@ -8,48 +8,6 @@ bin_PROGRAMS += test/test_dash
TEST_SRCDIR = test
TEST_BINARY=test/test_dash$(EXEEXT)
EXTRA_DIST += \
test/bctest.py \
test/bitcoin-util-test.py \
test/data/bitcoin-util-test.json \
test/data/blanktxv1.hex \
test/data/blanktxv1.json \
test/data/blanktxv2.hex \
test/data/blanktxv2.json \
test/data/tt-delin1-out.hex \
test/data/tt-delin1-out.json \
test/data/tt-delout1-out.hex \
test/data/tt-delout1-out.json \
test/data/tt-locktime317000-out.hex \
test/data/tt-locktime317000-out.json \
test/data/tx394b54bb.hex \
test/data/txcreate1.hex \
test/data/txcreate1.json \
test/data/txcreate2.hex \
test/data/txcreate2.json \
test/data/txcreatedata1.hex \
test/data/txcreatedata1.json \
test/data/txcreatedata2.hex \
test/data/txcreatedata2.json \
test/data/txcreatedata_seq0.hex \
test/data/txcreatedata_seq0.json \
test/data/txcreatedata_seq1.hex \
test/data/txcreatedata_seq1.json \
test/data/txcreatemultisig1.hex \
test/data/txcreatemultisig1.json \
test/data/txcreatemultisig2.hex \
test/data/txcreatemultisig2.json \
test/data/txcreateoutpubkey1.hex \
test/data/txcreateoutpubkey1.json \
test/data/txcreatescript1.hex \
test/data/txcreatescript1.json \
test/data/txcreatescript2.hex \
test/data/txcreatescript2.json \
test/data/txcreatesignv1.hex \
test/data/txcreatesignv1.json \
test/data/txcreatesignv2.hex
JSON_TEST_FILES = \
test/data/script_tests.json \
test/data/base58_keys_valid.json \
@ -174,9 +132,6 @@ CLEAN_BITCOIN_TEST = test/*.gcda test/*.gcno $(GENERATED_TEST_FILES)
CLEANFILES += $(CLEAN_BITCOIN_TEST)
# This file is problematic for out-of-tree builds if it exists.
DISTCLEANFILES += test/buildenv.pyc
dash_test: $(TEST_BINARY)
dash_test_check: $(TEST_BINARY) FORCE
@ -186,8 +141,8 @@ dash_test_clean : FORCE
rm -f $(CLEAN_BITCOIN_TEST) $(test_test_dash_OBJECTS) $(TEST_BINARY)
check-local:
@echo "Running test/bitcoin-util-test.py..."
$(AM_V_at)srcdir=$(srcdir) PYTHONPATH=$(builddir)/test $(PYTHON) $(srcdir)/test/bitcoin-util-test.py
@echo "Running test/util/bitcoin-util-test.py..."
$(PYTHON) $(top_builddir)/test/util/bitcoin-util-test.py
$(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C secp256k1 check
if EMBEDDED_UNIVALUE
$(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C univalue check

View File

@ -50,12 +50,3 @@ examine `uint256_tests.cpp`.
For further reading, I found the following website to be helpful in
explaining how the boost unit test framework works:
[http://www.alittlemadness.com/2009/03/31/c-unit-testing-with-boosttest/](http://www.alittlemadness.com/2009/03/31/c-unit-testing-with-boosttest/).
### bitcoin-util-test.py
The test directory also contains the bitcoin-util-test.py tool, which tests bitcoin utils (currently just dash-tx). This test gets run automatically during the `make check` build process. It is also possible to run the test manually from the src directory:
```
test/bitcoin-util-test.py --srcdir=[current directory]
```

View File

@ -1,45 +0,0 @@
#!/usr/bin/env python
# Copyright 2014 BitPay Inc.
# Copyright 2016 The Bitcoin Core developers
# Distributed under the MIT software license, see the accompanying
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
from __future__ import division,print_function,unicode_literals
import os
import bctest
import buildenv
import argparse
import logging
help_text="""Test framework for bitcoin utils.
Runs automatically during `make check`.
Can also be run manually from the src directory by specifying the source directory:
test/bitcoin-util-test.py --srcdir='srcdir' [--verbose]
"""
if __name__ == '__main__':
# Try to get the source directory from the environment variables. This will
# be set for `make check` automated runs. If environment variable is not set,
# then get the source directory from command line args.
try:
srcdir = os.environ["srcdir"]
verbose = False
except:
parser = argparse.ArgumentParser(description=help_text)
parser.add_argument('-s', '--srcdir')
parser.add_argument('-v', '--verbose', action='store_true')
args = parser.parse_args()
srcdir = args.srcdir
verbose = args.verbose
if verbose:
level = logging.DEBUG
else:
level = logging.ERROR
formatter = '%(asctime)s - %(levelname)s - %(message)s'
# Add the format/level to the logger
logging.basicConfig(format = formatter, level=level)
bctest.bctester(srcdir + "/test/data", "bitcoin-util-test.json", buildenv)

View File

@ -1,2 +0,0 @@
#!/usr/bin/env python
exeext="@EXEEXT@"

97
test/README.md Normal file
View File

@ -0,0 +1,97 @@
This directory contains integration tests that test dashd and its
utilities in their entirety. It does not contain unit tests, which
can be found in [/src/test](/src/test), [/src/wallet/test](/src/wallet/test),
etc.
There are currently two sets of tests in this directory:
- [functional](/test/functional) which test the functionality of
dashd and dash-qt by interacting with them through the RPC and P2P
interfaces.
- [util](test/util) which tests the dash utilities, currently only
dash-tx.
The util tests are run as part of `make check` target. The functional
tests are run by the travis continuous build process whenever a pull
request is opened. Both sets of tests can also be run locally.
Functional Test dependencies
============================
The ZMQ functional test requires a python ZMQ library. To install it:
- on Unix, run `sudo apt-get install python3-zmq`
- on mac OS, run `pip3 install pyzmq`
Running tests locally
=====================
Functional tests
----------------
You can run any single test by calling
test/functional/test_runner.py <testname>
Or you can run any combination of tests by calling
test/functional/test_runner.py <testname1> <testname2> <testname3> ...
Run the regression test suite with
test/functional/test_runner.py
Run all possible tests with
test/functional/test_runner.py --extended
By default, tests will be run in parallel. To specify how many jobs to run,
append `--jobs=n` (default n=4).
If you want to create a basic coverage report for the RPC test suite, append `--coverage`.
Possible options, which apply to each individual test run:
```
-h, --help show this help message and exit
--nocleanup Leave dashds and test.* datadir on exit or error
--noshutdown Don't stop dashds after the test execution
--srcdir=SRCDIR Source directory containing dashd/dash-cli
(default: ../../src)
--tmpdir=TMPDIR Root directory for datadirs
--tracerpc Print out all RPC calls as they are made
--coveragedir=COVERAGEDIR
Write tested RPC commands into this directory
```
If you set the environment variable `PYTHON_DEBUG=1` you will get some debug
output (example: `PYTHON_DEBUG=1 test/functional/test_runner.py wallet`).
A 200-block -regtest blockchain and wallets for four nodes
is created the first time a regression test is run and
is stored in the cache/ directory. Each node has 25 mature
blocks (25*500=12500 DASH) in its wallet.
After the first run, the cache/ blockchain and wallets are
copied into a temporary directory and used as the initial
test state.
If you get into a bad state, you should be able
to recover with:
```bash
rm -rf cache
killall dashd
```
Util tests
----------
Util tests can be run locally by running `test/util/bitcoin-util-test.py`.
Use the `-v` option for verbose output.
Writing functional tests
========================
You are encouraged to write functional tests for new or existing features.
Further information about the functional test framework and individual
tests is found in [test/functional](/test/functional).

View File

@ -3,7 +3,7 @@
# file COPYING or http://www.opensource.org/licenses/mit-license.php.
# These environment variables are set by the build process and read by
# rpc-tests.py
# test/functional/test_runner.py
[environment]
SRCDIR=@abs_top_srcdir@

View File

@ -5,7 +5,7 @@
"""Create a blockchain cache.
Creating a cache of the blockchain speeds up test execution when running
multiple qa tests. This helper script is executed by rpc-tests when multiple
multiple functional tests. This helper script is executed by test_runner when multiple
tests are being run in parallel.
"""

Some files were not shown because too many files have changed in this diff Show More