included-tests: generate binary data from test files for inclusion into test binaries

This change moves test data into the binaries rather than reading them from
the disk at runtime.

Advantages:
- Tests become distributable
- Cross-compile friendly. Build on one machine and execute in an arbitrary
  location on another.
- Easier testing for backports. Users can verify that tests pass without having
  to track down corresponding test data.
- More trustworthy test results and easier quality assurance as tests make
  fewer assumptions about their environment.
- Tests could theoretically run at client/daemon startup and exit on failure.

Disadvantages:
- Required 'hexdump' build-dependency. This is a standard bsd tool that should
  be usable everywhere. It is likely already installed on all build-machines.
- Tests can no longer be fudged after build by altering test-data.

.gitignore

@@ -57,6 +57,9 @@ src/qt/res/.dirstamp
 *.trs
 *.dmg
 
+*.json.h
+*.raw.h
+
 # Compilation and Qt preprocessor part
 *.qm
 Makefile

configure.ac

@@ -151,6 +151,7 @@ AC_PATH_PROGS([LUPDATE], [lupdate-qt4 lupdate4 lupdate],, $qt_bin_path:$PATH)
 AC_PATH_PROG([PROTOC], [protoc],, $protoc_bin_path:$PATH)
 AC_PATH_PROG(CCACHE,ccache)
 AC_PATH_PROG(XGETTEXT,xgettext)
+AC_PATH_PROG(HEXDUMP,hexdump)
 PKG_PROG_PKG_CONFIG
 
 ## TODO: Remove these hard-coded paths and flags. They are here for the sake of
@@ -353,6 +354,12 @@ AX_BOOST_THREAD
 AX_BOOST_CHRONO
 
 if test x$use_tests = xyes; then
+
+  if test x$HEXDUMP = x; then
+    AC_MSG_ERROR(hexdump is required for tests)
+  fi
+
+
   AX_BOOST_UNIT_TEST_FRAMEWORK
 fi
 

src/Makefile.am

@@ -1,6 +1,6 @@
 AM_CPPFLAGS =  $(INCLUDES) -I$(top_builddir)/src/obj \
   -I$(top_srcdir)/src/leveldb/include -I$(top_srcdir)/src/leveldb/helpers \
-  $(BOOST_INCLUDES)
+  -I$(builddir) $(BOOST_INCLUDES)
 AM_LDFLAGS = $(PTHREAD_CFLAGS)
 
 noinst_LIBRARIES = libbitcoin.a

src/Makefile.include

@@ -49,3 +49,19 @@ moc_%.cpp: %.h
 %.pb.cc %.pb.h: %.proto
 	test -f $(PROTOC) && $(PROTOC) --cpp_out=$(@D) --proto_path=$(abspath $(<D) $<)  || \
 	  echo error: could not build $@
+
+%.json.h: %.json
+	@$(MKDIR_P) $(@D)
+	@echo "namespace json_tests{" > $@
+	@echo "static unsigned const char $(*F)[] = {" >> $@
+	@$(HEXDUMP) -v -e '8/1 "0x%02x, "' -e '"\n"' $< | $(SED) -e 's/0x  ,//g' >> $@
+	@echo "};};" >> $@
+	@echo "Generated $@"
+
+%.raw.h: %.raw
+	@$(MKDIR_P) $(@D)
+	@echo "namespace alert_tests{" > $@
+	@echo "static unsigned const char $(*F)[] = {" >> $@
+	@$(HEXDUMP) -v -e '8/1 "0x%02x, "' -e '"\n"' $< | $(SED) -e 's/0x  ,//g' >> $@
+	@echo "};};" >> $@
+	@echo "Generated $@"

src/test/Makefile.am

@@ -10,19 +10,20 @@ bin_PROGRAMS = test_bitcoin
 
 TESTS = test_bitcoin
 
-TEST_DATA_DIR=$(srcdir)/data
+JSON_TEST_FILES= data/script_valid.json \
+  data/base58_keys_valid.json data/sig_canonical.json \
+  data/sig_noncanonical.json \
+  data/base58_encode_decode.json \
+  data/base58_keys_invalid.json \
+  data/script_invalid.json data/tx_invalid.json \
+  data/tx_valid.json
 
-TEST_DATA_FILES= $(TEST_DATA_DIR)/script_valid.json \
+RAW_TEST_FILES = data/alertTests.raw
-  $(TEST_DATA_DIR)/base58_keys_valid.json $(TEST_DATA_DIR)/sig_canonical.json \
+
-  $(TEST_DATA_DIR)/sig_noncanonical.json \
+BUILT_SOURCES = $(JSON_TEST_FILES:.json=.json.h) $(RAW_TEST_FILES:.raw=.raw.h)
-  $(TEST_DATA_DIR)/base58_encode_decode.json $(TEST_DATA_DIR)/alertTests \
-  $(TEST_DATA_DIR)/base58_keys_invalid.json \
-  $(TEST_DATA_DIR)/script_invalid.json $(TEST_DATA_DIR)/tx_invalid.json \
-  $(TEST_DATA_DIR)/tx_valid.json
 
 # test_bitcoin binary #
-test_bitcoin_CPPFLAGS = $(AM_CPPFLAGS) $(TESTDEFS) \
+test_bitcoin_CPPFLAGS = $(AM_CPPFLAGS) $(TESTDEFS)
-  -DTEST_DATA_DIR=$(abs_srcdir)/data
 test_bitcoin_LDADD = $(LIBBITCOIN) $(LIBLEVELDB) $(LIBMEMENV) \
   $(BOOST_LIBS) $(BOOST_UNIT_TEST_FRAMEWORK_LIB)
 test_bitcoin_SOURCES = accounting_tests.cpp alert_tests.cpp \
@@ -33,6 +34,8 @@ test_bitcoin_SOURCES = accounting_tests.cpp alert_tests.cpp \
   netbase_tests.cpp pmt_tests.cpp rpc_tests.cpp script_P2SH_tests.cpp \
   script_tests.cpp serialize_tests.cpp sigopcount_tests.cpp test_bitcoin.cpp \
   transaction_tests.cpp uint160_tests.cpp uint256_tests.cpp util_tests.cpp \
-  wallet_tests.cpp $(TEST_DATA_FILES)
+  wallet_tests.cpp $(JSON_TEST_FILES) $(RAW_TEST_FILES)
 
-CLEANFILES = *.gcda *.gcno
+nodist_test_bitcoin_SOURCES = $(BUILT_SOURCES)
+
+CLEANFILES = *.gcda *.gcno $(BUILT_SOURCES)

src/test/alert_tests.cpp

@@ -9,6 +9,7 @@
 #include "alert.h"
 #include "serialize.h"
 #include "util.h"
+#include "data/alertTests.raw.h"
 
 #if 0
 //
@@ -71,27 +72,13 @@ struct ReadAlerts
 {
     ReadAlerts()
     {
-        std::string filename("alertTests");
+        std::vector<unsigned char> vch(alert_tests::alertTests, alert_tests::alertTests + sizeof(alert_tests::alertTests));
-        namespace fs = boost::filesystem;
+        CDataStream stream(vch, SER_DISK, CLIENT_VERSION);
-        fs::path testFile = fs::current_path() / "data" / filename;
-#ifdef TEST_DATA_DIR
-        if (!fs::exists(testFile))
-        {
-            testFile = fs::path(BOOST_PP_STRINGIZE(TEST_DATA_DIR)) / filename;
-        }
-#endif
-        FILE* fp = fopen(testFile.string().c_str(), "rb");
-        if (!fp) return;
-
-
-        CAutoFile filein = CAutoFile(fp, SER_DISK, CLIENT_VERSION);
-        if (!filein) return;
-
         try {
-            while (!feof(filein))
+            while (stream.good())
             {
                 CAlert alert;
-                filein >> alert;
+                stream >> alert;
                 alerts.push_back(alert);
             }
         }

src/test/base58_tests.cpp

@@ -2,20 +2,21 @@
 #include "json/json_spirit_reader_template.h"
 #include "json/json_spirit_writer_template.h"
 #include "json/json_spirit_utils.h"
+#include "data/base58_encode_decode.json.h"
+#include "data/base58_keys_invalid.json.h"
+#include "data/base58_keys_valid.json.h"
 
 #include "base58.h"
 #include "util.h"
-
 using namespace json_spirit;
-extern Array read_json(const std::string& filename);
+extern Array read_json(const std::string& jsondata);
 
 BOOST_AUTO_TEST_SUITE(base58_tests)
 
 // Goal: test low-level base58 encoding functionality
 BOOST_AUTO_TEST_CASE(base58_EncodeBase58)
 {
-    Array tests = read_json("base58_encode_decode.json");
+    Array tests = read_json(std::string(json_tests::base58_encode_decode, json_tests::base58_encode_decode + sizeof(json_tests::base58_encode_decode)));
-
     BOOST_FOREACH(Value& tv, tests)
     {
         Array test = tv.get_array();
@@ -36,7 +37,7 @@ BOOST_AUTO_TEST_CASE(base58_EncodeBase58)
 // Goal: test low-level base58 decoding functionality
 BOOST_AUTO_TEST_CASE(base58_DecodeBase58)
 {
-    Array tests = read_json("base58_encode_decode.json");
+    Array tests = read_json(std::string(json_tests::base58_encode_decode, json_tests::base58_encode_decode + sizeof(json_tests::base58_encode_decode)));
     std::vector<unsigned char> result;
 
     BOOST_FOREACH(Value& tv, tests)
@@ -104,7 +105,7 @@ public:
 // Goal: check that parsed keys match test payload
 BOOST_AUTO_TEST_CASE(base58_keys_valid_parse)
 {
-    Array tests = read_json("base58_keys_valid.json");
+    Array tests = read_json(std::string(json_tests::base58_keys_valid, json_tests::base58_keys_valid + sizeof(json_tests::base58_keys_valid)));
     std::vector<unsigned char> result;
     CBitcoinSecret secret;
     CBitcoinAddress addr;
@@ -163,7 +164,7 @@ BOOST_AUTO_TEST_CASE(base58_keys_valid_parse)
 // Goal: check that generated keys match test vectors
 BOOST_AUTO_TEST_CASE(base58_keys_valid_gen)
 {
-    Array tests = read_json("base58_keys_valid.json");
+    Array tests = read_json(std::string(json_tests::base58_keys_valid, json_tests::base58_keys_valid + sizeof(json_tests::base58_keys_valid)));
     std::vector<unsigned char> result;
     BOOST_FOREACH(Value& tv, tests)
     {
@@ -231,7 +232,7 @@ BOOST_AUTO_TEST_CASE(base58_keys_valid_gen)
 // Goal: check that base58 parsing code is robust against a variety of corrupted data
 BOOST_AUTO_TEST_CASE(base58_keys_invalid)
 {
-    Array tests = read_json("base58_keys_invalid.json"); // Negative testcases
+    Array tests = read_json(std::string(json_tests::base58_keys_invalid, json_tests::base58_keys_invalid + sizeof(json_tests::base58_keys_invalid))); // Negative testcases
     std::vector<unsigned char> result;
     CBitcoinSecret secret;
     CBitcoinAddress addr;

src/test/canonical_tests.cpp

@@ -8,13 +8,15 @@
 #include "key.h"
 #include "script.h"
 #include "util.h"
+#include "data/sig_noncanonical.json.h"
+#include "data/sig_canonical.json.h"
 
 using namespace std;
 using namespace json_spirit;
 
 
 // In script_tests.cpp
-extern Array read_json(const std::string& filename);
+extern Array read_json(const std::string& jsondata);
 
 BOOST_AUTO_TEST_SUITE(canonical_tests)
 
@@ -58,7 +60,7 @@ bool static IsCanonicalSignature_OpenSSL(const std::vector<unsigned char> &vchSi
 
 BOOST_AUTO_TEST_CASE(script_canon)
 {
-    Array tests = read_json("sig_canonical.json");
+    Array tests = read_json(std::string(json_tests::sig_canonical, json_tests::sig_canonical + sizeof(json_tests::sig_canonical)));
 
     BOOST_FOREACH(Value &tv, tests) {
         string test = tv.get_str();
@@ -72,7 +74,7 @@ BOOST_AUTO_TEST_CASE(script_canon)
 
 BOOST_AUTO_TEST_CASE(script_noncanon)
 {
-    Array tests = read_json("sig_noncanonical.json");
+    Array tests = read_json(std::string(json_tests::sig_noncanonical, json_tests::sig_noncanonical + sizeof(json_tests::sig_noncanonical)));
 
     BOOST_FOREACH(Value &tv, tests) {
         string test = tv.get_str();

src/test/script_tests.cpp

@@ -14,6 +14,8 @@
 
 #include "main.h"
 #include "wallet.h"
+#include "data/script_invalid.json.h"
+#include "data/script_valid.json.h"
 
 using namespace std;
 using namespace json_spirit;
@@ -90,34 +92,15 @@ ParseScript(string s)
 }
 
 Array
-read_json(const std::string& filename)
+read_json(const std::string& jsondata)
 {
-    namespace fs = boost::filesystem;
-    fs::path testFile = fs::current_path() / "data" / filename;
-
-#ifdef TEST_DATA_DIR
-    if (!fs::exists(testFile))
-    {
-        testFile = fs::path(BOOST_PP_STRINGIZE(TEST_DATA_DIR)) / filename;
-    }
-#endif
-
-    ifstream ifs(testFile.string().c_str(), ifstream::in);
     Value v;
-    if (!read_stream(ifs, v))
-    {
-        if (ifs.fail())
-            BOOST_ERROR("Cound not find/open " << filename);
-        else
-            BOOST_ERROR("JSON syntax error in " << filename);
-        return Array();
-    }
-    if (v.type() != array_type)
-    {
-        BOOST_ERROR(filename << " does not contain a json array");
-        return Array();
-    }
 
+    if (!read_string(jsondata, v) || v.type() != array_type)
+    {
+        BOOST_ERROR("Parse error.");
+        return Array();
+    }
     return v.get_array();
 }
 
@@ -130,7 +113,7 @@ BOOST_AUTO_TEST_CASE(script_valid)
     // Inner arrays are [ "scriptSig", "scriptPubKey" ]
     // ... where scriptSig and scriptPubKey are stringified
     // scripts.
-    Array tests = read_json("script_valid.json");
+    Array tests = read_json(std::string(json_tests::script_valid, json_tests::script_valid + sizeof(json_tests::script_valid)));
 
     BOOST_FOREACH(Value& tv, tests)
     {
@@ -154,7 +137,7 @@ BOOST_AUTO_TEST_CASE(script_valid)
 BOOST_AUTO_TEST_CASE(script_invalid)
 {
     // Scripts that should evaluate as invalid
-    Array tests = read_json("script_invalid.json");
+    Array tests = read_json(std::string(json_tests::script_invalid, json_tests::script_invalid + sizeof(json_tests::script_invalid)));
 
     BOOST_FOREACH(Value& tv, tests)
     {

src/test/transaction_tests.cpp

@@ -5,12 +5,14 @@
 
 #include "main.h"
 #include "wallet.h"
+#include "data/tx_invalid.json.h"
+#include "data/tx_valid.json.h"
 
 using namespace std;
 using namespace json_spirit;
 
 // In script_tests.cpp
-extern Array read_json(const std::string& filename);
+extern Array read_json(const std::string& jsondata);
 extern CScript ParseScript(string s);
 
 BOOST_AUTO_TEST_SUITE(transaction_tests)
@@ -22,7 +24,7 @@ BOOST_AUTO_TEST_CASE(tx_valid)
     // Inner arrays are either [ "comment" ]
     // or [[[prevout hash, prevout index, prevout scriptPubKey], [input 2], ...],"], serializedTransaction, enforceP2SH
     // ... where all scripts are stringified scripts.
-    Array tests = read_json("tx_valid.json");
+    Array tests = read_json(std::string(json_tests::tx_valid, json_tests::tx_valid + sizeof(json_tests::tx_valid)));
 
     BOOST_FOREACH(Value& tv, tests)
     {
@@ -91,7 +93,7 @@ BOOST_AUTO_TEST_CASE(tx_invalid)
     // Inner arrays are either [ "comment" ]
     // or [[[prevout hash, prevout index, prevout scriptPubKey], [input 2], ...],"], serializedTransaction, enforceP2SH
     // ... where all scripts are stringified scripts.
-    Array tests = read_json("tx_invalid.json");
+    Array tests = read_json(std::string(json_tests::tx_invalid, json_tests::tx_invalid + sizeof(json_tests::tx_invalid)));
 
     BOOST_FOREACH(Value& tv, tests)
     {