Merge pull request #6650

42cb388 Add chainstate obfuscation to avoid spurious antivirus detection (James O'Beirne)

src/Makefile.test.include

@@ -53,6 +53,7 @@ BITCOIN_TESTS =\
   test/hash_tests.cpp \
   test/key_tests.cpp \
   test/limitedmap_tests.cpp \
+  test/leveldbwrapper_tests.cpp \
   test/main_tests.cpp \
   test/mempool_tests.cpp \
   test/miner_tests.cpp \
@@ -73,6 +74,7 @@ BITCOIN_TESTS =\
   test/sighash_tests.cpp \
   test/sigopcount_tests.cpp \
   test/skiplist_tests.cpp \
+  test/streams_tests.cpp \
   test/test_bitcoin.cpp \
   test/test_bitcoin.h \
   test/timedata_tests.cpp \

src/leveldbwrapper.cpp

@@ -5,6 +5,7 @@
 #include "leveldbwrapper.h"
 
 #include "util.h"
+#include "random.h"
 
 #include <boost/filesystem.hpp>
 
@@ -12,6 +13,7 @@
 #include <leveldb/env.h>
 #include <leveldb/filter_policy.h>
 #include <memenv.h>
+#include <stdint.h>
 
 void HandleError(const leveldb::Status& status) throw(leveldb_error)
 {
@@ -43,7 +45,7 @@ static leveldb::Options GetOptions(size_t nCacheSize)
     return options;
 }
 
-CLevelDBWrapper::CLevelDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory, bool fWipe)
+CLevelDBWrapper::CLevelDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory, bool fWipe, bool obfuscate)
 {
     penv = NULL;
     readoptions.verify_checksums = true;
@@ -67,6 +69,25 @@ CLevelDBWrapper::CLevelDBWrapper(const boost::filesystem::path& path, size_t nCa
     leveldb::Status status = leveldb::DB::Open(options, path.string(), &pdb);
     HandleError(status);
     LogPrintf("Opened LevelDB successfully\n");
+
+    // The base-case obfuscation key, which is a noop.
+    obfuscate_key = std::vector<unsigned char>(OBFUSCATE_KEY_NUM_BYTES, '\000');
+
+    bool key_exists = Read(OBFUSCATE_KEY_KEY, obfuscate_key);
+
+    if (!key_exists && obfuscate && IsEmpty()) {
+        // Initialize non-degenerate obfuscation if it won't upset 
+        // existing, non-obfuscated data.
+        std::vector<unsigned char> new_key = CreateObfuscateKey();
+
+        // Write `new_key` so we don't obfuscate the key with itself
+        Write(OBFUSCATE_KEY_KEY, new_key);
+        obfuscate_key = new_key;
+
+        LogPrintf("Wrote new obfuscate key for %s: %s\n", path.string(), GetObfuscateKeyHex());
+    }
+
+    LogPrintf("Using obfuscation key for %s: %s\n", path.string(), GetObfuscateKeyHex());
 }
 
 CLevelDBWrapper::~CLevelDBWrapper()
@@ -87,3 +108,40 @@ bool CLevelDBWrapper::WriteBatch(CLevelDBBatch& batch, bool fSync) throw(leveldb
     HandleError(status);
     return true;
 }
+
+// Prefixed with null character to avoid collisions with other keys
+//
+// We must use a string constructor which specifies length so that we copy
+// past the null-terminator.
+const std::string CLevelDBWrapper::OBFUSCATE_KEY_KEY("\000obfuscate_key", 14);
+
+const unsigned int CLevelDBWrapper::OBFUSCATE_KEY_NUM_BYTES = 8;
+
+/**
+ * Returns a string (consisting of 8 random bytes) suitable for use as an 
+ * obfuscating XOR key. 
+ */
+std::vector<unsigned char> CLevelDBWrapper::CreateObfuscateKey() const 
+{
+    unsigned char buff[OBFUSCATE_KEY_NUM_BYTES];
+    GetRandBytes(buff, OBFUSCATE_KEY_NUM_BYTES);
+    return std::vector<unsigned char>(&buff[0], &buff[OBFUSCATE_KEY_NUM_BYTES]);
+
+}
+
+bool CLevelDBWrapper::IsEmpty()
+{
+    boost::scoped_ptr<leveldb::Iterator> it(NewIterator());
+    it->SeekToFirst();
+    return !(it->Valid());
+}
+
+const std::vector<unsigned char>& CLevelDBWrapper::GetObfuscateKey() const 
+{ 
+    return obfuscate_key; 
+}
+
+std::string CLevelDBWrapper::GetObfuscateKeyHex() const
+{ 
+    return HexStr(obfuscate_key); 
+}

src/leveldbwrapper.h

@@ -9,6 +9,7 @@
 #include "serialize.h"
 #include "streams.h"
 #include "util.h"
+#include "utilstrencodings.h"
 #include "version.h"
 
 #include <boost/filesystem/path.hpp>
@@ -31,8 +32,14 @@ class CLevelDBBatch
 
 private:
     leveldb::WriteBatch batch;
+    const std::vector<unsigned char> obfuscate_key;
 
 public:
+    /**
+     * @param[in] obfuscate_key    If passed, XOR data with this key.
+     */
+    CLevelDBBatch(const std::vector<unsigned char>& obfuscate_key) : obfuscate_key(obfuscate_key) { };
+
     template <typename K, typename V>
     void Write(const K& key, const V& value)
     {
@@ -44,6 +51,7 @@ public:
         CDataStream ssValue(SER_DISK, CLIENT_VERSION);
         ssValue.reserve(ssValue.GetSerializeSize(value));
         ssValue << value;
+        ssValue.Xor(obfuscate_key);
         leveldb::Slice slValue(&ssValue[0], ssValue.size());
 
         batch.Put(slKey, slValue);
@@ -85,8 +93,27 @@ private:
     //! the database itself
     leveldb::DB* pdb;
 
+    //! a key used for optional XOR-obfuscation of the database
+    std::vector<unsigned char> obfuscate_key;
+
+    //! the key under which the obfuscation key is stored
+    static const std::string OBFUSCATE_KEY_KEY;
+    
+    //! the length of the obfuscate key in number of bytes
+    static const unsigned int OBFUSCATE_KEY_NUM_BYTES;
+    
+    std::vector<unsigned char> CreateObfuscateKey() const;
+
 public:
-    CLevelDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory = false, bool fWipe = false);
+    /**
+     * @param[in] path        Location in the filesystem where leveldb data will be stored.
+     * @param[in] nCacheSize  Configures various leveldb cache settings.
+     * @param[in] fMemory     If true, use leveldb's memory environment.
+     * @param[in] fWipe       If true, remove all existing data.
+     * @param[in] obfuscate   If true, store data obfuscated via simple XOR. If false, XOR
+     *                        with a zero'd byte array.
+     */
+    CLevelDBWrapper(const boost::filesystem::path& path, size_t nCacheSize, bool fMemory = false, bool fWipe = false, bool obfuscate = false);
     ~CLevelDBWrapper();
 
     template <typename K, typename V>
@@ -107,6 +134,7 @@ public:
         }
         try {
             CDataStream ssValue(strValue.data(), strValue.data() + strValue.size(), SER_DISK, CLIENT_VERSION);
+            ssValue.Xor(obfuscate_key);
             ssValue >> value;
         } catch (const std::exception&) {
             return false;
@@ -117,7 +145,7 @@ public:
     template <typename K, typename V>
     bool Write(const K& key, const V& value, bool fSync = false) throw(leveldb_error)
     {
-        CLevelDBBatch batch;
+        CLevelDBBatch batch(obfuscate_key);
         batch.Write(key, value);
         return WriteBatch(batch, fSync);
     }
@@ -144,7 +172,7 @@ public:
     template <typename K>
     bool Erase(const K& key, bool fSync = false) throw(leveldb_error)
     {
-        CLevelDBBatch batch;
+        CLevelDBBatch batch(obfuscate_key);
         batch.Erase(key);
         return WriteBatch(batch, fSync);
     }
@@ -159,7 +187,7 @@ public:
 
     bool Sync() throw(leveldb_error)
     {
-        CLevelDBBatch batch;
+        CLevelDBBatch batch(obfuscate_key);
         return WriteBatch(batch, true);
     }
 
@@ -168,6 +196,23 @@ public:
     {
         return pdb->NewIterator(iteroptions);
     }
+
+    /**
+     * Return true if the database managed by this class contains no entries.
+     */
+    bool IsEmpty();
+
+    /**
+     * Accessor for obfuscate_key.
+     */
+    const std::vector<unsigned char>& GetObfuscateKey() const;
+
+    /**
+     * Return the obfuscate_key as a hex-formatted string.
+     */
+    std::string GetObfuscateKeyHex() const;
+
 };
 
 #endif // BITCOIN_LEVELDBWRAPPER_H
+

src/streams.h

@@ -296,6 +296,29 @@ public:
         data.insert(data.end(), begin(), end());
         clear();
     }
+
+    /**
+     * XOR the contents of this stream with a certain key.
+     *
+     * @param[in] key    The key used to XOR the data in this stream.
+     */
+    void Xor(const std::vector<unsigned char>& key)
+    {
+        if (key.size() == 0) {
+            return;
+        }
+
+        for (size_type i = 0, j = 0; i != size(); i++) {
+            vch[i] ^= key[j++];
+
+            // This potentially acts on very many bytes of data, so it's
+            // important that we calculate `j`, i.e. the `key` index in this
+            // way instead of doing a %, which would effectively be a division
+            // for each byte Xor'd -- much slower than need be.
+            if (j == key.size())
+                j = 0;
+        }
+    }
 };
 
 

src/test/leveldbwrapper_tests.cpp

@@ -0,0 +1,128 @@
+// Copyright (c) 2012-2013 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include "leveldbwrapper.h"
+#include "uint256.h"
+#include "random.h"
+#include "test/test_bitcoin.h"
+
+#include <boost/assign/std/vector.hpp> // for 'operator+=()'
+#include <boost/assert.hpp>
+#include <boost/test/unit_test.hpp>
+                    
+using namespace std;
+using namespace boost::assign; // bring 'operator+=()' into scope
+using namespace boost::filesystem;
+         
+// Test if a string consists entirely of null characters
+bool is_null_key(const vector<unsigned char>& key) {
+    bool isnull = true;
+
+    for (unsigned int i = 0; i < key.size(); i++)
+        isnull &= (key[i] == '\x00');
+
+    return isnull;
+}
+ 
+BOOST_FIXTURE_TEST_SUITE(leveldbwrapper_tests, BasicTestingSetup)
+                       
+BOOST_AUTO_TEST_CASE(leveldbwrapper)
+{
+    // Perform tests both obfuscated and non-obfuscated.
+    for (int i = 0; i < 2; i++) {
+        bool obfuscate = (bool)i;
+        path ph = temp_directory_path() / unique_path();
+        CLevelDBWrapper dbw(ph, (1 << 20), true, false, obfuscate);
+        char key = 'k';
+        uint256 in = GetRandHash();
+        uint256 res;
+
+        // Ensure that we're doing real obfuscation when obfuscate=true
+        BOOST_CHECK(obfuscate != is_null_key(dbw.GetObfuscateKey()));
+
+        BOOST_CHECK(dbw.Write(key, in));
+        BOOST_CHECK(dbw.Read(key, res));
+        BOOST_CHECK_EQUAL(res.ToString(), in.ToString());
+    }
+}
+                       
+// Test that we do not obfuscation if there is existing data.
+BOOST_AUTO_TEST_CASE(existing_data_no_obfuscate)
+{
+    // We're going to share this path between two wrappers
+    path ph = temp_directory_path() / unique_path();
+    create_directories(ph);
+
+    // Set up a non-obfuscated wrapper to write some initial data.
+    CLevelDBWrapper* dbw = new CLevelDBWrapper(ph, (1 << 10), false, false, false);
+    char key = 'k';
+    uint256 in = GetRandHash();
+    uint256 res;
+
+    BOOST_CHECK(dbw->Write(key, in));
+    BOOST_CHECK(dbw->Read(key, res));
+    BOOST_CHECK_EQUAL(res.ToString(), in.ToString());
+
+    // Call the destructor to free leveldb LOCK
+    delete dbw;
+
+    // Now, set up another wrapper that wants to obfuscate the same directory
+    CLevelDBWrapper odbw(ph, (1 << 10), false, false, true);
+
+    // Check that the key/val we wrote with unobfuscated wrapper exists and 
+    // is readable.
+    uint256 res2;
+    BOOST_CHECK(odbw.Read(key, res2));
+    BOOST_CHECK_EQUAL(res2.ToString(), in.ToString());
+
+    BOOST_CHECK(!odbw.IsEmpty()); // There should be existing data
+    BOOST_CHECK(is_null_key(odbw.GetObfuscateKey())); // The key should be an empty string
+
+    uint256 in2 = GetRandHash();
+    uint256 res3;
+ 
+    // Check that we can write successfully
+    BOOST_CHECK(odbw.Write(key, in2));
+    BOOST_CHECK(odbw.Read(key, res3));
+    BOOST_CHECK_EQUAL(res3.ToString(), in2.ToString());
+}
+                        
+// Ensure that we start obfuscating during a reindex.
+BOOST_AUTO_TEST_CASE(existing_data_reindex)
+{
+    // We're going to share this path between two wrappers
+    path ph = temp_directory_path() / unique_path();
+    create_directories(ph);
+
+    // Set up a non-obfuscated wrapper to write some initial data.
+    CLevelDBWrapper* dbw = new CLevelDBWrapper(ph, (1 << 10), false, false, false);
+    char key = 'k';
+    uint256 in = GetRandHash();
+    uint256 res;
+
+    BOOST_CHECK(dbw->Write(key, in));
+    BOOST_CHECK(dbw->Read(key, res));
+    BOOST_CHECK_EQUAL(res.ToString(), in.ToString());
+
+    // Call the destructor to free leveldb LOCK
+    delete dbw;
+
+    // Simulate a -reindex by wiping the existing data store
+    CLevelDBWrapper odbw(ph, (1 << 10), false, true, true);
+
+    // Check that the key/val we wrote with unobfuscated wrapper doesn't exist
+    uint256 res2;
+    BOOST_CHECK(!odbw.Read(key, res2));
+    BOOST_CHECK(!is_null_key(odbw.GetObfuscateKey()));
+
+    uint256 in2 = GetRandHash();
+    uint256 res3;
+ 
+    // Check that we can write successfully
+    BOOST_CHECK(odbw.Write(key, in2));
+    BOOST_CHECK(odbw.Read(key, res3));
+    BOOST_CHECK_EQUAL(res3.ToString(), in2.ToString());
+}
+ 
+BOOST_AUTO_TEST_SUITE_END()

src/test/streams_tests.cpp

@@ -0,0 +1,67 @@
+// Copyright (c) 2012-2013 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include "streams.h"
+#include "support/allocators/zeroafterfree.h"
+#include "test/test_bitcoin.h"
+
+#include <boost/assign/std/vector.hpp> // for 'operator+=()'
+#include <boost/assert.hpp>
+#include <boost/test/unit_test.hpp>
+                    
+using namespace std;
+using namespace boost::assign; // bring 'operator+=()' into scope
+
+BOOST_FIXTURE_TEST_SUITE(streams_tests, BasicTestingSetup)
+
+BOOST_AUTO_TEST_CASE(streams_serializedata_xor)
+{
+    std::vector<char> in;
+    std::vector<char> expected_xor;
+    std::vector<unsigned char> key;
+    CDataStream ds(in, 0, 0);
+
+    // Degenerate case
+    
+    key += '\x00','\x00';
+    ds.Xor(key);
+    BOOST_CHECK_EQUAL(
+            std::string(expected_xor.begin(), expected_xor.end()), 
+            std::string(ds.begin(), ds.end()));
+
+    in += '\x0f','\xf0';
+    expected_xor += '\xf0','\x0f';
+    
+    // Single character key
+
+    ds.clear();
+    ds.insert(ds.begin(), in.begin(), in.end());
+    key.clear();
+
+    key += '\xff';
+    ds.Xor(key);
+    BOOST_CHECK_EQUAL(
+            std::string(expected_xor.begin(), expected_xor.end()), 
+            std::string(ds.begin(), ds.end())); 
+    
+    // Multi character key
+
+    in.clear();
+    expected_xor.clear();
+    in += '\xf0','\x0f';
+    expected_xor += '\x0f','\x00';
+                        
+    ds.clear();
+    ds.insert(ds.begin(), in.begin(), in.end());
+
+    key.clear();
+    key += '\xff','\x0f';
+
+    ds.Xor(key);
+    BOOST_CHECK_EQUAL(
+            std::string(expected_xor.begin(), expected_xor.end()), 
+            std::string(ds.begin(), ds.end()));  
+}         
+
+BOOST_AUTO_TEST_SUITE_END()

src/txdb.cpp

@@ -29,18 +29,8 @@ static const char DB_REINDEX_FLAG = 'R';
 static const char DB_LAST_BLOCK = 'l';
 
 
-void static BatchWriteCoins(CLevelDBBatch &batch, const uint256 &hash, const CCoins &coins) {
-    if (coins.IsPruned())
-        batch.Erase(make_pair(DB_COINS, hash));
-    else
-        batch.Write(make_pair(DB_COINS, hash), coins);
-}
-
-void static BatchWriteHashBestChain(CLevelDBBatch &batch, const uint256 &hash) {
-    batch.Write(DB_BEST_BLOCK, hash);
-}
-
-CCoinsViewDB::CCoinsViewDB(size_t nCacheSize, bool fMemory, bool fWipe) : db(GetDataDir() / "chainstate", nCacheSize, fMemory, fWipe) {
+CCoinsViewDB::CCoinsViewDB(size_t nCacheSize, bool fMemory, bool fWipe) : db(GetDataDir() / "chainstate", nCacheSize, fMemory, fWipe, true) 
+{
 }
 
 bool CCoinsViewDB::GetCoins(const uint256 &txid, CCoins &coins) const {
@@ -59,12 +49,15 @@ uint256 CCoinsViewDB::GetBestBlock() const {
 }
 
 bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) {
-    CLevelDBBatch batch;
+    CLevelDBBatch batch(db.GetObfuscateKey());
     size_t count = 0;
     size_t changed = 0;
     for (CCoinsMap::iterator it = mapCoins.begin(); it != mapCoins.end();) {
         if (it->second.flags & CCoinsCacheEntry::DIRTY) {
-            BatchWriteCoins(batch, it->first, it->second.coins);
+            if (it->second.coins.IsPruned())
+                batch.Erase(make_pair(DB_COINS, it->first));
+            else
+                batch.Write(make_pair(DB_COINS, it->first), it->second.coins);
             changed++;
         }
         count++;
@@ -72,7 +65,7 @@ bool CCoinsViewDB::BatchWrite(CCoinsMap &mapCoins, const uint256 &hashBlock) {
         mapCoins.erase(itOld);
     }
     if (!hashBlock.IsNull())
-        BatchWriteHashBestChain(batch, hashBlock);
+        batch.Write(DB_BEST_BLOCK, hashBlock);
 
     LogPrint("coindb", "Committing %u changed transactions (out of %u) to coin database...\n", (unsigned int)changed, (unsigned int)count);
     return db.WriteBatch(batch);
@@ -158,7 +151,7 @@ bool CCoinsViewDB::GetStats(CCoinsStats &stats) const {
 }
 
 bool CBlockTreeDB::WriteBatchSync(const std::vector<std::pair<int, const CBlockFileInfo*> >& fileInfo, int nLastFile, const std::vector<const CBlockIndex*>& blockinfo) {
-    CLevelDBBatch batch;
+    CLevelDBBatch batch(GetObfuscateKey());
     for (std::vector<std::pair<int, const CBlockFileInfo*> >::const_iterator it=fileInfo.begin(); it != fileInfo.end(); it++) {
         batch.Write(make_pair(DB_BLOCK_FILES, it->first), *it->second);
     }
@@ -174,7 +167,7 @@ bool CBlockTreeDB::ReadTxIndex(const uint256 &txid, CDiskTxPos &pos) {
 }
 
 bool CBlockTreeDB::WriteTxIndex(const std::vector<std::pair<uint256, CDiskTxPos> >&vect) {
-    CLevelDBBatch batch;
+    CLevelDBBatch batch(GetObfuscateKey());
     for (std::vector<std::pair<uint256,CDiskTxPos> >::const_iterator it=vect.begin(); it!=vect.end(); it++)
         batch.Write(make_pair(DB_TXINDEX, it->first), it->second);
     return WriteBatch(batch);