From 84d6d69fc69662b2709fffbeaf3c3b4f53c535b1 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Sun, 18 Aug 2013 00:58:04 +0200 Subject: [PATCH] Squashed 'src/leveldb/' changes from ae6c262..a02ddf9 a02ddf9 Added GNU/kFreeBSD kernel name (TARGET_OS) 8487468 CondVar::SignalAll was broken, leading to deadlocks on Windows builds. http://code.google.com/p/leveldb/issues/detail?id=149 f6d84d1 Allow files to be opened for reading multiple times cb8e3f7 Checking whether closing succeeds d5317e8 Print actual Win32 error that occurred on file creation failure. 907f308 Port leveldb to MinGW32 9def2bf Mingw support for Windows LevelDB port 0a7b074 Pre-Vista leveldb::port::InitOnce implementation 31a2b09 Native Windows LevelDB port 058a035 Remove Snappy support 5bd76dc Release leveldb 1.12 7b094f1 Release leveldb 1.11 28dad91 Release leveldb 1.10 514c943 Make DB::Open fail if sst files are missing. d84c825 Fix corruption bug found and analyzed by dhruba@gmail.com ea2e919 added utility to dump leveldb files REVERT: ae6c262 Merge branch 'leveldb' into ripple-fork REVERT: 28fa222 Looks like a bit more delay is needed to smooth the latency. REVERT: a18f3e6 Tidy up JobQueue, add ripple_core module REVERT: ab82e57 Release leveldb 1.12 REVERT: 02c6259 Release leveldb 1.11 REVERT: 5bbb544 Rate limit compactions with a 25ms pause after each complete file. REVERT: 8c29c47 LevelDB issue 178 fix: cannot resize a level 0 compaction set REVERT: 18b245c Added GNU/kFreeBSD kernel name (TARGET_OS) REVERT: 8be9d12 CondVar::SignalAll was broken, leading to deadlocks on Windows builds. http://code.google.com/p/leveldb/issues/detail?id=149 REVERT: c9fc070 Upgrade LevelDB to 1.10.0, mostly for better write stall logging. REVERT: 8215b15 Tweak to variable name to support unity build REVERT: aca1ffc Allow files to be opened for reading multiple times REVERT: 693a70c Checking whether closing succeeds REVERT: 0144d04 Print actual Win32 error that occurred on file creation failure. REVERT: 43ed517 Fix corruption bug found and analyzed by dhruba@gmail.com REVERT: 413c74c added utility to dump leveldb files REVERT: 96eda85 Port leveldb to MinGW32 REVERT: 0967260 Mingw support for Windows LevelDB port REVERT: ee3f9bd Pre-Vista leveldb::port::InitOnce implementation REVERT: f5d0a41 Native Windows LevelDB port REVERT: 28b35f1 Remove Snappy support git-subtree-dir: src/leveldb git-subtree-split: a02ddf9b14d145e88185ee209ab8b01d8826663a --- .gitignore | 1 + Makefile | 2 +- build_detect_platform | 12 +++++- db/db_impl.cc | 5 +-- issues/issue178_test.cc | 92 +++++++++++++++++++++++++++++++++++++++++ util/comparator.cc | 4 +- 6 files changed, 107 insertions(+), 9 deletions(-) create mode 100644 issues/issue178_test.cc diff --git a/.gitignore b/.gitignore index 55ba072e0e..71d87a4eeb 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ build_config.mk *.so.* *_test db_bench +leveldbutil Release Debug Benchmark diff --git a/Makefile b/Makefile index 38b9bf7729..96af7765be 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ OPT ?= -O2 -DNDEBUG # (A) Production use (optimized mode) #----------------------------------------------- # detect what platform we're building on -$(shell CC=$(CC) CXX=$(CXX) TARGET_OS=$(TARGET_OS) \ +$(shell CC="$(CC)" CXX="$(CXX)" TARGET_OS="$(TARGET_OS)" \ ./build_detect_platform build_config.mk ./) # this file is generated by the previous line to set build flags and sources include build_config.mk diff --git a/build_detect_platform b/build_detect_platform index a3ad057eee..bdfd64172c 100755 --- a/build_detect_platform +++ b/build_detect_platform @@ -44,6 +44,10 @@ if test -z "$CXX"; then CXX=g++ fi +if test -z "$TMPDIR"; then + TMPDIR=/tmp +fi + # Detect OS if test -z "$TARGET_OS"; then TARGET_OS=`uname -s` @@ -169,8 +173,10 @@ if [ "$CROSS_COMPILE" = "true" ]; then # Cross-compiling; do not try any compilation tests. true else + CXXOUTPUT="${TMPDIR}/leveldb_build_detect_platform-cxx.$$" + # If -std=c++0x works, use . Otherwise use port_posix.h. - $CXX $CXXFLAGS -std=c++0x -x c++ - -o /dev/null 2>/dev/null </dev/null < int main() {} EOF @@ -182,12 +188,14 @@ EOF fi # Test whether tcmalloc is available - $CXX $CXXFLAGS -x c++ - -o /dev/null -ltcmalloc 2>/dev/null </dev/null </dev/null fi PLATFORM_CCFLAGS="$PLATFORM_CCFLAGS $COMMON_FLAGS" diff --git a/db/db_impl.cc b/db/db_impl.cc index af02467b33..395d3172ad 100644 --- a/db/db_impl.cc +++ b/db/db_impl.cc @@ -322,7 +322,7 @@ Status DBImpl::Recover(VersionEdit* edit) { if (ParseFileName(filenames[i], &number, &type)) { expected.erase(number); if (type == kLogFile && ((number >= min_log) || (number == prev_log))) - logs.push_back(number); + logs.push_back(number); } } if (!expected.empty()) { @@ -822,9 +822,6 @@ Status DBImpl::FinishCompactionOutputFile(CompactionState* compact, (unsigned long long) output_number, (unsigned long long) current_entries, (unsigned long long) current_bytes); - - // rate-limit compaction file creation with a 100ms pause - env_->SleepForMicroseconds(100000); } } return s; diff --git a/issues/issue178_test.cc b/issues/issue178_test.cc new file mode 100644 index 0000000000..1b1cf8bb28 --- /dev/null +++ b/issues/issue178_test.cc @@ -0,0 +1,92 @@ +// Copyright (c) 2013 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. See the AUTHORS file for names of contributors. + +// Test for issue 178: a manual compaction causes deleted data to reappear. +#include +#include +#include + +#include "leveldb/db.h" +#include "leveldb/write_batch.h" +#include "util/testharness.h" + +namespace { + +const int kNumKeys = 1100000; + +std::string Key1(int i) { + char buf[100]; + snprintf(buf, sizeof(buf), "my_key_%d", i); + return buf; +} + +std::string Key2(int i) { + return Key1(i) + "_xxx"; +} + +class Issue178 { }; + +TEST(Issue178, Test) { + // Get rid of any state from an old run. + std::string dbpath = leveldb::test::TmpDir() + "/leveldb_cbug_test"; + DestroyDB(dbpath, leveldb::Options()); + + // Open database. Disable compression since it affects the creation + // of layers and the code below is trying to test against a very + // specific scenario. + leveldb::DB* db; + leveldb::Options db_options; + db_options.create_if_missing = true; + db_options.compression = leveldb::kNoCompression; + ASSERT_OK(leveldb::DB::Open(db_options, dbpath, &db)); + + // create first key range + leveldb::WriteBatch batch; + for (size_t i = 0; i < kNumKeys; i++) { + batch.Put(Key1(i), "value for range 1 key"); + } + ASSERT_OK(db->Write(leveldb::WriteOptions(), &batch)); + + // create second key range + batch.Clear(); + for (size_t i = 0; i < kNumKeys; i++) { + batch.Put(Key2(i), "value for range 2 key"); + } + ASSERT_OK(db->Write(leveldb::WriteOptions(), &batch)); + + // delete second key range + batch.Clear(); + for (size_t i = 0; i < kNumKeys; i++) { + batch.Delete(Key2(i)); + } + ASSERT_OK(db->Write(leveldb::WriteOptions(), &batch)); + + // compact database + std::string start_key = Key1(0); + std::string end_key = Key1(kNumKeys - 1); + leveldb::Slice least(start_key.data(), start_key.size()); + leveldb::Slice greatest(end_key.data(), end_key.size()); + + // commenting out the line below causes the example to work correctly + db->CompactRange(&least, &greatest); + + // count the keys + leveldb::Iterator* iter = db->NewIterator(leveldb::ReadOptions()); + size_t num_keys = 0; + for (iter->SeekToFirst(); iter->Valid(); iter->Next()) { + num_keys++; + } + delete iter; + ASSERT_EQ(kNumKeys, num_keys) << "Bad number of keys"; + + // close database + delete db; + DestroyDB(dbpath, leveldb::Options()); +} + +} // anonymous namespace + +int main(int argc, char** argv) { + return leveldb::test::RunAllTests(); +} diff --git a/util/comparator.cc b/util/comparator.cc index 6cc319242e..4b7b5724ef 100644 --- a/util/comparator.cc +++ b/util/comparator.cc @@ -66,7 +66,7 @@ class BytewiseComparatorImpl : public Comparator { }; } // namespace -static port::OnceType once_comparator = LEVELDB_ONCE_INIT; +static port::OnceType once = LEVELDB_ONCE_INIT; static const Comparator* bytewise; static void InitModule() { @@ -74,7 +74,7 @@ static void InitModule() { } const Comparator* BytewiseComparator() { - port::InitOnce(&once_comparator, InitModule); + port::InitOnce(&once, InitModule); return bytewise; }