Squashed 'src/leveldb/' changes from 7924331..7d41e6f
7d41e6f Merge upstream LevelDB 1.18 803d692 Release 1.18 git-subtree-dir: src/leveldb git-subtree-split: 7d41e6f89ff04ce9e6a742932924796f69c6e23d
This commit is contained in:
parent
938b689846
commit
4b0e2d75d7
36
CONTRIBUTING.md
Normal file
36
CONTRIBUTING.md
Normal file
@ -0,0 +1,36 @@
|
||||
# Contributing
|
||||
|
||||
We'd love to accept your code patches! However, before we can take them, we
|
||||
have to jump a couple of legal hurdles.
|
||||
|
||||
## Contributor License Agreements
|
||||
|
||||
Please fill out either the individual or corporate Contributor License
|
||||
Agreement as appropriate.
|
||||
|
||||
* If you are an individual writing original source code and you're sure you
|
||||
own the intellectual property, then sign an [individual CLA](https://developers.google.com/open-source/cla/individual).
|
||||
* If you work for a company that wants to allow you to contribute your work,
|
||||
then sign a [corporate CLA](https://developers.google.com/open-source/cla/corporate).
|
||||
|
||||
Follow either of the two links above to access the appropriate CLA and
|
||||
instructions for how to sign and return it.
|
||||
|
||||
## Submitting a Patch
|
||||
|
||||
1. Sign the contributors license agreement above.
|
||||
2. Decide which code you want to submit. A submission should be a set of changes
|
||||
that addresses one issue in the [issue tracker](https://github.com/google/leveldb/issues).
|
||||
Please don't mix more than one logical change per submission, because it makes
|
||||
the history hard to follow. If you want to make a change
|
||||
(e.g. add a sample or feature) that doesn't have a corresponding issue in the
|
||||
issue tracker, please create one.
|
||||
3. **Submitting**: When you are ready to submit, send us a Pull Request. Be
|
||||
sure to include the issue number you fixed and the name you used to sign
|
||||
the CLA.
|
||||
|
||||
## Writing Code ##
|
||||
|
||||
If your contribution contains code, please make sure that it follows
|
||||
[the style guide](http://google-styleguide.googlecode.com/svn/trunk/cppguide.xml).
|
||||
Otherwise we will have to ask you to make changes, and that's no fun for anyone.
|
28
Makefile
28
Makefile
@ -6,9 +6,12 @@
|
||||
# Uncomment exactly one of the lines labelled (A), (B), and (C) below
|
||||
# to switch between compilation modes.
|
||||
|
||||
OPT ?= -O2 -DNDEBUG # (A) Production use (optimized mode)
|
||||
# OPT ?= -g2 # (B) Debug mode, w/ full line-level debugging symbols
|
||||
# OPT ?= -O2 -g2 -DNDEBUG # (C) Profiling mode: opt, but w/debugging symbols
|
||||
# (A) Production use (optimized mode)
|
||||
OPT ?= -O2 -DNDEBUG
|
||||
# (B) Debug mode, w/ full line-level debugging symbols
|
||||
# OPT ?= -g2
|
||||
# (C) Profiling mode: opt, but w/debugging symbols
|
||||
# OPT ?= -O2 -g2 -DNDEBUG
|
||||
#-----------------------------------------------
|
||||
|
||||
# detect what platform we're building on
|
||||
@ -29,6 +32,11 @@ MEMENVOBJECTS = $(MEMENV_SOURCES:.cc=.o)
|
||||
TESTUTIL = ./util/testutil.o
|
||||
TESTHARNESS = ./util/testharness.o $(TESTUTIL)
|
||||
|
||||
# Note: iOS should probably be using libtool, not ar.
|
||||
ifeq ($(PLATFORM), IOS)
|
||||
AR=xcrun ar
|
||||
endif
|
||||
|
||||
TESTS = \
|
||||
arena_test \
|
||||
autocompact_test \
|
||||
@ -43,6 +51,7 @@ TESTS = \
|
||||
env_test \
|
||||
filename_test \
|
||||
filter_block_test \
|
||||
hash_test \
|
||||
issue178_test \
|
||||
issue200_test \
|
||||
log_test \
|
||||
@ -72,7 +81,7 @@ SHARED = $(SHARED1)
|
||||
else
|
||||
# Update db.h if you change these.
|
||||
SHARED_MAJOR = 1
|
||||
SHARED_MINOR = 17
|
||||
SHARED_MINOR = 18
|
||||
SHARED1 = libleveldb.$(PLATFORM_SHARED_EXT)
|
||||
SHARED2 = $(SHARED1).$(SHARED_MAJOR)
|
||||
SHARED3 = $(SHARED1).$(SHARED_MAJOR).$(SHARED_MINOR)
|
||||
@ -152,6 +161,9 @@ filename_test: db/filename_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
filter_block_test: table/filter_block_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) $(LDFLAGS) table/filter_block_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
|
||||
|
||||
hash_test: util/hash_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) $(LDFLAGS) util/hash_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
|
||||
|
||||
issue178_test: issues/issue178_test.o $(LIBOBJECTS) $(TESTHARNESS)
|
||||
$(CXX) $(LDFLAGS) issues/issue178_test.o $(LIBOBJECTS) $(TESTHARNESS) -o $@ $(LIBS)
|
||||
|
||||
@ -194,17 +206,17 @@ IOSARCH=-arch armv6 -arch armv7 -arch armv7s -arch arm64
|
||||
|
||||
.cc.o:
|
||||
mkdir -p ios-x86/$(dir $@)
|
||||
$(CXX) $(CXXFLAGS) -isysroot $(SIMULATORROOT)/SDKs/iPhoneSimulator$(IOSVERSION).sdk -arch i686 -arch x86_64 -c $< -o ios-x86/$@
|
||||
xcrun -sdk iphonesimulator $(CXX) $(CXXFLAGS) -isysroot $(SIMULATORROOT)/SDKs/iPhoneSimulator$(IOSVERSION).sdk -arch i686 -arch x86_64 -c $< -o ios-x86/$@
|
||||
mkdir -p ios-arm/$(dir $@)
|
||||
xcrun -sdk iphoneos $(CXX) $(CXXFLAGS) -isysroot $(DEVICEROOT)/SDKs/iPhoneOS$(IOSVERSION).sdk $(IOSARCH) -c $< -o ios-arm/$@
|
||||
lipo ios-x86/$@ ios-arm/$@ -create -output $@
|
||||
xcrun lipo ios-x86/$@ ios-arm/$@ -create -output $@
|
||||
|
||||
.c.o:
|
||||
mkdir -p ios-x86/$(dir $@)
|
||||
$(CC) $(CFLAGS) -isysroot $(SIMULATORROOT)/SDKs/iPhoneSimulator$(IOSVERSION).sdk -arch i686 -arch x86_64 -c $< -o ios-x86/$@
|
||||
xcrun -sdk iphonesimulator $(CC) $(CFLAGS) -isysroot $(SIMULATORROOT)/SDKs/iPhoneSimulator$(IOSVERSION).sdk -arch i686 -arch x86_64 -c $< -o ios-x86/$@
|
||||
mkdir -p ios-arm/$(dir $@)
|
||||
xcrun -sdk iphoneos $(CC) $(CFLAGS) -isysroot $(DEVICEROOT)/SDKs/iPhoneOS$(IOSVERSION).sdk $(IOSARCH) -c $< -o ios-arm/$@
|
||||
lipo ios-x86/$@ ios-arm/$@ -create -output $@
|
||||
xcrun lipo ios-x86/$@ ios-arm/$@ -create -output $@
|
||||
|
||||
else
|
||||
.cc.o:
|
||||
|
138
README.md
Normal file
138
README.md
Normal file
@ -0,0 +1,138 @@
|
||||
**LevelDB is a fast key-value storage library written at Google that provides an ordered mapping from string keys to string values.**
|
||||
|
||||
Authors: Sanjay Ghemawat (sanjay@google.com) and Jeff Dean (jeff@google.com)
|
||||
|
||||
# Features
|
||||
* Keys and values are arbitrary byte arrays.
|
||||
* Data is stored sorted by key.
|
||||
* Callers can provide a custom comparison function to override the sort order.
|
||||
* The basic operations are `Put(key,value)`, `Get(key)`, `Delete(key)`.
|
||||
* Multiple changes can be made in one atomic batch.
|
||||
* Users can create a transient snapshot to get a consistent view of data.
|
||||
* Forward and backward iteration is supported over the data.
|
||||
* Data is automatically compressed using the [Snappy compression library](http://code.google.com/p/snappy).
|
||||
* External activity (file system operations etc.) is relayed through a virtual interface so users can customize the operating system interactions.
|
||||
* [Detailed documentation](http://htmlpreview.github.io/?https://github.com/google/leveldb/blob/master/doc/index.html) about how to use the library is included with the source code.
|
||||
|
||||
|
||||
# Limitations
|
||||
* This is not a SQL database. It does not have a relational data model, it does not support SQL queries, and it has no support for indexes.
|
||||
* Only a single process (possibly multi-threaded) can access a particular database at a time.
|
||||
* There is no client-server support builtin to the library. An application that needs such support will have to wrap their own server around the library.
|
||||
|
||||
# Performance
|
||||
|
||||
Here is a performance report (with explanations) from the run of the
|
||||
included db_bench program. The results are somewhat noisy, but should
|
||||
be enough to get a ballpark performance estimate.
|
||||
|
||||
## Setup
|
||||
|
||||
We use a database with a million entries. Each entry has a 16 byte
|
||||
key, and a 100 byte value. Values used by the benchmark compress to
|
||||
about half their original size.
|
||||
|
||||
LevelDB: version 1.1
|
||||
Date: Sun May 1 12:11:26 2011
|
||||
CPU: 4 x Intel(R) Core(TM)2 Quad CPU Q6600 @ 2.40GHz
|
||||
CPUCache: 4096 KB
|
||||
Keys: 16 bytes each
|
||||
Values: 100 bytes each (50 bytes after compression)
|
||||
Entries: 1000000
|
||||
Raw Size: 110.6 MB (estimated)
|
||||
File Size: 62.9 MB (estimated)
|
||||
|
||||
## Write performance
|
||||
|
||||
The "fill" benchmarks create a brand new database, in either
|
||||
sequential, or random order. The "fillsync" benchmark flushes data
|
||||
from the operating system to the disk after every operation; the other
|
||||
write operations leave the data sitting in the operating system buffer
|
||||
cache for a while. The "overwrite" benchmark does random writes that
|
||||
update existing keys in the database.
|
||||
|
||||
fillseq : 1.765 micros/op; 62.7 MB/s
|
||||
fillsync : 268.409 micros/op; 0.4 MB/s (10000 ops)
|
||||
fillrandom : 2.460 micros/op; 45.0 MB/s
|
||||
overwrite : 2.380 micros/op; 46.5 MB/s
|
||||
|
||||
Each "op" above corresponds to a write of a single key/value pair.
|
||||
I.e., a random write benchmark goes at approximately 400,000 writes per second.
|
||||
|
||||
Each "fillsync" operation costs much less (0.3 millisecond)
|
||||
than a disk seek (typically 10 milliseconds). We suspect that this is
|
||||
because the hard disk itself is buffering the update in its memory and
|
||||
responding before the data has been written to the platter. This may
|
||||
or may not be safe based on whether or not the hard disk has enough
|
||||
power to save its memory in the event of a power failure.
|
||||
|
||||
## Read performance
|
||||
|
||||
We list the performance of reading sequentially in both the forward
|
||||
and reverse direction, and also the performance of a random lookup.
|
||||
Note that the database created by the benchmark is quite small.
|
||||
Therefore the report characterizes the performance of leveldb when the
|
||||
working set fits in memory. The cost of reading a piece of data that
|
||||
is not present in the operating system buffer cache will be dominated
|
||||
by the one or two disk seeks needed to fetch the data from disk.
|
||||
Write performance will be mostly unaffected by whether or not the
|
||||
working set fits in memory.
|
||||
|
||||
readrandom : 16.677 micros/op; (approximately 60,000 reads per second)
|
||||
readseq : 0.476 micros/op; 232.3 MB/s
|
||||
readreverse : 0.724 micros/op; 152.9 MB/s
|
||||
|
||||
LevelDB compacts its underlying storage data in the background to
|
||||
improve read performance. The results listed above were done
|
||||
immediately after a lot of random writes. The results after
|
||||
compactions (which are usually triggered automatically) are better.
|
||||
|
||||
readrandom : 11.602 micros/op; (approximately 85,000 reads per second)
|
||||
readseq : 0.423 micros/op; 261.8 MB/s
|
||||
readreverse : 0.663 micros/op; 166.9 MB/s
|
||||
|
||||
Some of the high cost of reads comes from repeated decompression of blocks
|
||||
read from disk. If we supply enough cache to the leveldb so it can hold the
|
||||
uncompressed blocks in memory, the read performance improves again:
|
||||
|
||||
readrandom : 9.775 micros/op; (approximately 100,000 reads per second before compaction)
|
||||
readrandom : 5.215 micros/op; (approximately 190,000 reads per second after compaction)
|
||||
|
||||
## Repository contents
|
||||
|
||||
See doc/index.html for more explanation. See doc/impl.html for a brief overview of the implementation.
|
||||
|
||||
The public interface is in include/*.h. Callers should not include or
|
||||
rely on the details of any other header files in this package. Those
|
||||
internal APIs may be changed without warning.
|
||||
|
||||
Guide to header files:
|
||||
|
||||
* **include/db.h**: Main interface to the DB: Start here
|
||||
|
||||
* **include/options.h**: Control over the behavior of an entire database,
|
||||
and also control over the behavior of individual reads and writes.
|
||||
|
||||
* **include/comparator.h**: Abstraction for user-specified comparison function.
|
||||
If you want just bytewise comparison of keys, you can use the default
|
||||
comparator, but clients can write their own comparator implementations if they
|
||||
want custom ordering (e.g. to handle different character encodings, etc.)
|
||||
|
||||
* **include/iterator.h**: Interface for iterating over data. You can get
|
||||
an iterator from a DB object.
|
||||
|
||||
* **include/write_batch.h**: Interface for atomically applying multiple
|
||||
updates to a database.
|
||||
|
||||
* **include/slice.h**: A simple module for maintaining a pointer and a
|
||||
length into some other byte array.
|
||||
|
||||
* **include/status.h**: Status is returned from many of the public interfaces
|
||||
and is used to report success and various kinds of errors.
|
||||
|
||||
* **include/env.h**:
|
||||
Abstraction of the OS environment. A posix implementation of this interface is
|
||||
in util/env_posix.cc
|
||||
|
||||
* **include/table.h, include/table_builder.h**: Lower-level modules that most
|
||||
clients probably won't use directly
|
@ -20,7 +20,7 @@
|
||||
#
|
||||
# The PLATFORM_CCFLAGS and PLATFORM_CXXFLAGS might include the following:
|
||||
#
|
||||
# -DLEVELDB_CSTDATOMIC_PRESENT if <cstdatomic> is present
|
||||
# -DLEVELDB_ATOMIC_PRESENT if <atomic> is present
|
||||
# -DLEVELDB_PLATFORM_POSIX for Posix-based platforms
|
||||
# -DSNAPPY if the Snappy library is present
|
||||
#
|
||||
@ -72,6 +72,12 @@ if [ "$CXX" = "g++" ]; then
|
||||
fi
|
||||
|
||||
case "$TARGET_OS" in
|
||||
CYGWIN_*)
|
||||
PLATFORM=OS_LINUX
|
||||
COMMON_FLAGS="$MEMCMP_FLAG -lpthread -DOS_LINUX -DCYGWIN"
|
||||
PLATFORM_LDFLAGS="-lpthread"
|
||||
PORT_FILE=port/port_posix.cc
|
||||
;;
|
||||
Darwin)
|
||||
PLATFORM=OS_MACOSX
|
||||
COMMON_FLAGS="$MEMCMP_FLAG -DOS_MACOSX"
|
||||
@ -185,13 +191,14 @@ if [ "$CROSS_COMPILE" = "true" ]; then
|
||||
else
|
||||
CXXOUTPUT="${TMPDIR}/leveldb_build_detect_platform-cxx.$$"
|
||||
|
||||
# If -std=c++0x works, use <cstdatomic>. Otherwise use port_posix.h.
|
||||
# If -std=c++0x works, use <atomic> as fallback for when memory barriers
|
||||
# are not available.
|
||||
$CXX $CXXFLAGS -std=c++0x -x c++ - -o $CXXOUTPUT 2>/dev/null <<EOF
|
||||
#include <cstdatomic>
|
||||
#include <atomic>
|
||||
int main() {}
|
||||
EOF
|
||||
if [ "$?" = 0 ]; then
|
||||
COMMON_FLAGS="$COMMON_FLAGS -DLEVELDB_PLATFORM_POSIX -DLEVELDB_CSTDATOMIC_PRESENT"
|
||||
COMMON_FLAGS="$COMMON_FLAGS -DLEVELDB_PLATFORM_POSIX -DLEVELDB_ATOMIC_PRESENT"
|
||||
PLATFORM_CXXFLAGS="-std=c++0x"
|
||||
else
|
||||
COMMON_FLAGS="$COMMON_FLAGS -DLEVELDB_PLATFORM_POSIX"
|
||||
|
@ -431,7 +431,7 @@ class Benchmark {
|
||||
benchmarks = sep + 1;
|
||||
}
|
||||
|
||||
// Reset parameters that may be overriddden bwlow
|
||||
// Reset parameters that may be overridden below
|
||||
num_ = FLAGS_num;
|
||||
reads_ = (FLAGS_reads < 0 ? FLAGS_num : FLAGS_reads);
|
||||
value_size_ = FLAGS_value_size;
|
||||
@ -811,7 +811,6 @@ class Benchmark {
|
||||
|
||||
void SeekRandom(ThreadState* thread) {
|
||||
ReadOptions options;
|
||||
std::string value;
|
||||
int found = 0;
|
||||
for (int i = 0; i < reads_; i++) {
|
||||
Iterator* iter = db_->NewIterator(options);
|
||||
|
@ -392,7 +392,7 @@ Status DBImpl::RecoverLogFile(uint64_t log_number,
|
||||
reporter.info_log = options_.info_log;
|
||||
reporter.fname = fname.c_str();
|
||||
reporter.status = (options_.paranoid_checks ? &status : NULL);
|
||||
// We intentially make log::Reader do checksumming even if
|
||||
// We intentionally make log::Reader do checksumming even if
|
||||
// paranoid_checks==false so that corruptions cause entire commits
|
||||
// to be skipped instead of propagating bad information (like overly
|
||||
// large sequence numbers).
|
||||
@ -1267,7 +1267,7 @@ WriteBatch* DBImpl::BuildBatchGroup(Writer** last_writer) {
|
||||
break;
|
||||
}
|
||||
|
||||
// Append to *reuslt
|
||||
// Append to *result
|
||||
if (result == first->batch) {
|
||||
// Switch to temporary batch instead of disturbing caller's batch
|
||||
result = tmp_batch_;
|
||||
|
@ -626,7 +626,7 @@ TEST(DBTest, GetEncountersEmptyLevel) {
|
||||
// * sstable B in level 2
|
||||
// Then do enough Get() calls to arrange for an automatic compaction
|
||||
// of sstable A. A bug would cause the compaction to be marked as
|
||||
// occuring at level 1 (instead of the correct level 0).
|
||||
// occurring at level 1 (instead of the correct level 0).
|
||||
|
||||
// Step 1: First place sstables in levels 0 and 2
|
||||
int compaction_count = 0;
|
||||
|
@ -2,8 +2,8 @@
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#ifndef STORAGE_LEVELDB_DB_FORMAT_H_
|
||||
#define STORAGE_LEVELDB_DB_FORMAT_H_
|
||||
#ifndef STORAGE_LEVELDB_DB_DBFORMAT_H_
|
||||
#define STORAGE_LEVELDB_DB_DBFORMAT_H_
|
||||
|
||||
#include <stdio.h>
|
||||
#include "leveldb/comparator.h"
|
||||
@ -227,4 +227,4 @@ inline LookupKey::~LookupKey() {
|
||||
|
||||
} // namespace leveldb
|
||||
|
||||
#endif // STORAGE_LEVELDB_DB_FORMAT_H_
|
||||
#endif // STORAGE_LEVELDB_DB_DBFORMAT_H_
|
||||
|
225
db/dumpfile.cc
Normal file
225
db/dumpfile.cc
Normal file
@ -0,0 +1,225 @@
|
||||
// Copyright (c) 2012 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include <stdio.h>
|
||||
#include "db/dbformat.h"
|
||||
#include "db/filename.h"
|
||||
#include "db/log_reader.h"
|
||||
#include "db/version_edit.h"
|
||||
#include "db/write_batch_internal.h"
|
||||
#include "leveldb/env.h"
|
||||
#include "leveldb/iterator.h"
|
||||
#include "leveldb/options.h"
|
||||
#include "leveldb/status.h"
|
||||
#include "leveldb/table.h"
|
||||
#include "leveldb/write_batch.h"
|
||||
#include "util/logging.h"
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
namespace {
|
||||
|
||||
bool GuessType(const std::string& fname, FileType* type) {
|
||||
size_t pos = fname.rfind('/');
|
||||
std::string basename;
|
||||
if (pos == std::string::npos) {
|
||||
basename = fname;
|
||||
} else {
|
||||
basename = std::string(fname.data() + pos + 1, fname.size() - pos - 1);
|
||||
}
|
||||
uint64_t ignored;
|
||||
return ParseFileName(basename, &ignored, type);
|
||||
}
|
||||
|
||||
// Notified when log reader encounters corruption.
|
||||
class CorruptionReporter : public log::Reader::Reporter {
|
||||
public:
|
||||
WritableFile* dst_;
|
||||
virtual void Corruption(size_t bytes, const Status& status) {
|
||||
std::string r = "corruption: ";
|
||||
AppendNumberTo(&r, bytes);
|
||||
r += " bytes; ";
|
||||
r += status.ToString();
|
||||
r.push_back('\n');
|
||||
dst_->Append(r);
|
||||
}
|
||||
};
|
||||
|
||||
// Print contents of a log file. (*func)() is called on every record.
|
||||
Status PrintLogContents(Env* env, const std::string& fname,
|
||||
void (*func)(uint64_t, Slice, WritableFile*),
|
||||
WritableFile* dst) {
|
||||
SequentialFile* file;
|
||||
Status s = env->NewSequentialFile(fname, &file);
|
||||
if (!s.ok()) {
|
||||
return s;
|
||||
}
|
||||
CorruptionReporter reporter;
|
||||
reporter.dst_ = dst;
|
||||
log::Reader reader(file, &reporter, true, 0);
|
||||
Slice record;
|
||||
std::string scratch;
|
||||
while (reader.ReadRecord(&record, &scratch)) {
|
||||
(*func)(reader.LastRecordOffset(), record, dst);
|
||||
}
|
||||
delete file;
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
// Called on every item found in a WriteBatch.
|
||||
class WriteBatchItemPrinter : public WriteBatch::Handler {
|
||||
public:
|
||||
WritableFile* dst_;
|
||||
virtual void Put(const Slice& key, const Slice& value) {
|
||||
std::string r = " put '";
|
||||
AppendEscapedStringTo(&r, key);
|
||||
r += "' '";
|
||||
AppendEscapedStringTo(&r, value);
|
||||
r += "'\n";
|
||||
dst_->Append(r);
|
||||
}
|
||||
virtual void Delete(const Slice& key) {
|
||||
std::string r = " del '";
|
||||
AppendEscapedStringTo(&r, key);
|
||||
r += "'\n";
|
||||
dst_->Append(r);
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
// Called on every log record (each one of which is a WriteBatch)
|
||||
// found in a kLogFile.
|
||||
static void WriteBatchPrinter(uint64_t pos, Slice record, WritableFile* dst) {
|
||||
std::string r = "--- offset ";
|
||||
AppendNumberTo(&r, pos);
|
||||
r += "; ";
|
||||
if (record.size() < 12) {
|
||||
r += "log record length ";
|
||||
AppendNumberTo(&r, record.size());
|
||||
r += " is too small\n";
|
||||
dst->Append(r);
|
||||
return;
|
||||
}
|
||||
WriteBatch batch;
|
||||
WriteBatchInternal::SetContents(&batch, record);
|
||||
r += "sequence ";
|
||||
AppendNumberTo(&r, WriteBatchInternal::Sequence(&batch));
|
||||
r.push_back('\n');
|
||||
dst->Append(r);
|
||||
WriteBatchItemPrinter batch_item_printer;
|
||||
batch_item_printer.dst_ = dst;
|
||||
Status s = batch.Iterate(&batch_item_printer);
|
||||
if (!s.ok()) {
|
||||
dst->Append(" error: " + s.ToString() + "\n");
|
||||
}
|
||||
}
|
||||
|
||||
Status DumpLog(Env* env, const std::string& fname, WritableFile* dst) {
|
||||
return PrintLogContents(env, fname, WriteBatchPrinter, dst);
|
||||
}
|
||||
|
||||
// Called on every log record (each one of which is a WriteBatch)
|
||||
// found in a kDescriptorFile.
|
||||
static void VersionEditPrinter(uint64_t pos, Slice record, WritableFile* dst) {
|
||||
std::string r = "--- offset ";
|
||||
AppendNumberTo(&r, pos);
|
||||
r += "; ";
|
||||
VersionEdit edit;
|
||||
Status s = edit.DecodeFrom(record);
|
||||
if (!s.ok()) {
|
||||
r += s.ToString();
|
||||
r.push_back('\n');
|
||||
} else {
|
||||
r += edit.DebugString();
|
||||
}
|
||||
dst->Append(r);
|
||||
}
|
||||
|
||||
Status DumpDescriptor(Env* env, const std::string& fname, WritableFile* dst) {
|
||||
return PrintLogContents(env, fname, VersionEditPrinter, dst);
|
||||
}
|
||||
|
||||
Status DumpTable(Env* env, const std::string& fname, WritableFile* dst) {
|
||||
uint64_t file_size;
|
||||
RandomAccessFile* file = NULL;
|
||||
Table* table = NULL;
|
||||
Status s = env->GetFileSize(fname, &file_size);
|
||||
if (s.ok()) {
|
||||
s = env->NewRandomAccessFile(fname, &file);
|
||||
}
|
||||
if (s.ok()) {
|
||||
// We use the default comparator, which may or may not match the
|
||||
// comparator used in this database. However this should not cause
|
||||
// problems since we only use Table operations that do not require
|
||||
// any comparisons. In particular, we do not call Seek or Prev.
|
||||
s = Table::Open(Options(), file, file_size, &table);
|
||||
}
|
||||
if (!s.ok()) {
|
||||
delete table;
|
||||
delete file;
|
||||
return s;
|
||||
}
|
||||
|
||||
ReadOptions ro;
|
||||
ro.fill_cache = false;
|
||||
Iterator* iter = table->NewIterator(ro);
|
||||
std::string r;
|
||||
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
|
||||
r.clear();
|
||||
ParsedInternalKey key;
|
||||
if (!ParseInternalKey(iter->key(), &key)) {
|
||||
r = "badkey '";
|
||||
AppendEscapedStringTo(&r, iter->key());
|
||||
r += "' => '";
|
||||
AppendEscapedStringTo(&r, iter->value());
|
||||
r += "'\n";
|
||||
dst->Append(r);
|
||||
} else {
|
||||
r = "'";
|
||||
AppendEscapedStringTo(&r, key.user_key);
|
||||
r += "' @ ";
|
||||
AppendNumberTo(&r, key.sequence);
|
||||
r += " : ";
|
||||
if (key.type == kTypeDeletion) {
|
||||
r += "del";
|
||||
} else if (key.type == kTypeValue) {
|
||||
r += "val";
|
||||
} else {
|
||||
AppendNumberTo(&r, key.type);
|
||||
}
|
||||
r += " => '";
|
||||
AppendEscapedStringTo(&r, iter->value());
|
||||
r += "'\n";
|
||||
dst->Append(r);
|
||||
}
|
||||
}
|
||||
s = iter->status();
|
||||
if (!s.ok()) {
|
||||
dst->Append("iterator error: " + s.ToString() + "\n");
|
||||
}
|
||||
|
||||
delete iter;
|
||||
delete table;
|
||||
delete file;
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
Status DumpFile(Env* env, const std::string& fname, WritableFile* dst) {
|
||||
FileType ftype;
|
||||
if (!GuessType(fname, &ftype)) {
|
||||
return Status::InvalidArgument(fname + ": unknown file type");
|
||||
}
|
||||
switch (ftype) {
|
||||
case kLogFile: return DumpLog(env, fname, dst);
|
||||
case kDescriptorFile: return DumpDescriptor(env, fname, dst);
|
||||
case kTableFile: return DumpTable(env, fname, dst);
|
||||
default:
|
||||
break;
|
||||
}
|
||||
return Status::InvalidArgument(fname + ": not a dump-able file type");
|
||||
}
|
||||
|
||||
} // namespace leveldb
|
@ -3,212 +3,38 @@
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include <stdio.h>
|
||||
#include "db/dbformat.h"
|
||||
#include "db/filename.h"
|
||||
#include "db/log_reader.h"
|
||||
#include "db/version_edit.h"
|
||||
#include "db/write_batch_internal.h"
|
||||
#include "leveldb/dumpfile.h"
|
||||
#include "leveldb/env.h"
|
||||
#include "leveldb/iterator.h"
|
||||
#include "leveldb/options.h"
|
||||
#include "leveldb/status.h"
|
||||
#include "leveldb/table.h"
|
||||
#include "leveldb/write_batch.h"
|
||||
#include "util/logging.h"
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
namespace {
|
||||
|
||||
bool GuessType(const std::string& fname, FileType* type) {
|
||||
size_t pos = fname.rfind('/');
|
||||
std::string basename;
|
||||
if (pos == std::string::npos) {
|
||||
basename = fname;
|
||||
} else {
|
||||
basename = std::string(fname.data() + pos + 1, fname.size() - pos - 1);
|
||||
}
|
||||
uint64_t ignored;
|
||||
return ParseFileName(basename, &ignored, type);
|
||||
}
|
||||
|
||||
// Notified when log reader encounters corruption.
|
||||
class CorruptionReporter : public log::Reader::Reporter {
|
||||
class StdoutPrinter : public WritableFile {
|
||||
public:
|
||||
virtual void Corruption(size_t bytes, const Status& status) {
|
||||
printf("corruption: %d bytes; %s\n",
|
||||
static_cast<int>(bytes),
|
||||
status.ToString().c_str());
|
||||
virtual Status Append(const Slice& data) {
|
||||
fwrite(data.data(), 1, data.size(), stdout);
|
||||
return Status::OK();
|
||||
}
|
||||
virtual Status Close() { return Status::OK(); }
|
||||
virtual Status Flush() { return Status::OK(); }
|
||||
virtual Status Sync() { return Status::OK(); }
|
||||
};
|
||||
|
||||
// Print contents of a log file. (*func)() is called on every record.
|
||||
bool PrintLogContents(Env* env, const std::string& fname,
|
||||
void (*func)(Slice)) {
|
||||
SequentialFile* file;
|
||||
Status s = env->NewSequentialFile(fname, &file);
|
||||
if (!s.ok()) {
|
||||
fprintf(stderr, "%s\n", s.ToString().c_str());
|
||||
return false;
|
||||
}
|
||||
CorruptionReporter reporter;
|
||||
log::Reader reader(file, &reporter, true, 0);
|
||||
Slice record;
|
||||
std::string scratch;
|
||||
while (reader.ReadRecord(&record, &scratch)) {
|
||||
printf("--- offset %llu; ",
|
||||
static_cast<unsigned long long>(reader.LastRecordOffset()));
|
||||
(*func)(record);
|
||||
}
|
||||
delete file;
|
||||
return true;
|
||||
}
|
||||
|
||||
// Called on every item found in a WriteBatch.
|
||||
class WriteBatchItemPrinter : public WriteBatch::Handler {
|
||||
public:
|
||||
uint64_t offset_;
|
||||
uint64_t sequence_;
|
||||
|
||||
virtual void Put(const Slice& key, const Slice& value) {
|
||||
printf(" put '%s' '%s'\n",
|
||||
EscapeString(key).c_str(),
|
||||
EscapeString(value).c_str());
|
||||
}
|
||||
virtual void Delete(const Slice& key) {
|
||||
printf(" del '%s'\n",
|
||||
EscapeString(key).c_str());
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
// Called on every log record (each one of which is a WriteBatch)
|
||||
// found in a kLogFile.
|
||||
static void WriteBatchPrinter(Slice record) {
|
||||
if (record.size() < 12) {
|
||||
printf("log record length %d is too small\n",
|
||||
static_cast<int>(record.size()));
|
||||
return;
|
||||
}
|
||||
WriteBatch batch;
|
||||
WriteBatchInternal::SetContents(&batch, record);
|
||||
printf("sequence %llu\n",
|
||||
static_cast<unsigned long long>(WriteBatchInternal::Sequence(&batch)));
|
||||
WriteBatchItemPrinter batch_item_printer;
|
||||
Status s = batch.Iterate(&batch_item_printer);
|
||||
if (!s.ok()) {
|
||||
printf(" error: %s\n", s.ToString().c_str());
|
||||
}
|
||||
}
|
||||
|
||||
bool DumpLog(Env* env, const std::string& fname) {
|
||||
return PrintLogContents(env, fname, WriteBatchPrinter);
|
||||
}
|
||||
|
||||
// Called on every log record (each one of which is a WriteBatch)
|
||||
// found in a kDescriptorFile.
|
||||
static void VersionEditPrinter(Slice record) {
|
||||
VersionEdit edit;
|
||||
Status s = edit.DecodeFrom(record);
|
||||
if (!s.ok()) {
|
||||
printf("%s\n", s.ToString().c_str());
|
||||
return;
|
||||
}
|
||||
printf("%s", edit.DebugString().c_str());
|
||||
}
|
||||
|
||||
bool DumpDescriptor(Env* env, const std::string& fname) {
|
||||
return PrintLogContents(env, fname, VersionEditPrinter);
|
||||
}
|
||||
|
||||
bool DumpTable(Env* env, const std::string& fname) {
|
||||
uint64_t file_size;
|
||||
RandomAccessFile* file = NULL;
|
||||
Table* table = NULL;
|
||||
Status s = env->GetFileSize(fname, &file_size);
|
||||
if (s.ok()) {
|
||||
s = env->NewRandomAccessFile(fname, &file);
|
||||
}
|
||||
if (s.ok()) {
|
||||
// We use the default comparator, which may or may not match the
|
||||
// comparator used in this database. However this should not cause
|
||||
// problems since we only use Table operations that do not require
|
||||
// any comparisons. In particular, we do not call Seek or Prev.
|
||||
s = Table::Open(Options(), file, file_size, &table);
|
||||
}
|
||||
if (!s.ok()) {
|
||||
fprintf(stderr, "%s\n", s.ToString().c_str());
|
||||
delete table;
|
||||
delete file;
|
||||
return false;
|
||||
}
|
||||
|
||||
ReadOptions ro;
|
||||
ro.fill_cache = false;
|
||||
Iterator* iter = table->NewIterator(ro);
|
||||
for (iter->SeekToFirst(); iter->Valid(); iter->Next()) {
|
||||
ParsedInternalKey key;
|
||||
if (!ParseInternalKey(iter->key(), &key)) {
|
||||
printf("badkey '%s' => '%s'\n",
|
||||
EscapeString(iter->key()).c_str(),
|
||||
EscapeString(iter->value()).c_str());
|
||||
} else {
|
||||
char kbuf[20];
|
||||
const char* type;
|
||||
if (key.type == kTypeDeletion) {
|
||||
type = "del";
|
||||
} else if (key.type == kTypeValue) {
|
||||
type = "val";
|
||||
} else {
|
||||
snprintf(kbuf, sizeof(kbuf), "%d", static_cast<int>(key.type));
|
||||
type = kbuf;
|
||||
}
|
||||
printf("'%s' @ %8llu : %s => '%s'\n",
|
||||
EscapeString(key.user_key).c_str(),
|
||||
static_cast<unsigned long long>(key.sequence),
|
||||
type,
|
||||
EscapeString(iter->value()).c_str());
|
||||
}
|
||||
}
|
||||
s = iter->status();
|
||||
if (!s.ok()) {
|
||||
printf("iterator error: %s\n", s.ToString().c_str());
|
||||
}
|
||||
|
||||
delete iter;
|
||||
delete table;
|
||||
delete file;
|
||||
return true;
|
||||
}
|
||||
|
||||
bool DumpFile(Env* env, const std::string& fname) {
|
||||
FileType ftype;
|
||||
if (!GuessType(fname, &ftype)) {
|
||||
fprintf(stderr, "%s: unknown file type\n", fname.c_str());
|
||||
return false;
|
||||
}
|
||||
switch (ftype) {
|
||||
case kLogFile: return DumpLog(env, fname);
|
||||
case kDescriptorFile: return DumpDescriptor(env, fname);
|
||||
case kTableFile: return DumpTable(env, fname);
|
||||
|
||||
default: {
|
||||
fprintf(stderr, "%s: not a dump-able file type\n", fname.c_str());
|
||||
break;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
bool HandleDumpCommand(Env* env, char** files, int num) {
|
||||
StdoutPrinter printer;
|
||||
bool ok = true;
|
||||
for (int i = 0; i < num; i++) {
|
||||
ok &= DumpFile(env, files[i]);
|
||||
Status s = DumpFile(env, files[i], &printer);
|
||||
if (!s.ok()) {
|
||||
fprintf(stderr, "%s\n", s.ToString().c_str());
|
||||
ok = false;
|
||||
}
|
||||
}
|
||||
return ok;
|
||||
}
|
||||
|
||||
}
|
||||
} // namespace
|
||||
} // namespace leveldb
|
||||
|
||||
static void Usage() {
|
||||
|
@ -26,8 +26,8 @@ static const int kMaxRecordType = kLastType;
|
||||
|
||||
static const int kBlockSize = 32768;
|
||||
|
||||
// Header is checksum (4 bytes), type (1 byte), length (2 bytes).
|
||||
static const int kHeaderSize = 4 + 1 + 2;
|
||||
// Header is checksum (4 bytes), length (2 bytes), type (1 byte).
|
||||
static const int kHeaderSize = 4 + 2 + 1;
|
||||
|
||||
} // namespace log
|
||||
} // namespace leveldb
|
||||
|
@ -167,14 +167,14 @@ uint64_t Reader::LastRecordOffset() {
|
||||
return last_record_offset_;
|
||||
}
|
||||
|
||||
void Reader::ReportCorruption(size_t bytes, const char* reason) {
|
||||
void Reader::ReportCorruption(uint64_t bytes, const char* reason) {
|
||||
ReportDrop(bytes, Status::Corruption(reason));
|
||||
}
|
||||
|
||||
void Reader::ReportDrop(size_t bytes, const Status& reason) {
|
||||
void Reader::ReportDrop(uint64_t bytes, const Status& reason) {
|
||||
if (reporter_ != NULL &&
|
||||
end_of_buffer_offset_ - buffer_.size() - bytes >= initial_offset_) {
|
||||
reporter_->Corruption(bytes, reason);
|
||||
reporter_->Corruption(static_cast<size_t>(bytes), reason);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -94,8 +94,8 @@ class Reader {
|
||||
|
||||
// Reports dropped bytes to the reporter.
|
||||
// buffer_ must be updated to remove the dropped bytes prior to invocation.
|
||||
void ReportCorruption(size_t bytes, const char* reason);
|
||||
void ReportDrop(size_t bytes, const Status& reason);
|
||||
void ReportCorruption(uint64_t bytes, const char* reason);
|
||||
void ReportDrop(uint64_t bytes, const Status& reason);
|
||||
|
||||
// No copying allowed
|
||||
Reader(const Reader&);
|
||||
|
@ -463,7 +463,7 @@ TEST(LogTest, ErrorJoinsRecords) {
|
||||
|
||||
ASSERT_EQ("correct", Read());
|
||||
ASSERT_EQ("EOF", Read());
|
||||
const int dropped = DroppedBytes();
|
||||
const size_t dropped = DroppedBytes();
|
||||
ASSERT_LE(dropped, 2*kBlockSize + 100);
|
||||
ASSERT_GE(dropped, 2*kBlockSize);
|
||||
}
|
||||
|
@ -186,7 +186,7 @@ class Repairer {
|
||||
reporter.env = env_;
|
||||
reporter.info_log = options_.info_log;
|
||||
reporter.lognum = log;
|
||||
// We intentially make log::Reader do checksumming so that
|
||||
// We intentionally make log::Reader do checksumming so that
|
||||
// corruptions cause entire commits to be skipped instead of
|
||||
// propagating bad information (like overly large sequence
|
||||
// numbers).
|
||||
|
@ -1,3 +1,6 @@
|
||||
#ifndef STORAGE_LEVELDB_DB_SKIPLIST_H_
|
||||
#define STORAGE_LEVELDB_DB_SKIPLIST_H_
|
||||
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
@ -377,3 +380,5 @@ bool SkipList<Key,Comparator>::Contains(const Key& key) const {
|
||||
}
|
||||
|
||||
} // namespace leveldb
|
||||
|
||||
#endif // STORAGE_LEVELDB_DB_SKIPLIST_H_
|
||||
|
@ -21,10 +21,10 @@ class WriteBatchInternal {
|
||||
// Set the count for the number of entries in the batch.
|
||||
static void SetCount(WriteBatch* batch, int n);
|
||||
|
||||
// Return the seqeunce number for the start of this batch.
|
||||
// Return the sequence number for the start of this batch.
|
||||
static SequenceNumber Sequence(const WriteBatch* batch);
|
||||
|
||||
// Store the specified number as the seqeunce number for the start of
|
||||
// Store the specified number as the sequence number for the start of
|
||||
// this batch.
|
||||
static void SetSequence(WriteBatch* batch, SequenceNumber seq);
|
||||
|
||||
|
@ -338,7 +338,7 @@ class Benchmark {
|
||||
bool write_sync = false;
|
||||
if (name == Slice("fillseq")) {
|
||||
Write(write_sync, SEQUENTIAL, FRESH, num_, FLAGS_value_size, 1);
|
||||
|
||||
DBSynchronize(db_);
|
||||
} else if (name == Slice("fillrandom")) {
|
||||
Write(write_sync, RANDOM, FRESH, num_, FLAGS_value_size, 1);
|
||||
DBSynchronize(db_);
|
||||
|
@ -111,7 +111,7 @@ A compaction merges the contents of the picked files to produce a
|
||||
sequence of level-(L+1) files. We switch to producing a new
|
||||
level-(L+1) file after the current output file has reached the target
|
||||
file size (2MB). We also switch to a new output file when the key
|
||||
range of the current output file has grown enough to overlap more then
|
||||
range of the current output file has grown enough to overlap more than
|
||||
ten level-(L+2) files. This last rule ensures that a later compaction
|
||||
of a level-(L+1) file will not pick up too much data from level-(L+2).
|
||||
|
||||
@ -151,7 +151,7 @@ compaction cost will be approximately 0.5 second.
|
||||
If we throttle the background writing to something small, say 10% of
|
||||
the full 100MB/s speed, a compaction may take up to 5 seconds. If the
|
||||
user is writing at 10MB/s, we might build up lots of level-0 files
|
||||
(~50 to hold the 5*10MB). This may signficantly increase the cost of
|
||||
(~50 to hold the 5*10MB). This may significantly increase the cost of
|
||||
reads due to the overhead of merging more files together on every
|
||||
read.
|
||||
|
||||
|
@ -11,7 +11,7 @@ Each block consists of a sequence of records:
|
||||
|
||||
A record never starts within the last six bytes of a block (since it
|
||||
won't fit). Any leftover bytes here form the trailer, which must
|
||||
consist entirely of zero bytes and must be skipped by readers.
|
||||
consist entirely of zero bytes and must be skipped by readers.
|
||||
|
||||
Aside: if exactly seven bytes are left in the current block, and a new
|
||||
non-zero length record is added, the writer must emit a FIRST record
|
||||
@ -33,8 +33,8 @@ The FULL record contains the contents of an entire user record.
|
||||
FIRST, MIDDLE, LAST are types used for user records that have been
|
||||
split into multiple fragments (typically because of block boundaries).
|
||||
FIRST is the type of the first fragment of a user record, LAST is the
|
||||
type of the last fragment of a user record, and MID is the type of all
|
||||
interior fragments of a user record.
|
||||
type of the last fragment of a user record, and MIDDLE is the type of
|
||||
all interior fragments of a user record.
|
||||
|
||||
Example: consider a sequence of user records:
|
||||
A: length 1000
|
||||
|
@ -55,14 +55,15 @@ class FileState {
|
||||
}
|
||||
const uint64_t available = size_ - offset;
|
||||
if (n > available) {
|
||||
n = available;
|
||||
n = static_cast<size_t>(available);
|
||||
}
|
||||
if (n == 0) {
|
||||
*result = Slice();
|
||||
return Status::OK();
|
||||
}
|
||||
|
||||
size_t block = offset / kBlockSize;
|
||||
assert(offset / kBlockSize <= SIZE_MAX);
|
||||
size_t block = static_cast<size_t>(offset / kBlockSize);
|
||||
size_t block_offset = offset % kBlockSize;
|
||||
|
||||
if (n <= kBlockSize - block_offset) {
|
||||
@ -167,7 +168,7 @@ class SequentialFileImpl : public SequentialFile {
|
||||
if (pos_ > file_->Size()) {
|
||||
return Status::IOError("pos_ > file_->Size()");
|
||||
}
|
||||
const size_t available = file_->Size() - pos_;
|
||||
const uint64_t available = file_->Size() - pos_;
|
||||
if (n > available) {
|
||||
n = available;
|
||||
}
|
||||
@ -177,7 +178,7 @@ class SequentialFileImpl : public SequentialFile {
|
||||
|
||||
private:
|
||||
FileState* file_;
|
||||
size_t pos_;
|
||||
uint64_t pos_;
|
||||
};
|
||||
|
||||
class RandomAccessFileImpl : public RandomAccessFile {
|
||||
|
@ -96,4 +96,4 @@ class Cache {
|
||||
|
||||
} // namespace leveldb
|
||||
|
||||
#endif // STORAGE_LEVELDB_UTIL_CACHE_H_
|
||||
#endif // STORAGE_LEVELDB_INCLUDE_CACHE_H_
|
||||
|
@ -14,7 +14,7 @@ namespace leveldb {
|
||||
|
||||
// Update Makefile if you change these
|
||||
static const int kMajorVersion = 1;
|
||||
static const int kMinorVersion = 17;
|
||||
static const int kMinorVersion = 18;
|
||||
|
||||
struct Options;
|
||||
struct ReadOptions;
|
||||
|
25
include/leveldb/dumpfile.h
Normal file
25
include/leveldb/dumpfile.h
Normal file
@ -0,0 +1,25 @@
|
||||
// Copyright (c) 2014 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#ifndef STORAGE_LEVELDB_INCLUDE_DUMPFILE_H_
|
||||
#define STORAGE_LEVELDB_INCLUDE_DUMPFILE_H_
|
||||
|
||||
#include <string>
|
||||
#include "leveldb/env.h"
|
||||
#include "leveldb/status.h"
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
// Dump the contents of the file named by fname in text format to
|
||||
// *dst. Makes a sequence of dst->Append() calls; each call is passed
|
||||
// the newline-terminated text corresponding to a single item found
|
||||
// in the file.
|
||||
//
|
||||
// Returns a non-OK result if fname does not name a leveldb storage
|
||||
// file, or if the file cannot be read.
|
||||
Status DumpFile(Env* env, const std::string& fname, WritableFile* dst);
|
||||
|
||||
} // namespace leveldb
|
||||
|
||||
#endif // STORAGE_LEVELDB_INCLUDE_DUMPFILE_H_
|
@ -142,7 +142,7 @@ class Env {
|
||||
// useful for computing deltas of time.
|
||||
virtual uint64_t NowMicros() = 0;
|
||||
|
||||
// Sleep/delay the thread for the perscribed number of micro-seconds.
|
||||
// Sleep/delay the thread for the prescribed number of micro-seconds.
|
||||
virtual void SleepForMicroseconds(int micros) = 0;
|
||||
|
||||
private:
|
||||
|
@ -61,7 +61,7 @@ class Iterator {
|
||||
// Return the value for the current entry. The underlying storage for
|
||||
// the returned slice is valid only until the next modification of
|
||||
// the iterator.
|
||||
// REQUIRES: !AtEnd() && !AtStart()
|
||||
// REQUIRES: Valid()
|
||||
virtual Slice value() const = 0;
|
||||
|
||||
// If an error has occurred, return it. Else return an ok status.
|
||||
|
@ -153,7 +153,7 @@ struct ReadOptions {
|
||||
|
||||
// If "snapshot" is non-NULL, read as of the supplied snapshot
|
||||
// (which must belong to the DB that is being read and which must
|
||||
// not have been released). If "snapshot" is NULL, use an impliicit
|
||||
// not have been released). If "snapshot" is NULL, use an implicit
|
||||
// snapshot of the state at the beginning of this read operation.
|
||||
// Default: NULL
|
||||
const Snapshot* snapshot;
|
||||
|
@ -5,14 +5,13 @@
|
||||
// AtomicPointer provides storage for a lock-free pointer.
|
||||
// Platform-dependent implementation of AtomicPointer:
|
||||
// - If the platform provides a cheap barrier, we use it with raw pointers
|
||||
// - If cstdatomic is present (on newer versions of gcc, it is), we use
|
||||
// a cstdatomic-based AtomicPointer. However we prefer the memory
|
||||
// - If <atomic> is present (on newer versions of gcc, it is), we use
|
||||
// a <atomic>-based AtomicPointer. However we prefer the memory
|
||||
// barrier based version, because at least on a gcc 4.4 32-bit build
|
||||
// on linux, we have encountered a buggy <cstdatomic>
|
||||
// implementation. Also, some <cstdatomic> implementations are much
|
||||
// slower than a memory-barrier based implementation (~16ns for
|
||||
// <cstdatomic> based acquire-load vs. ~1ns for a barrier based
|
||||
// acquire-load).
|
||||
// on linux, we have encountered a buggy <atomic> implementation.
|
||||
// Also, some <atomic> implementations are much slower than a memory-barrier
|
||||
// based implementation (~16ns for <atomic> based acquire-load vs. ~1ns for
|
||||
// a barrier based acquire-load).
|
||||
// This code is based on atomicops-internals-* in Google's perftools:
|
||||
// http://code.google.com/p/google-perftools/source/browse/#svn%2Ftrunk%2Fsrc%2Fbase
|
||||
|
||||
@ -20,8 +19,8 @@
|
||||
#define PORT_ATOMIC_POINTER_H_
|
||||
|
||||
#include <stdint.h>
|
||||
#ifdef LEVELDB_CSTDATOMIC_PRESENT
|
||||
#include <cstdatomic>
|
||||
#ifdef LEVELDB_ATOMIC_PRESENT
|
||||
#include <atomic>
|
||||
#endif
|
||||
#ifdef OS_WIN
|
||||
#include <windows.h>
|
||||
@ -126,7 +125,7 @@ class AtomicPointer {
|
||||
};
|
||||
|
||||
// AtomicPointer based on <cstdatomic>
|
||||
#elif defined(LEVELDB_CSTDATOMIC_PRESENT)
|
||||
#elif defined(LEVELDB_ATOMIC_PRESENT)
|
||||
class AtomicPointer {
|
||||
private:
|
||||
std::atomic<void*> rep_;
|
||||
@ -207,7 +206,7 @@ class AtomicPointer {
|
||||
inline void NoBarrier_Store(void* v) { rep_ = v; }
|
||||
};
|
||||
|
||||
// We have neither MemoryBarrier(), nor <cstdatomic>
|
||||
// We have neither MemoryBarrier(), nor <atomic>
|
||||
#else
|
||||
#error Please implement AtomicPointer for this platform.
|
||||
|
||||
|
@ -21,14 +21,11 @@
|
||||
#else
|
||||
#define PLATFORM_IS_LITTLE_ENDIAN false
|
||||
#endif
|
||||
#elif defined(OS_FREEBSD)
|
||||
#elif defined(OS_FREEBSD) || defined(OS_OPENBSD) ||\
|
||||
defined(OS_NETBSD) || defined(OS_DRAGONFLYBSD)
|
||||
#include <sys/types.h>
|
||||
#include <sys/endian.h>
|
||||
#define PLATFORM_IS_LITTLE_ENDIAN (_BYTE_ORDER == _LITTLE_ENDIAN)
|
||||
#elif defined(OS_OPENBSD) || defined(OS_NETBSD) ||\
|
||||
defined(OS_DRAGONFLYBSD)
|
||||
#include <sys/types.h>
|
||||
#include <sys/endian.h>
|
||||
#elif defined(OS_HPUX)
|
||||
#define PLATFORM_IS_LITTLE_ENDIAN false
|
||||
#elif defined(OS_ANDROID)
|
||||
@ -55,7 +52,7 @@
|
||||
|
||||
#if defined(OS_MACOSX) || defined(OS_SOLARIS) || defined(OS_FREEBSD) ||\
|
||||
defined(OS_NETBSD) || defined(OS_OPENBSD) || defined(OS_DRAGONFLYBSD) ||\
|
||||
defined(OS_ANDROID) || defined(OS_HPUX)
|
||||
defined(OS_ANDROID) || defined(OS_HPUX) || defined(CYGWIN)
|
||||
// Use fread/fwrite/fflush on platforms without _unlocked variants
|
||||
#define fread_unlocked fread
|
||||
#define fwrite_unlocked fwrite
|
||||
|
@ -2,7 +2,8 @@
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#ifndef STORAGE_LEVELDB_PORT_THREAD_ANNOTATIONS_H
|
||||
#ifndef STORAGE_LEVELDB_PORT_THREAD_ANNOTATIONS_H_
|
||||
#define STORAGE_LEVELDB_PORT_THREAD_ANNOTATIONS_H_
|
||||
|
||||
// Some environments provide custom macros to aid in static thread-safety
|
||||
// analysis. Provide empty definitions of such macros unless they are already
|
||||
@ -56,4 +57,4 @@
|
||||
#define NO_THREAD_SAFETY_ANALYSIS
|
||||
#endif
|
||||
|
||||
#endif // STORAGE_LEVELDB_PORT_THREAD_ANNOTATIONS_H
|
||||
#endif // STORAGE_LEVELDB_PORT_THREAD_ANNOTATIONS_H_
|
||||
|
@ -46,7 +46,7 @@ Block::~Block() {
|
||||
// Helper routine: decode the next block entry starting at "p",
|
||||
// storing the number of shared key bytes, non_shared key bytes,
|
||||
// and the length of the value in "*shared", "*non_shared", and
|
||||
// "*value_length", respectively. Will not derefence past "limit".
|
||||
// "*value_length", respectively. Will not dereference past "limit".
|
||||
//
|
||||
// If any errors are detected, returns NULL. Otherwise, returns a
|
||||
// pointer to the key delta (just past the three decoded values).
|
||||
|
@ -21,7 +21,7 @@ class BlockBuilder {
|
||||
// Reset the contents as if the BlockBuilder was just constructed.
|
||||
void Reset();
|
||||
|
||||
// REQUIRES: Finish() has not been callled since the last call to Reset().
|
||||
// REQUIRES: Finish() has not been called since the last call to Reset().
|
||||
// REQUIRES: key is larger than any previously added key
|
||||
void Add(const Slice& key, const Slice& value);
|
||||
|
||||
|
@ -48,7 +48,7 @@ Status Footer::DecodeFrom(Slice* input) {
|
||||
const uint64_t magic = ((static_cast<uint64_t>(magic_hi) << 32) |
|
||||
(static_cast<uint64_t>(magic_lo)));
|
||||
if (magic != kTableMagicNumber) {
|
||||
return Status::InvalidArgument("not an sstable (bad magic number)");
|
||||
return Status::Corruption("not an sstable (bad magic number)");
|
||||
}
|
||||
|
||||
Status result = metaindex_handle_.DecodeFrom(input);
|
||||
|
@ -41,7 +41,7 @@ Status Table::Open(const Options& options,
|
||||
Table** table) {
|
||||
*table = NULL;
|
||||
if (size < Footer::kEncodedLength) {
|
||||
return Status::InvalidArgument("file is too short to be an sstable");
|
||||
return Status::Corruption("file is too short to be an sstable");
|
||||
}
|
||||
|
||||
char footer_space[Footer::kEncodedLength];
|
||||
@ -58,7 +58,11 @@ Status Table::Open(const Options& options,
|
||||
BlockContents contents;
|
||||
Block* index_block = NULL;
|
||||
if (s.ok()) {
|
||||
s = ReadBlock(file, ReadOptions(), footer.index_handle(), &contents);
|
||||
ReadOptions opt;
|
||||
if (options.paranoid_checks) {
|
||||
opt.verify_checksums = true;
|
||||
}
|
||||
s = ReadBlock(file, opt, footer.index_handle(), &contents);
|
||||
if (s.ok()) {
|
||||
index_block = new Block(contents);
|
||||
}
|
||||
@ -92,6 +96,9 @@ void Table::ReadMeta(const Footer& footer) {
|
||||
// TODO(sanjay): Skip this if footer.metaindex_handle() size indicates
|
||||
// it is an empty block.
|
||||
ReadOptions opt;
|
||||
if (rep_->options.paranoid_checks) {
|
||||
opt.verify_checksums = true;
|
||||
}
|
||||
BlockContents contents;
|
||||
if (!ReadBlock(rep_->file, opt, footer.metaindex_handle(), &contents).ok()) {
|
||||
// Do not propagate errors since meta info is not needed for operation
|
||||
@ -120,6 +127,9 @@ void Table::ReadFilter(const Slice& filter_handle_value) {
|
||||
// We might want to unify with ReadBlock() if we start
|
||||
// requiring checksum verification in Table::Open.
|
||||
ReadOptions opt;
|
||||
if (rep_->options.paranoid_checks) {
|
||||
opt.verify_checksums = true;
|
||||
}
|
||||
BlockContents block;
|
||||
if (!ReadBlock(rep_->file, opt, filter_handle, &block).ok()) {
|
||||
return;
|
||||
|
@ -29,7 +29,7 @@ class BloomFilterPolicy : public FilterPolicy {
|
||||
}
|
||||
|
||||
virtual const char* Name() const {
|
||||
return "leveldb.BuiltinBloomFilter";
|
||||
return "leveldb.BuiltinBloomFilter2";
|
||||
}
|
||||
|
||||
virtual void CreateFilter(const Slice* keys, int n, std::string* dst) const {
|
||||
|
@ -3,8 +3,6 @@
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
#if !defined(LEVELDB_PLATFORM_WINDOWS)
|
||||
|
||||
#include <deque>
|
||||
#include <set>
|
||||
#include <dirent.h>
|
||||
#include <errno.h>
|
||||
#include <fcntl.h>
|
||||
@ -18,9 +16,8 @@
|
||||
#include <sys/types.h>
|
||||
#include <time.h>
|
||||
#include <unistd.h>
|
||||
#if defined(LEVELDB_PLATFORM_ANDROID)
|
||||
#include <sys/stat.h>
|
||||
#endif
|
||||
#include <deque>
|
||||
#include <set>
|
||||
#include "leveldb/env.h"
|
||||
#include "leveldb/slice.h"
|
||||
#include "port/port.h"
|
||||
@ -296,7 +293,8 @@ class PosixEnv : public Env {
|
||||
public:
|
||||
PosixEnv();
|
||||
virtual ~PosixEnv() {
|
||||
fprintf(stderr, "Destroying Env::Default()\n");
|
||||
char msg[] = "Destroying Env::Default()\n";
|
||||
fwrite(msg, 1, sizeof(msg), stderr);
|
||||
abort();
|
||||
}
|
||||
|
||||
|
@ -34,13 +34,13 @@ uint32_t Hash(const char* data, size_t n, uint32_t seed) {
|
||||
// Pick up remaining bytes
|
||||
switch (limit - data) {
|
||||
case 3:
|
||||
h += data[2] << 16;
|
||||
h += static_cast<unsigned char>(data[2]) << 16;
|
||||
FALLTHROUGH_INTENDED;
|
||||
case 2:
|
||||
h += data[1] << 8;
|
||||
h += static_cast<unsigned char>(data[1]) << 8;
|
||||
FALLTHROUGH_INTENDED;
|
||||
case 1:
|
||||
h += data[0];
|
||||
h += static_cast<unsigned char>(data[0]);
|
||||
h *= m;
|
||||
h ^= (h >> r);
|
||||
break;
|
||||
|
54
util/hash_test.cc
Normal file
54
util/hash_test.cc
Normal file
@ -0,0 +1,54 @@
|
||||
// Copyright (c) 2011 The LevelDB Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file. See the AUTHORS file for names of contributors.
|
||||
|
||||
#include "util/hash.h"
|
||||
#include "util/testharness.h"
|
||||
|
||||
namespace leveldb {
|
||||
|
||||
class HASH { };
|
||||
|
||||
TEST(HASH, SignedUnsignedIssue) {
|
||||
const unsigned char data1[1] = {0x62};
|
||||
const unsigned char data2[2] = {0xc3, 0x97};
|
||||
const unsigned char data3[3] = {0xe2, 0x99, 0xa5};
|
||||
const unsigned char data4[4] = {0xe1, 0x80, 0xb9, 0x32};
|
||||
const unsigned char data5[48] = {
|
||||
0x01, 0xc0, 0x00, 0x00,
|
||||
0x00, 0x00, 0x00, 0x00,
|
||||
0x00, 0x00, 0x00, 0x00,
|
||||
0x00, 0x00, 0x00, 0x00,
|
||||
0x14, 0x00, 0x00, 0x00,
|
||||
0x00, 0x00, 0x04, 0x00,
|
||||
0x00, 0x00, 0x00, 0x14,
|
||||
0x00, 0x00, 0x00, 0x18,
|
||||
0x28, 0x00, 0x00, 0x00,
|
||||
0x00, 0x00, 0x00, 0x00,
|
||||
0x02, 0x00, 0x00, 0x00,
|
||||
0x00, 0x00, 0x00, 0x00,
|
||||
};
|
||||
|
||||
ASSERT_EQ(Hash(0, 0, 0xbc9f1d34), 0xbc9f1d34);
|
||||
ASSERT_EQ(
|
||||
Hash(reinterpret_cast<const char*>(data1), sizeof(data1), 0xbc9f1d34),
|
||||
0xef1345c4);
|
||||
ASSERT_EQ(
|
||||
Hash(reinterpret_cast<const char*>(data2), sizeof(data2), 0xbc9f1d34),
|
||||
0x5b663814);
|
||||
ASSERT_EQ(
|
||||
Hash(reinterpret_cast<const char*>(data3), sizeof(data3), 0xbc9f1d34),
|
||||
0x323c078f);
|
||||
ASSERT_EQ(
|
||||
Hash(reinterpret_cast<const char*>(data4), sizeof(data4), 0xbc9f1d34),
|
||||
0xed21633a);
|
||||
ASSERT_EQ(
|
||||
Hash(reinterpret_cast<const char*>(data5), sizeof(data5), 0x12345678),
|
||||
0xf333dabb);
|
||||
}
|
||||
|
||||
} // namespace leveldb
|
||||
|
||||
int main(int argc, char** argv) {
|
||||
return leveldb::test::RunAllTests();
|
||||
}
|
@ -45,15 +45,6 @@ std::string EscapeString(const Slice& value) {
|
||||
return r;
|
||||
}
|
||||
|
||||
bool ConsumeChar(Slice* in, char c) {
|
||||
if (!in->empty() && (*in)[0] == c) {
|
||||
in->remove_prefix(1);
|
||||
return true;
|
||||
} else {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
bool ConsumeDecimalNumber(Slice* in, uint64_t* val) {
|
||||
uint64_t v = 0;
|
||||
int digits = 0;
|
||||
|
@ -32,10 +32,6 @@ extern std::string NumberToString(uint64_t num);
|
||||
// Escapes any non-printable characters found in "value".
|
||||
extern std::string EscapeString(const Slice& value);
|
||||
|
||||
// If *in starts with "c", advances *in past the first character and
|
||||
// returns true. Otherwise, returns false.
|
||||
extern bool ConsumeChar(Slice* in, char c);
|
||||
|
||||
// Parse a human-readable number from "*in" into *value. On success,
|
||||
// advances "*in" past the consumed number and sets "*val" to the
|
||||
// numeric value. Otherwise, returns false and leaves *in in an
|
||||
|
Loading…
Reference in New Issue
Block a user