Skip to content

Commit

Permalink
sync with upstream @ 21409451
Browse files Browse the repository at this point in the history 
Check the NEWS file for details of what changed.

git-svn-id: https://leveldb.googlecode.com/svn/trunk@28 62dab493-f737-651d-591e-8d6aee1b9529
  • Loading branch information
dgrogan@chromium.org committed May 21, 2011
1 parent 3c11133 commit da79909
Show file tree
Hide file tree
Showing 34 changed files with 953 additions and 406 deletions.
17 changes: 17 additions & 0 deletions NEWS
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
Release 1.2 2011-05-16
----------------------

Fixes for larger databases (tested up to one billion 100-byte entries,
i.e., ~100GB).

(1) Place hard limit on number of level-0 files. This fixes errors
of the form "too many open files".

(2) Fixed memtable management. Before the fix, a heavy write burst
could cause unbounded memory usage.

A fix for a logging bug where the reader would incorrectly complain
about corruption.

Allow public access to WriteBatch contents so that users can easily
wrap a DB.
66 changes: 55 additions & 11 deletions db/db_bench.cc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,10 @@
// overwrite -- overwrite N values in random key order in async mode
// fillsync -- write N/100 values in random key order in sync mode
// fill100K -- write N/1000 100K values in random order in async mode
// readseq -- read N values sequentially
// readreverse -- read N values in reverse order
// readrandom -- read N values in random order
// readseq -- read N times sequentially
// readreverse -- read N times in reverse order
// readrandom -- read N times in random order
// readhot -- read N times in random order from 1% section of DB
// crc32c -- repeated crc32c of 4K of data
// Meta operations:
// compact -- Compact the entire DB
Expand Down Expand Up @@ -54,6 +55,9 @@ static const char* FLAGS_benchmarks =
// Number of key/values to place in database
static int FLAGS_num = 1000000;

// Number of read operations to do. If negative, do FLAGS_num reads.
static int FLAGS_reads = -1;

// Size of each value
static int FLAGS_value_size = 100;

Expand All @@ -72,6 +76,14 @@ static int FLAGS_write_buffer_size = 0;
// Negative means use default settings.
static int FLAGS_cache_size = -1;

// Maximum number of files to keep open at the same time (use default if == 0)
static int FLAGS_open_files = 0;

// If true, do not destroy the existing database. If you set this
// flag and also specify a benchmark that wants a fresh database, that
// benchmark will fail.
static bool FLAGS_use_existing_db = false;

namespace leveldb {

// Helper for quickly generating random data.
Expand Down Expand Up @@ -126,6 +138,7 @@ class Benchmark {
Cache* cache_;
DB* db_;
int num_;
int reads_;
int heap_counter_;
double start_;
double last_op_finish_;
Expand Down Expand Up @@ -298,6 +311,7 @@ class Benchmark {
: cache_(FLAGS_cache_size >= 0 ? NewLRUCache(FLAGS_cache_size) : NULL),
db_(NULL),
num_(FLAGS_num),
reads_(FLAGS_reads < 0 ? FLAGS_num : FLAGS_reads),
heap_counter_(0),
bytes_(0),
rand_(301) {
Expand All @@ -308,7 +322,9 @@ class Benchmark {
Env::Default()->DeleteFile("/tmp/dbbench/" + files[i]);
}
}
DestroyDB("/tmp/dbbench", Options());
if (!FLAGS_use_existing_db) {
DestroyDB("/tmp/dbbench", Options());
}
}

~Benchmark() {
Expand Down Expand Up @@ -355,11 +371,13 @@ class Benchmark {
ReadReverse();
} else if (name == Slice("readrandom")) {
ReadRandom();
} else if (name == Slice("readhot")) {
ReadHot();
} else if (name == Slice("readrandomsmall")) {
int n = num_;
num_ /= 1000;
int n = reads_;
reads_ /= 1000;
ReadRandom();
num_ = n;
reads_ = n;
} else if (name == Slice("compact")) {
Compact();
} else if (name == Slice("crc32c")) {
Expand Down Expand Up @@ -449,7 +467,7 @@ class Benchmark {
void Open() {
assert(db_ == NULL);
Options options;
options.create_if_missing = true;
options.create_if_missing = !FLAGS_use_existing_db;
options.block_cache = cache_;
options.write_buffer_size = FLAGS_write_buffer_size;
Status s = DB::Open(options, "/tmp/dbbench", &db_);
Expand All @@ -462,6 +480,10 @@ class Benchmark {
void Write(const WriteOptions& options, Order order, DBState state,
int num_entries, int value_size, int entries_per_batch) {
if (state == FRESH) {
if (FLAGS_use_existing_db) {
message_ = "skipping (--use_existing_db is true)";
return;
}
delete db_;
db_ = NULL;
DestroyDB("/tmp/dbbench", Options());
Expand Down Expand Up @@ -499,7 +521,7 @@ class Benchmark {
void ReadSequential() {
Iterator* iter = db_->NewIterator(ReadOptions());
int i = 0;
for (iter->SeekToFirst(); i < num_ && iter->Valid(); iter->Next()) {
for (iter->SeekToFirst(); i < reads_ && iter->Valid(); iter->Next()) {
bytes_ += iter->key().size() + iter->value().size();
FinishedSingleOp();
++i;
Expand All @@ -510,7 +532,7 @@ class Benchmark {
void ReadReverse() {
Iterator* iter = db_->NewIterator(ReadOptions());
int i = 0;
for (iter->SeekToLast(); i < num_ && iter->Valid(); iter->Prev()) {
for (iter->SeekToLast(); i < reads_ && iter->Valid(); iter->Prev()) {
bytes_ += iter->key().size() + iter->value().size();
FinishedSingleOp();
++i;
Expand All @@ -521,7 +543,7 @@ class Benchmark {
void ReadRandom() {
ReadOptions options;
std::string value;
for (int i = 0; i < num_; i++) {
for (int i = 0; i < reads_; i++) {
char key[100];
const int k = rand_.Next() % FLAGS_num;
snprintf(key, sizeof(key), "%016d", k);
Expand All @@ -530,6 +552,19 @@ class Benchmark {
}
}

void ReadHot() {
ReadOptions options;
std::string value;
const int range = (FLAGS_num + 99) / 100;
for (int i = 0; i < reads_; i++) {
char key[100];
const int k = rand_.Next() % range;
snprintf(key, sizeof(key), "%016d", k);
db_->Get(options, key, &value);
FinishedSingleOp();
}
}

void Compact() {
DBImpl* dbi = reinterpret_cast<DBImpl*>(db_);
dbi->TEST_CompactMemTable();
Expand Down Expand Up @@ -582,6 +617,8 @@ class Benchmark {

int main(int argc, char** argv) {
FLAGS_write_buffer_size = leveldb::Options().write_buffer_size;
FLAGS_open_files = leveldb::Options().max_open_files;

for (int i = 1; i < argc; i++) {
double d;
int n;
Expand All @@ -593,14 +630,21 @@ int main(int argc, char** argv) {
} else if (sscanf(argv[i], "--histogram=%d%c", &n, &junk) == 1 &&
(n == 0 || n == 1)) {
FLAGS_histogram = n;
} else if (sscanf(argv[i], "--use_existing_db=%d%c", &n, &junk) == 1 &&
(n == 0 || n == 1)) {
FLAGS_use_existing_db = n;
} else if (sscanf(argv[i], "--num=%d%c", &n, &junk) == 1) {
FLAGS_num = n;
} else if (sscanf(argv[i], "--reads=%d%c", &n, &junk) == 1) {
FLAGS_reads = n;
} else if (sscanf(argv[i], "--value_size=%d%c", &n, &junk) == 1) {
FLAGS_value_size = n;
} else if (sscanf(argv[i], "--write_buffer_size=%d%c", &n, &junk) == 1) {
FLAGS_write_buffer_size = n;
} else if (sscanf(argv[i], "--cache_size=%d%c", &n, &junk) == 1) {
FLAGS_cache_size = n;
} else if (sscanf(argv[i], "--open_files=%d%c", &n, &junk) == 1) {
FLAGS_open_files = n;
} else {
fprintf(stderr, "Invalid flag '%s'\n", argv[i]);
exit(1);
Expand Down
Loading

0 comments on commit da79909

Please sign in to comment.