From d618510128aff49b39efd28bd1c72a18ad563e94 Mon Sep 17 00:00:00 2001 From: Conner Hewitt Date: Sun, 3 Nov 2019 14:55:51 -0600 Subject: [PATCH] Implemented Go modules --- .travis.yml | 6 +- cmd/main.go | 54 - go.mod | 5 + go.sum | 18 + vendor/github.com/golang/snappy/.gitignore | 16 - vendor/github.com/golang/snappy/AUTHORS | 15 - vendor/github.com/golang/snappy/CONTRIBUTORS | 37 - vendor/github.com/golang/snappy/LICENSE | 27 - vendor/github.com/golang/snappy/README | 107 - .../golang/snappy/cmd/snappytool/main.cpp | 77 - vendor/github.com/golang/snappy/decode.go | 237 -- .../github.com/golang/snappy/decode_amd64.go | 14 - .../github.com/golang/snappy/decode_amd64.s | 490 --- .../github.com/golang/snappy/decode_other.go | 101 - vendor/github.com/golang/snappy/encode.go | 285 -- .../github.com/golang/snappy/encode_amd64.go | 29 - .../github.com/golang/snappy/encode_amd64.s | 730 ---- .../github.com/golang/snappy/encode_other.go | 238 -- .../github.com/golang/snappy/golden_test.go | 1965 ----------- vendor/github.com/golang/snappy/snappy.go | 87 - .../github.com/golang/snappy/snappy_test.go | 1353 -------- .../snappy/testdata/Mark.Twain-Tom.Sawyer.txt | 396 --- .../Mark.Twain-Tom.Sawyer.txt.rawsnappy | Bin 9871 -> 0 bytes .../github.com/syndtr/goleveldb/.travis.yml | 12 - vendor/github.com/syndtr/goleveldb/LICENSE | 24 - vendor/github.com/syndtr/goleveldb/README.md | 105 - .../syndtr/goleveldb/leveldb/batch.go | 349 -- .../syndtr/goleveldb/leveldb/batch_test.go | 147 - .../syndtr/goleveldb/leveldb/bench_test.go | 507 --- .../goleveldb/leveldb/cache/bench_test.go | 29 - .../syndtr/goleveldb/leveldb/cache/cache.go | 705 ---- .../goleveldb/leveldb/cache/cache_test.go | 563 ---- .../syndtr/goleveldb/leveldb/cache/lru.go | 195 -- .../syndtr/goleveldb/leveldb/comparer.go | 67 - .../leveldb/comparer/bytes_comparer.go | 51 - .../goleveldb/leveldb/comparer/comparer.go | 57 - .../syndtr/goleveldb/leveldb/corrupt_test.go | 496 --- .../github.com/syndtr/goleveldb/leveldb/db.go | 1091 ------ .../syndtr/goleveldb/leveldb/db_compaction.go | 826 ----- .../syndtr/goleveldb/leveldb/db_iter.go | 360 -- .../syndtr/goleveldb/leveldb/db_snapshot.go | 183 -- .../syndtr/goleveldb/leveldb/db_state.go | 239 -- .../syndtr/goleveldb/leveldb/db_test.go | 2925 ----------------- .../goleveldb/leveldb/db_transaction.go | 325 -- .../syndtr/goleveldb/leveldb/db_util.go | 102 - .../syndtr/goleveldb/leveldb/db_write.go | 452 --- .../syndtr/goleveldb/leveldb/doc.go | 90 - .../syndtr/goleveldb/leveldb/errors.go | 20 - .../syndtr/goleveldb/leveldb/errors/errors.go | 78 - .../syndtr/goleveldb/leveldb/external_test.go | 117 - .../syndtr/goleveldb/leveldb/filter.go | 31 - .../syndtr/goleveldb/leveldb/filter/bloom.go | 116 - .../goleveldb/leveldb/filter/bloom_test.go | 142 - .../syndtr/goleveldb/leveldb/filter/filter.go | 60 - .../goleveldb/leveldb/iterator/array_iter.go | 184 -- .../leveldb/iterator/array_iter_test.go | 30 - .../leveldb/iterator/indexed_iter.go | 242 -- .../leveldb/iterator/indexed_iter_test.go | 83 - .../syndtr/goleveldb/leveldb/iterator/iter.go | 132 - .../leveldb/iterator/iter_suite_test.go | 11 - .../goleveldb/leveldb/iterator/merged_iter.go | 304 -- .../leveldb/iterator/merged_iter_test.go | 60 - .../goleveldb/leveldb/journal/journal.go | 524 --- .../goleveldb/leveldb/journal/journal_test.go | 818 ----- .../syndtr/goleveldb/leveldb/key.go | 143 - .../syndtr/goleveldb/leveldb/key_test.go | 133 - .../goleveldb/leveldb/leveldb_suite_test.go | 11 - .../goleveldb/leveldb/memdb/bench_test.go | 75 - .../syndtr/goleveldb/leveldb/memdb/memdb.go | 475 --- .../leveldb/memdb/memdb_suite_test.go | 11 - .../goleveldb/leveldb/memdb/memdb_test.go | 135 - .../syndtr/goleveldb/leveldb/opt/options.go | 684 ---- .../syndtr/goleveldb/leveldb/options.go | 107 - .../syndtr/goleveldb/leveldb/session.go | 210 -- .../goleveldb/leveldb/session_compaction.go | 302 -- .../goleveldb/leveldb/session_record.go | 323 -- .../goleveldb/leveldb/session_record_test.go | 62 - .../syndtr/goleveldb/leveldb/session_util.go | 271 -- .../goleveldb/leveldb/storage/file_storage.go | 583 ---- .../leveldb/storage/file_storage_nacl.go | 34 - .../leveldb/storage/file_storage_plan9.go | 65 - .../leveldb/storage/file_storage_solaris.go | 81 - .../leveldb/storage/file_storage_test.go | 176 - .../leveldb/storage/file_storage_unix.go | 86 - .../leveldb/storage/file_storage_windows.go | 78 - .../goleveldb/leveldb/storage/mem_storage.go | 218 -- .../leveldb/storage/mem_storage_test.go | 65 - .../goleveldb/leveldb/storage/storage.go | 179 - .../syndtr/goleveldb/leveldb/table.go | 529 --- .../goleveldb/leveldb/table/block_test.go | 139 - .../syndtr/goleveldb/leveldb/table/reader.go | 1135 ------- .../syndtr/goleveldb/leveldb/table/table.go | 177 - .../leveldb/table/table_suite_test.go | 11 - .../goleveldb/leveldb/table/table_test.go | 123 - .../syndtr/goleveldb/leveldb/table/writer.go | 375 --- .../syndtr/goleveldb/leveldb/testutil/db.go | 222 -- .../goleveldb/leveldb/testutil/ginkgo.go | 21 - .../syndtr/goleveldb/leveldb/testutil/iter.go | 327 -- .../syndtr/goleveldb/leveldb/testutil/kv.go | 352 -- .../goleveldb/leveldb/testutil/kvtest.go | 212 -- .../goleveldb/leveldb/testutil/storage.go | 693 ---- .../syndtr/goleveldb/leveldb/testutil/util.go | 171 - .../syndtr/goleveldb/leveldb/testutil_test.go | 91 - .../syndtr/goleveldb/leveldb/util.go | 98 - .../syndtr/goleveldb/leveldb/util/buffer.go | 293 -- .../goleveldb/leveldb/util/buffer_pool.go | 239 -- .../goleveldb/leveldb/util/buffer_test.go | 369 --- .../syndtr/goleveldb/leveldb/util/crc32.go | 30 - .../syndtr/goleveldb/leveldb/util/hash.go | 48 - .../goleveldb/leveldb/util/hash_test.go | 46 - .../syndtr/goleveldb/leveldb/util/range.go | 32 - .../syndtr/goleveldb/leveldb/util/util.go | 73 - .../syndtr/goleveldb/leveldb/version.go | 528 --- .../syndtr/goleveldb/leveldb/version_test.go | 181 - .../goleveldb/manualtest/dbstress/key.go | 137 - .../goleveldb/manualtest/dbstress/main.go | 628 ---- .../goleveldb/manualtest/filelock/main.go | 85 - 117 files changed, 25 insertions(+), 30986 deletions(-) delete mode 100644 cmd/main.go create mode 100644 go.mod create mode 100644 go.sum delete mode 100644 vendor/github.com/golang/snappy/.gitignore delete mode 100644 vendor/github.com/golang/snappy/AUTHORS delete mode 100644 vendor/github.com/golang/snappy/CONTRIBUTORS delete mode 100644 vendor/github.com/golang/snappy/LICENSE delete mode 100644 vendor/github.com/golang/snappy/README delete mode 100644 vendor/github.com/golang/snappy/cmd/snappytool/main.cpp delete mode 100644 vendor/github.com/golang/snappy/decode.go delete mode 100644 vendor/github.com/golang/snappy/decode_amd64.go delete mode 100644 vendor/github.com/golang/snappy/decode_amd64.s delete mode 100644 vendor/github.com/golang/snappy/decode_other.go delete mode 100644 vendor/github.com/golang/snappy/encode.go delete mode 100644 vendor/github.com/golang/snappy/encode_amd64.go delete mode 100644 vendor/github.com/golang/snappy/encode_amd64.s delete mode 100644 vendor/github.com/golang/snappy/encode_other.go delete mode 100644 vendor/github.com/golang/snappy/golden_test.go delete mode 100644 vendor/github.com/golang/snappy/snappy.go delete mode 100644 vendor/github.com/golang/snappy/snappy_test.go delete mode 100644 vendor/github.com/golang/snappy/testdata/Mark.Twain-Tom.Sawyer.txt delete mode 100644 vendor/github.com/golang/snappy/testdata/Mark.Twain-Tom.Sawyer.txt.rawsnappy delete mode 100644 vendor/github.com/syndtr/goleveldb/.travis.yml delete mode 100644 vendor/github.com/syndtr/goleveldb/LICENSE delete mode 100644 vendor/github.com/syndtr/goleveldb/README.md delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/batch.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/batch_test.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/bench_test.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/cache/bench_test.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/cache/cache.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/cache/cache_test.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/cache/lru.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/comparer.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/comparer/bytes_comparer.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/comparer/comparer.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/corrupt_test.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/db.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/db_iter.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/db_snapshot.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/db_state.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/db_test.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/db_transaction.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/db_util.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/db_write.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/doc.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/errors.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/errors/errors.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/external_test.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/filter.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/filter/bloom.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/filter/bloom_test.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/filter/filter.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/iterator/array_iter.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/iterator/array_iter_test.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/iterator/indexed_iter.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/iterator/indexed_iter_test.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/iterator/iter.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/iterator/iter_suite_test.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/iterator/merged_iter.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/iterator/merged_iter_test.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/journal/journal.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/journal/journal_test.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/key.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/key_test.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/leveldb_suite_test.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/memdb/bench_test.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/memdb/memdb.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/memdb/memdb_suite_test.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/memdb/memdb_test.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/opt/options.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/options.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/session.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/session_compaction.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/session_record.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/session_record_test.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/session_util.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/storage/file_storage.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/storage/file_storage_nacl.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/storage/file_storage_plan9.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/storage/file_storage_solaris.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/storage/file_storage_test.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/storage/file_storage_unix.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/storage/file_storage_windows.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/storage/mem_storage.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/storage/mem_storage_test.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/storage/storage.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/table.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/table/block_test.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/table/reader.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/table/table.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/table/table_suite_test.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/table/table_test.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/table/writer.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/testutil/db.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/testutil/ginkgo.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/testutil/iter.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/testutil/kv.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/testutil/kvtest.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/testutil/storage.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/testutil/util.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/testutil_test.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/util.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/util/buffer.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/util/buffer_pool.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/util/buffer_test.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/util/crc32.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/util/hash.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/util/hash_test.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/util/range.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/util/util.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/version.go delete mode 100644 vendor/github.com/syndtr/goleveldb/leveldb/version_test.go delete mode 100644 vendor/github.com/syndtr/goleveldb/manualtest/dbstress/key.go delete mode 100644 vendor/github.com/syndtr/goleveldb/manualtest/dbstress/main.go delete mode 100644 vendor/github.com/syndtr/goleveldb/manualtest/filelock/main.go diff --git a/.travis.yml b/.travis.yml index 30f4180..d5eb553 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,10 +1,8 @@ language: go + go: - 1.13 - tip -env: - - GO15VENDOREXPERIMENT=1 - script: - - go test -v $(go list ./... | grep -v /vendor/) \ No newline at end of file + - go test -v ./... \ No newline at end of file diff --git a/cmd/main.go b/cmd/main.go deleted file mode 100644 index 2b8a439..0000000 --- a/cmd/main.go +++ /dev/null @@ -1,54 +0,0 @@ -package main - -import ( - "fmt" - "log" - - "github.com/beeker1121/goque" -) - -type Obj struct { - A *int - B int -} - -func (o Obj) String() string { - if o.A != nil { - return fmt.Sprintf("%v %v %v", o.A, *o.A, o.B) - } else { - return fmt.Sprintf("%v %v", o.A, o.B) - } -} - -func main() { - queue, err := goque.OpenPriorityQueue("queuetest", goque.ASC) - if err != nil { - log.Fatal("OpenPriorityQueue()", err) - } - - o1 := Obj{new(int), 1} - *o1.A = 2 // not necessary - // *o1.A = 10 // this will work - // o1.A = nil // will work, too - - log.Println("Writing to queue:", o1) - - _, err = queue.EnqueueObject(0, o1) - if err != nil { - log.Println("EnqueueObject()", o1, err) - } - - item, err := queue.Dequeue() - if err != nil { - log.Fatal("Dequeue()", err) - } - o2 := Obj{} - err = item.ToObject(&o2) - if err != nil { - log.Fatal("ToObject()", err) - } - - log.Println("Read from queue:", o2) - - queue.Drop() -} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..710f72e --- /dev/null +++ b/go.mod @@ -0,0 +1,5 @@ +module github.com/beeker1121/goque + +go 1.13 + +require github.com/syndtr/goleveldb v1.0.0 diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..e188042 --- /dev/null +++ b/go.sum @@ -0,0 +1,18 @@ +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db h1:woRePGFeVFfLKN/pOkfl+p/TAqKOfFu+7KPlMVpok/w= +github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/syndtr/goleveldb v1.0.0 h1:fBdIW9lB4Iz0n9khmH8w27SJ3QEJ7+IgjPEwGSZiFdE= +github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/vendor/github.com/golang/snappy/.gitignore b/vendor/github.com/golang/snappy/.gitignore deleted file mode 100644 index 042091d..0000000 --- a/vendor/github.com/golang/snappy/.gitignore +++ /dev/null @@ -1,16 +0,0 @@ -cmd/snappytool/snappytool -testdata/bench - -# These explicitly listed benchmark data files are for an obsolete version of -# snappy_test.go. -testdata/alice29.txt -testdata/asyoulik.txt -testdata/fireworks.jpeg -testdata/geo.protodata -testdata/html -testdata/html_x_4 -testdata/kppkn.gtb -testdata/lcet10.txt -testdata/paper-100k.pdf -testdata/plrabn12.txt -testdata/urls.10K diff --git a/vendor/github.com/golang/snappy/AUTHORS b/vendor/github.com/golang/snappy/AUTHORS deleted file mode 100644 index bcfa195..0000000 --- a/vendor/github.com/golang/snappy/AUTHORS +++ /dev/null @@ -1,15 +0,0 @@ -# This is the official list of Snappy-Go authors for copyright purposes. -# This file is distinct from the CONTRIBUTORS files. -# See the latter for an explanation. - -# Names should be added to this file as -# Name or Organization -# The email address is not required for organizations. - -# Please keep the list sorted. - -Damian Gryski -Google Inc. -Jan Mercl <0xjnml@gmail.com> -Rodolfo Carvalho -Sebastien Binet diff --git a/vendor/github.com/golang/snappy/CONTRIBUTORS b/vendor/github.com/golang/snappy/CONTRIBUTORS deleted file mode 100644 index 931ae31..0000000 --- a/vendor/github.com/golang/snappy/CONTRIBUTORS +++ /dev/null @@ -1,37 +0,0 @@ -# This is the official list of people who can contribute -# (and typically have contributed) code to the Snappy-Go repository. -# The AUTHORS file lists the copyright holders; this file -# lists people. For example, Google employees are listed here -# but not in AUTHORS, because Google holds the copyright. -# -# The submission process automatically checks to make sure -# that people submitting code are listed in this file (by email address). -# -# Names should be added to this file only after verifying that -# the individual or the individual's organization has agreed to -# the appropriate Contributor License Agreement, found here: -# -# http://code.google.com/legal/individual-cla-v1.0.html -# http://code.google.com/legal/corporate-cla-v1.0.html -# -# The agreement for individuals can be filled out on the web. -# -# When adding J Random Contributor's name to this file, -# either J's name or J's organization's name should be -# added to the AUTHORS file, depending on whether the -# individual or corporate CLA was used. - -# Names should be added to this file like so: -# Name - -# Please keep the list sorted. - -Damian Gryski -Jan Mercl <0xjnml@gmail.com> -Kai Backman -Marc-Antoine Ruel -Nigel Tao -Rob Pike -Rodolfo Carvalho -Russ Cox -Sebastien Binet diff --git a/vendor/github.com/golang/snappy/LICENSE b/vendor/github.com/golang/snappy/LICENSE deleted file mode 100644 index 6050c10..0000000 --- a/vendor/github.com/golang/snappy/LICENSE +++ /dev/null @@ -1,27 +0,0 @@ -Copyright (c) 2011 The Snappy-Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/golang/snappy/README b/vendor/github.com/golang/snappy/README deleted file mode 100644 index cea1287..0000000 --- a/vendor/github.com/golang/snappy/README +++ /dev/null @@ -1,107 +0,0 @@ -The Snappy compression format in the Go programming language. - -To download and install from source: -$ go get github.com/golang/snappy - -Unless otherwise noted, the Snappy-Go source files are distributed -under the BSD-style license found in the LICENSE file. - - - -Benchmarks. - -The golang/snappy benchmarks include compressing (Z) and decompressing (U) ten -or so files, the same set used by the C++ Snappy code (github.com/google/snappy -and note the "google", not "golang"). On an "Intel(R) Core(TM) i7-3770 CPU @ -3.40GHz", Go's GOARCH=amd64 numbers as of 2016-05-29: - -"go test -test.bench=." - -_UFlat0-8 2.19GB/s ± 0% html -_UFlat1-8 1.41GB/s ± 0% urls -_UFlat2-8 23.5GB/s ± 2% jpg -_UFlat3-8 1.91GB/s ± 0% jpg_200 -_UFlat4-8 14.0GB/s ± 1% pdf -_UFlat5-8 1.97GB/s ± 0% html4 -_UFlat6-8 814MB/s ± 0% txt1 -_UFlat7-8 785MB/s ± 0% txt2 -_UFlat8-8 857MB/s ± 0% txt3 -_UFlat9-8 719MB/s ± 1% txt4 -_UFlat10-8 2.84GB/s ± 0% pb -_UFlat11-8 1.05GB/s ± 0% gaviota - -_ZFlat0-8 1.04GB/s ± 0% html -_ZFlat1-8 534MB/s ± 0% urls -_ZFlat2-8 15.7GB/s ± 1% jpg -_ZFlat3-8 740MB/s ± 3% jpg_200 -_ZFlat4-8 9.20GB/s ± 1% pdf -_ZFlat5-8 991MB/s ± 0% html4 -_ZFlat6-8 379MB/s ± 0% txt1 -_ZFlat7-8 352MB/s ± 0% txt2 -_ZFlat8-8 396MB/s ± 1% txt3 -_ZFlat9-8 327MB/s ± 1% txt4 -_ZFlat10-8 1.33GB/s ± 1% pb -_ZFlat11-8 605MB/s ± 1% gaviota - - - -"go test -test.bench=. -tags=noasm" - -_UFlat0-8 621MB/s ± 2% html -_UFlat1-8 494MB/s ± 1% urls -_UFlat2-8 23.2GB/s ± 1% jpg -_UFlat3-8 1.12GB/s ± 1% jpg_200 -_UFlat4-8 4.35GB/s ± 1% pdf -_UFlat5-8 609MB/s ± 0% html4 -_UFlat6-8 296MB/s ± 0% txt1 -_UFlat7-8 288MB/s ± 0% txt2 -_UFlat8-8 309MB/s ± 1% txt3 -_UFlat9-8 280MB/s ± 1% txt4 -_UFlat10-8 753MB/s ± 0% pb -_UFlat11-8 400MB/s ± 0% gaviota - -_ZFlat0-8 409MB/s ± 1% html -_ZFlat1-8 250MB/s ± 1% urls -_ZFlat2-8 12.3GB/s ± 1% jpg -_ZFlat3-8 132MB/s ± 0% jpg_200 -_ZFlat4-8 2.92GB/s ± 0% pdf -_ZFlat5-8 405MB/s ± 1% html4 -_ZFlat6-8 179MB/s ± 1% txt1 -_ZFlat7-8 170MB/s ± 1% txt2 -_ZFlat8-8 189MB/s ± 1% txt3 -_ZFlat9-8 164MB/s ± 1% txt4 -_ZFlat10-8 479MB/s ± 1% pb -_ZFlat11-8 270MB/s ± 1% gaviota - - - -For comparison (Go's encoded output is byte-for-byte identical to C++'s), here -are the numbers from C++ Snappy's - -make CXXFLAGS="-O2 -DNDEBUG -g" clean snappy_unittest.log && cat snappy_unittest.log - -BM_UFlat/0 2.4GB/s html -BM_UFlat/1 1.4GB/s urls -BM_UFlat/2 21.8GB/s jpg -BM_UFlat/3 1.5GB/s jpg_200 -BM_UFlat/4 13.3GB/s pdf -BM_UFlat/5 2.1GB/s html4 -BM_UFlat/6 1.0GB/s txt1 -BM_UFlat/7 959.4MB/s txt2 -BM_UFlat/8 1.0GB/s txt3 -BM_UFlat/9 864.5MB/s txt4 -BM_UFlat/10 2.9GB/s pb -BM_UFlat/11 1.2GB/s gaviota - -BM_ZFlat/0 944.3MB/s html (22.31 %) -BM_ZFlat/1 501.6MB/s urls (47.78 %) -BM_ZFlat/2 14.3GB/s jpg (99.95 %) -BM_ZFlat/3 538.3MB/s jpg_200 (73.00 %) -BM_ZFlat/4 8.3GB/s pdf (83.30 %) -BM_ZFlat/5 903.5MB/s html4 (22.52 %) -BM_ZFlat/6 336.0MB/s txt1 (57.88 %) -BM_ZFlat/7 312.3MB/s txt2 (61.91 %) -BM_ZFlat/8 353.1MB/s txt3 (54.99 %) -BM_ZFlat/9 289.9MB/s txt4 (66.26 %) -BM_ZFlat/10 1.2GB/s pb (19.68 %) -BM_ZFlat/11 527.4MB/s gaviota (37.72 %) diff --git a/vendor/github.com/golang/snappy/cmd/snappytool/main.cpp b/vendor/github.com/golang/snappy/cmd/snappytool/main.cpp deleted file mode 100644 index fc31f51..0000000 --- a/vendor/github.com/golang/snappy/cmd/snappytool/main.cpp +++ /dev/null @@ -1,77 +0,0 @@ -/* -To build the snappytool binary: -g++ main.cpp /usr/lib/libsnappy.a -o snappytool -or, if you have built the C++ snappy library from source: -g++ main.cpp /path/to/your/snappy/.libs/libsnappy.a -o snappytool -after running "make" from your snappy checkout directory. -*/ - -#include -#include -#include -#include - -#include "snappy.h" - -#define N 1000000 - -char dst[N]; -char src[N]; - -int main(int argc, char** argv) { - // Parse args. - if (argc != 2) { - fprintf(stderr, "exactly one of -d or -e must be given\n"); - return 1; - } - bool decode = strcmp(argv[1], "-d") == 0; - bool encode = strcmp(argv[1], "-e") == 0; - if (decode == encode) { - fprintf(stderr, "exactly one of -d or -e must be given\n"); - return 1; - } - - // Read all of stdin into src[:s]. - size_t s = 0; - while (1) { - if (s == N) { - fprintf(stderr, "input too large\n"); - return 1; - } - ssize_t n = read(0, src+s, N-s); - if (n == 0) { - break; - } - if (n < 0) { - fprintf(stderr, "read error: %s\n", strerror(errno)); - // TODO: handle EAGAIN, EINTR? - return 1; - } - s += n; - } - - // Encode or decode src[:s] to dst[:d], and write to stdout. - size_t d = 0; - if (encode) { - if (N < snappy::MaxCompressedLength(s)) { - fprintf(stderr, "input too large after encoding\n"); - return 1; - } - snappy::RawCompress(src, s, dst, &d); - } else { - if (!snappy::GetUncompressedLength(src, s, &d)) { - fprintf(stderr, "could not get uncompressed length\n"); - return 1; - } - if (N < d) { - fprintf(stderr, "input too large after decoding\n"); - return 1; - } - if (!snappy::RawUncompress(src, s, dst)) { - fprintf(stderr, "input was not valid Snappy-compressed data\n"); - return 1; - } - } - write(1, dst, d); - return 0; -} diff --git a/vendor/github.com/golang/snappy/decode.go b/vendor/github.com/golang/snappy/decode.go deleted file mode 100644 index 72efb03..0000000 --- a/vendor/github.com/golang/snappy/decode.go +++ /dev/null @@ -1,237 +0,0 @@ -// Copyright 2011 The Snappy-Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package snappy - -import ( - "encoding/binary" - "errors" - "io" -) - -var ( - // ErrCorrupt reports that the input is invalid. - ErrCorrupt = errors.New("snappy: corrupt input") - // ErrTooLarge reports that the uncompressed length is too large. - ErrTooLarge = errors.New("snappy: decoded block is too large") - // ErrUnsupported reports that the input isn't supported. - ErrUnsupported = errors.New("snappy: unsupported input") - - errUnsupportedLiteralLength = errors.New("snappy: unsupported literal length") -) - -// DecodedLen returns the length of the decoded block. -func DecodedLen(src []byte) (int, error) { - v, _, err := decodedLen(src) - return v, err -} - -// decodedLen returns the length of the decoded block and the number of bytes -// that the length header occupied. -func decodedLen(src []byte) (blockLen, headerLen int, err error) { - v, n := binary.Uvarint(src) - if n <= 0 || v > 0xffffffff { - return 0, 0, ErrCorrupt - } - - const wordSize = 32 << (^uint(0) >> 32 & 1) - if wordSize == 32 && v > 0x7fffffff { - return 0, 0, ErrTooLarge - } - return int(v), n, nil -} - -const ( - decodeErrCodeCorrupt = 1 - decodeErrCodeUnsupportedLiteralLength = 2 -) - -// Decode returns the decoded form of src. The returned slice may be a sub- -// slice of dst if dst was large enough to hold the entire decoded block. -// Otherwise, a newly allocated slice will be returned. -// -// The dst and src must not overlap. It is valid to pass a nil dst. -func Decode(dst, src []byte) ([]byte, error) { - dLen, s, err := decodedLen(src) - if err != nil { - return nil, err - } - if dLen <= len(dst) { - dst = dst[:dLen] - } else { - dst = make([]byte, dLen) - } - switch decode(dst, src[s:]) { - case 0: - return dst, nil - case decodeErrCodeUnsupportedLiteralLength: - return nil, errUnsupportedLiteralLength - } - return nil, ErrCorrupt -} - -// NewReader returns a new Reader that decompresses from r, using the framing -// format described at -// https://github.com/google/snappy/blob/master/framing_format.txt -func NewReader(r io.Reader) *Reader { - return &Reader{ - r: r, - decoded: make([]byte, maxBlockSize), - buf: make([]byte, maxEncodedLenOfMaxBlockSize+checksumSize), - } -} - -// Reader is an io.Reader that can read Snappy-compressed bytes. -type Reader struct { - r io.Reader - err error - decoded []byte - buf []byte - // decoded[i:j] contains decoded bytes that have not yet been passed on. - i, j int - readHeader bool -} - -// Reset discards any buffered data, resets all state, and switches the Snappy -// reader to read from r. This permits reusing a Reader rather than allocating -// a new one. -func (r *Reader) Reset(reader io.Reader) { - r.r = reader - r.err = nil - r.i = 0 - r.j = 0 - r.readHeader = false -} - -func (r *Reader) readFull(p []byte, allowEOF bool) (ok bool) { - if _, r.err = io.ReadFull(r.r, p); r.err != nil { - if r.err == io.ErrUnexpectedEOF || (r.err == io.EOF && !allowEOF) { - r.err = ErrCorrupt - } - return false - } - return true -} - -// Read satisfies the io.Reader interface. -func (r *Reader) Read(p []byte) (int, error) { - if r.err != nil { - return 0, r.err - } - for { - if r.i < r.j { - n := copy(p, r.decoded[r.i:r.j]) - r.i += n - return n, nil - } - if !r.readFull(r.buf[:4], true) { - return 0, r.err - } - chunkType := r.buf[0] - if !r.readHeader { - if chunkType != chunkTypeStreamIdentifier { - r.err = ErrCorrupt - return 0, r.err - } - r.readHeader = true - } - chunkLen := int(r.buf[1]) | int(r.buf[2])<<8 | int(r.buf[3])<<16 - if chunkLen > len(r.buf) { - r.err = ErrUnsupported - return 0, r.err - } - - // The chunk types are specified at - // https://github.com/google/snappy/blob/master/framing_format.txt - switch chunkType { - case chunkTypeCompressedData: - // Section 4.2. Compressed data (chunk type 0x00). - if chunkLen < checksumSize { - r.err = ErrCorrupt - return 0, r.err - } - buf := r.buf[:chunkLen] - if !r.readFull(buf, false) { - return 0, r.err - } - checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24 - buf = buf[checksumSize:] - - n, err := DecodedLen(buf) - if err != nil { - r.err = err - return 0, r.err - } - if n > len(r.decoded) { - r.err = ErrCorrupt - return 0, r.err - } - if _, err := Decode(r.decoded, buf); err != nil { - r.err = err - return 0, r.err - } - if crc(r.decoded[:n]) != checksum { - r.err = ErrCorrupt - return 0, r.err - } - r.i, r.j = 0, n - continue - - case chunkTypeUncompressedData: - // Section 4.3. Uncompressed data (chunk type 0x01). - if chunkLen < checksumSize { - r.err = ErrCorrupt - return 0, r.err - } - buf := r.buf[:checksumSize] - if !r.readFull(buf, false) { - return 0, r.err - } - checksum := uint32(buf[0]) | uint32(buf[1])<<8 | uint32(buf[2])<<16 | uint32(buf[3])<<24 - // Read directly into r.decoded instead of via r.buf. - n := chunkLen - checksumSize - if n > len(r.decoded) { - r.err = ErrCorrupt - return 0, r.err - } - if !r.readFull(r.decoded[:n], false) { - return 0, r.err - } - if crc(r.decoded[:n]) != checksum { - r.err = ErrCorrupt - return 0, r.err - } - r.i, r.j = 0, n - continue - - case chunkTypeStreamIdentifier: - // Section 4.1. Stream identifier (chunk type 0xff). - if chunkLen != len(magicBody) { - r.err = ErrCorrupt - return 0, r.err - } - if !r.readFull(r.buf[:len(magicBody)], false) { - return 0, r.err - } - for i := 0; i < len(magicBody); i++ { - if r.buf[i] != magicBody[i] { - r.err = ErrCorrupt - return 0, r.err - } - } - continue - } - - if chunkType <= 0x7f { - // Section 4.5. Reserved unskippable chunks (chunk types 0x02-0x7f). - r.err = ErrUnsupported - return 0, r.err - } - // Section 4.4 Padding (chunk type 0xfe). - // Section 4.6. Reserved skippable chunks (chunk types 0x80-0xfd). - if !r.readFull(r.buf[:chunkLen], false) { - return 0, r.err - } - } -} diff --git a/vendor/github.com/golang/snappy/decode_amd64.go b/vendor/github.com/golang/snappy/decode_amd64.go deleted file mode 100644 index fcd192b..0000000 --- a/vendor/github.com/golang/snappy/decode_amd64.go +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright 2016 The Snappy-Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build !appengine -// +build gc -// +build !noasm - -package snappy - -// decode has the same semantics as in decode_other.go. -// -//go:noescape -func decode(dst, src []byte) int diff --git a/vendor/github.com/golang/snappy/decode_amd64.s b/vendor/github.com/golang/snappy/decode_amd64.s deleted file mode 100644 index e6179f6..0000000 --- a/vendor/github.com/golang/snappy/decode_amd64.s +++ /dev/null @@ -1,490 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build !appengine -// +build gc -// +build !noasm - -#include "textflag.h" - -// The asm code generally follows the pure Go code in decode_other.go, except -// where marked with a "!!!". - -// func decode(dst, src []byte) int -// -// All local variables fit into registers. The non-zero stack size is only to -// spill registers and push args when issuing a CALL. The register allocation: -// - AX scratch -// - BX scratch -// - CX length or x -// - DX offset -// - SI &src[s] -// - DI &dst[d] -// + R8 dst_base -// + R9 dst_len -// + R10 dst_base + dst_len -// + R11 src_base -// + R12 src_len -// + R13 src_base + src_len -// - R14 used by doCopy -// - R15 used by doCopy -// -// The registers R8-R13 (marked with a "+") are set at the start of the -// function, and after a CALL returns, and are not otherwise modified. -// -// The d variable is implicitly DI - R8, and len(dst)-d is R10 - DI. -// The s variable is implicitly SI - R11, and len(src)-s is R13 - SI. -TEXT ·decode(SB), NOSPLIT, $48-56 - // Initialize SI, DI and R8-R13. - MOVQ dst_base+0(FP), R8 - MOVQ dst_len+8(FP), R9 - MOVQ R8, DI - MOVQ R8, R10 - ADDQ R9, R10 - MOVQ src_base+24(FP), R11 - MOVQ src_len+32(FP), R12 - MOVQ R11, SI - MOVQ R11, R13 - ADDQ R12, R13 - -loop: - // for s < len(src) - CMPQ SI, R13 - JEQ end - - // CX = uint32(src[s]) - // - // switch src[s] & 0x03 - MOVBLZX (SI), CX - MOVL CX, BX - ANDL $3, BX - CMPL BX, $1 - JAE tagCopy - - // ---------------------------------------- - // The code below handles literal tags. - - // case tagLiteral: - // x := uint32(src[s] >> 2) - // switch - SHRL $2, CX - CMPL CX, $60 - JAE tagLit60Plus - - // case x < 60: - // s++ - INCQ SI - -doLit: - // This is the end of the inner "switch", when we have a literal tag. - // - // We assume that CX == x and x fits in a uint32, where x is the variable - // used in the pure Go decode_other.go code. - - // length = int(x) + 1 - // - // Unlike the pure Go code, we don't need to check if length <= 0 because - // CX can hold 64 bits, so the increment cannot overflow. - INCQ CX - - // Prepare to check if copying length bytes will run past the end of dst or - // src. - // - // AX = len(dst) - d - // BX = len(src) - s - MOVQ R10, AX - SUBQ DI, AX - MOVQ R13, BX - SUBQ SI, BX - - // !!! Try a faster technique for short (16 or fewer bytes) copies. - // - // if length > 16 || len(dst)-d < 16 || len(src)-s < 16 { - // goto callMemmove // Fall back on calling runtime·memmove. - // } - // - // The C++ snappy code calls this TryFastAppend. It also checks len(src)-s - // against 21 instead of 16, because it cannot assume that all of its input - // is contiguous in memory and so it needs to leave enough source bytes to - // read the next tag without refilling buffers, but Go's Decode assumes - // contiguousness (the src argument is a []byte). - CMPQ CX, $16 - JGT callMemmove - CMPQ AX, $16 - JLT callMemmove - CMPQ BX, $16 - JLT callMemmove - - // !!! Implement the copy from src to dst as a 16-byte load and store. - // (Decode's documentation says that dst and src must not overlap.) - // - // This always copies 16 bytes, instead of only length bytes, but that's - // OK. If the input is a valid Snappy encoding then subsequent iterations - // will fix up the overrun. Otherwise, Decode returns a nil []byte (and a - // non-nil error), so the overrun will be ignored. - // - // Note that on amd64, it is legal and cheap to issue unaligned 8-byte or - // 16-byte loads and stores. This technique probably wouldn't be as - // effective on architectures that are fussier about alignment. - MOVOU 0(SI), X0 - MOVOU X0, 0(DI) - - // d += length - // s += length - ADDQ CX, DI - ADDQ CX, SI - JMP loop - -callMemmove: - // if length > len(dst)-d || length > len(src)-s { etc } - CMPQ CX, AX - JGT errCorrupt - CMPQ CX, BX - JGT errCorrupt - - // copy(dst[d:], src[s:s+length]) - // - // This means calling runtime·memmove(&dst[d], &src[s], length), so we push - // DI, SI and CX as arguments. Coincidentally, we also need to spill those - // three registers to the stack, to save local variables across the CALL. - MOVQ DI, 0(SP) - MOVQ SI, 8(SP) - MOVQ CX, 16(SP) - MOVQ DI, 24(SP) - MOVQ SI, 32(SP) - MOVQ CX, 40(SP) - CALL runtime·memmove(SB) - - // Restore local variables: unspill registers from the stack and - // re-calculate R8-R13. - MOVQ 24(SP), DI - MOVQ 32(SP), SI - MOVQ 40(SP), CX - MOVQ dst_base+0(FP), R8 - MOVQ dst_len+8(FP), R9 - MOVQ R8, R10 - ADDQ R9, R10 - MOVQ src_base+24(FP), R11 - MOVQ src_len+32(FP), R12 - MOVQ R11, R13 - ADDQ R12, R13 - - // d += length - // s += length - ADDQ CX, DI - ADDQ CX, SI - JMP loop - -tagLit60Plus: - // !!! This fragment does the - // - // s += x - 58; if uint(s) > uint(len(src)) { etc } - // - // checks. In the asm version, we code it once instead of once per switch case. - ADDQ CX, SI - SUBQ $58, SI - MOVQ SI, BX - SUBQ R11, BX - CMPQ BX, R12 - JA errCorrupt - - // case x == 60: - CMPL CX, $61 - JEQ tagLit61 - JA tagLit62Plus - - // x = uint32(src[s-1]) - MOVBLZX -1(SI), CX - JMP doLit - -tagLit61: - // case x == 61: - // x = uint32(src[s-2]) | uint32(src[s-1])<<8 - MOVWLZX -2(SI), CX - JMP doLit - -tagLit62Plus: - CMPL CX, $62 - JA tagLit63 - - // case x == 62: - // x = uint32(src[s-3]) | uint32(src[s-2])<<8 | uint32(src[s-1])<<16 - MOVWLZX -3(SI), CX - MOVBLZX -1(SI), BX - SHLL $16, BX - ORL BX, CX - JMP doLit - -tagLit63: - // case x == 63: - // x = uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24 - MOVL -4(SI), CX - JMP doLit - -// The code above handles literal tags. -// ---------------------------------------- -// The code below handles copy tags. - -tagCopy4: - // case tagCopy4: - // s += 5 - ADDQ $5, SI - - // if uint(s) > uint(len(src)) { etc } - MOVQ SI, BX - SUBQ R11, BX - CMPQ BX, R12 - JA errCorrupt - - // length = 1 + int(src[s-5])>>2 - SHRQ $2, CX - INCQ CX - - // offset = int(uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24) - MOVLQZX -4(SI), DX - JMP doCopy - -tagCopy2: - // case tagCopy2: - // s += 3 - ADDQ $3, SI - - // if uint(s) > uint(len(src)) { etc } - MOVQ SI, BX - SUBQ R11, BX - CMPQ BX, R12 - JA errCorrupt - - // length = 1 + int(src[s-3])>>2 - SHRQ $2, CX - INCQ CX - - // offset = int(uint32(src[s-2]) | uint32(src[s-1])<<8) - MOVWQZX -2(SI), DX - JMP doCopy - -tagCopy: - // We have a copy tag. We assume that: - // - BX == src[s] & 0x03 - // - CX == src[s] - CMPQ BX, $2 - JEQ tagCopy2 - JA tagCopy4 - - // case tagCopy1: - // s += 2 - ADDQ $2, SI - - // if uint(s) > uint(len(src)) { etc } - MOVQ SI, BX - SUBQ R11, BX - CMPQ BX, R12 - JA errCorrupt - - // offset = int(uint32(src[s-2])&0xe0<<3 | uint32(src[s-1])) - MOVQ CX, DX - ANDQ $0xe0, DX - SHLQ $3, DX - MOVBQZX -1(SI), BX - ORQ BX, DX - - // length = 4 + int(src[s-2])>>2&0x7 - SHRQ $2, CX - ANDQ $7, CX - ADDQ $4, CX - -doCopy: - // This is the end of the outer "switch", when we have a copy tag. - // - // We assume that: - // - CX == length && CX > 0 - // - DX == offset - - // if offset <= 0 { etc } - CMPQ DX, $0 - JLE errCorrupt - - // if d < offset { etc } - MOVQ DI, BX - SUBQ R8, BX - CMPQ BX, DX - JLT errCorrupt - - // if length > len(dst)-d { etc } - MOVQ R10, BX - SUBQ DI, BX - CMPQ CX, BX - JGT errCorrupt - - // forwardCopy(dst[d:d+length], dst[d-offset:]); d += length - // - // Set: - // - R14 = len(dst)-d - // - R15 = &dst[d-offset] - MOVQ R10, R14 - SUBQ DI, R14 - MOVQ DI, R15 - SUBQ DX, R15 - - // !!! Try a faster technique for short (16 or fewer bytes) forward copies. - // - // First, try using two 8-byte load/stores, similar to the doLit technique - // above. Even if dst[d:d+length] and dst[d-offset:] can overlap, this is - // still OK if offset >= 8. Note that this has to be two 8-byte load/stores - // and not one 16-byte load/store, and the first store has to be before the - // second load, due to the overlap if offset is in the range [8, 16). - // - // if length > 16 || offset < 8 || len(dst)-d < 16 { - // goto slowForwardCopy - // } - // copy 16 bytes - // d += length - CMPQ CX, $16 - JGT slowForwardCopy - CMPQ DX, $8 - JLT slowForwardCopy - CMPQ R14, $16 - JLT slowForwardCopy - MOVQ 0(R15), AX - MOVQ AX, 0(DI) - MOVQ 8(R15), BX - MOVQ BX, 8(DI) - ADDQ CX, DI - JMP loop - -slowForwardCopy: - // !!! If the forward copy is longer than 16 bytes, or if offset < 8, we - // can still try 8-byte load stores, provided we can overrun up to 10 extra - // bytes. As above, the overrun will be fixed up by subsequent iterations - // of the outermost loop. - // - // The C++ snappy code calls this technique IncrementalCopyFastPath. Its - // commentary says: - // - // ---- - // - // The main part of this loop is a simple copy of eight bytes at a time - // until we've copied (at least) the requested amount of bytes. However, - // if d and d-offset are less than eight bytes apart (indicating a - // repeating pattern of length < 8), we first need to expand the pattern in - // order to get the correct results. For instance, if the buffer looks like - // this, with the eight-byte and patterns marked as - // intervals: - // - // abxxxxxxxxxxxx - // [------] d-offset - // [------] d - // - // a single eight-byte copy from to will repeat the pattern - // once, after which we can move two bytes without moving : - // - // ababxxxxxxxxxx - // [------] d-offset - // [------] d - // - // and repeat the exercise until the two no longer overlap. - // - // This allows us to do very well in the special case of one single byte - // repeated many times, without taking a big hit for more general cases. - // - // The worst case of extra writing past the end of the match occurs when - // offset == 1 and length == 1; the last copy will read from byte positions - // [0..7] and write to [4..11], whereas it was only supposed to write to - // position 1. Thus, ten excess bytes. - // - // ---- - // - // That "10 byte overrun" worst case is confirmed by Go's - // TestSlowForwardCopyOverrun, which also tests the fixUpSlowForwardCopy - // and finishSlowForwardCopy algorithm. - // - // if length > len(dst)-d-10 { - // goto verySlowForwardCopy - // } - SUBQ $10, R14 - CMPQ CX, R14 - JGT verySlowForwardCopy - -makeOffsetAtLeast8: - // !!! As above, expand the pattern so that offset >= 8 and we can use - // 8-byte load/stores. - // - // for offset < 8 { - // copy 8 bytes from dst[d-offset:] to dst[d:] - // length -= offset - // d += offset - // offset += offset - // // The two previous lines together means that d-offset, and therefore - // // R15, is unchanged. - // } - CMPQ DX, $8 - JGE fixUpSlowForwardCopy - MOVQ (R15), BX - MOVQ BX, (DI) - SUBQ DX, CX - ADDQ DX, DI - ADDQ DX, DX - JMP makeOffsetAtLeast8 - -fixUpSlowForwardCopy: - // !!! Add length (which might be negative now) to d (implied by DI being - // &dst[d]) so that d ends up at the right place when we jump back to the - // top of the loop. Before we do that, though, we save DI to AX so that, if - // length is positive, copying the remaining length bytes will write to the - // right place. - MOVQ DI, AX - ADDQ CX, DI - -finishSlowForwardCopy: - // !!! Repeat 8-byte load/stores until length <= 0. Ending with a negative - // length means that we overrun, but as above, that will be fixed up by - // subsequent iterations of the outermost loop. - CMPQ CX, $0 - JLE loop - MOVQ (R15), BX - MOVQ BX, (AX) - ADDQ $8, R15 - ADDQ $8, AX - SUBQ $8, CX - JMP finishSlowForwardCopy - -verySlowForwardCopy: - // verySlowForwardCopy is a simple implementation of forward copy. In C - // parlance, this is a do/while loop instead of a while loop, since we know - // that length > 0. In Go syntax: - // - // for { - // dst[d] = dst[d - offset] - // d++ - // length-- - // if length == 0 { - // break - // } - // } - MOVB (R15), BX - MOVB BX, (DI) - INCQ R15 - INCQ DI - DECQ CX - JNZ verySlowForwardCopy - JMP loop - -// The code above handles copy tags. -// ---------------------------------------- - -end: - // This is the end of the "for s < len(src)". - // - // if d != len(dst) { etc } - CMPQ DI, R10 - JNE errCorrupt - - // return 0 - MOVQ $0, ret+48(FP) - RET - -errCorrupt: - // return decodeErrCodeCorrupt - MOVQ $1, ret+48(FP) - RET diff --git a/vendor/github.com/golang/snappy/decode_other.go b/vendor/github.com/golang/snappy/decode_other.go deleted file mode 100644 index 8c9f204..0000000 --- a/vendor/github.com/golang/snappy/decode_other.go +++ /dev/null @@ -1,101 +0,0 @@ -// Copyright 2016 The Snappy-Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build !amd64 appengine !gc noasm - -package snappy - -// decode writes the decoding of src to dst. It assumes that the varint-encoded -// length of the decompressed bytes has already been read, and that len(dst) -// equals that length. -// -// It returns 0 on success or a decodeErrCodeXxx error code on failure. -func decode(dst, src []byte) int { - var d, s, offset, length int - for s < len(src) { - switch src[s] & 0x03 { - case tagLiteral: - x := uint32(src[s] >> 2) - switch { - case x < 60: - s++ - case x == 60: - s += 2 - if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. - return decodeErrCodeCorrupt - } - x = uint32(src[s-1]) - case x == 61: - s += 3 - if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. - return decodeErrCodeCorrupt - } - x = uint32(src[s-2]) | uint32(src[s-1])<<8 - case x == 62: - s += 4 - if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. - return decodeErrCodeCorrupt - } - x = uint32(src[s-3]) | uint32(src[s-2])<<8 | uint32(src[s-1])<<16 - case x == 63: - s += 5 - if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. - return decodeErrCodeCorrupt - } - x = uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24 - } - length = int(x) + 1 - if length <= 0 { - return decodeErrCodeUnsupportedLiteralLength - } - if length > len(dst)-d || length > len(src)-s { - return decodeErrCodeCorrupt - } - copy(dst[d:], src[s:s+length]) - d += length - s += length - continue - - case tagCopy1: - s += 2 - if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. - return decodeErrCodeCorrupt - } - length = 4 + int(src[s-2])>>2&0x7 - offset = int(uint32(src[s-2])&0xe0<<3 | uint32(src[s-1])) - - case tagCopy2: - s += 3 - if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. - return decodeErrCodeCorrupt - } - length = 1 + int(src[s-3])>>2 - offset = int(uint32(src[s-2]) | uint32(src[s-1])<<8) - - case tagCopy4: - s += 5 - if uint(s) > uint(len(src)) { // The uint conversions catch overflow from the previous line. - return decodeErrCodeCorrupt - } - length = 1 + int(src[s-5])>>2 - offset = int(uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24) - } - - if offset <= 0 || d < offset || length > len(dst)-d { - return decodeErrCodeCorrupt - } - // Copy from an earlier sub-slice of dst to a later sub-slice. Unlike - // the built-in copy function, this byte-by-byte copy always runs - // forwards, even if the slices overlap. Conceptually, this is: - // - // d += forwardCopy(dst[d:d+length], dst[d-offset:]) - for end := d + length; d != end; d++ { - dst[d] = dst[d-offset] - } - } - if d != len(dst) { - return decodeErrCodeCorrupt - } - return 0 -} diff --git a/vendor/github.com/golang/snappy/encode.go b/vendor/github.com/golang/snappy/encode.go deleted file mode 100644 index 8d393e9..0000000 --- a/vendor/github.com/golang/snappy/encode.go +++ /dev/null @@ -1,285 +0,0 @@ -// Copyright 2011 The Snappy-Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package snappy - -import ( - "encoding/binary" - "errors" - "io" -) - -// Encode returns the encoded form of src. The returned slice may be a sub- -// slice of dst if dst was large enough to hold the entire encoded block. -// Otherwise, a newly allocated slice will be returned. -// -// The dst and src must not overlap. It is valid to pass a nil dst. -func Encode(dst, src []byte) []byte { - if n := MaxEncodedLen(len(src)); n < 0 { - panic(ErrTooLarge) - } else if len(dst) < n { - dst = make([]byte, n) - } - - // The block starts with the varint-encoded length of the decompressed bytes. - d := binary.PutUvarint(dst, uint64(len(src))) - - for len(src) > 0 { - p := src - src = nil - if len(p) > maxBlockSize { - p, src = p[:maxBlockSize], p[maxBlockSize:] - } - if len(p) < minNonLiteralBlockSize { - d += emitLiteral(dst[d:], p) - } else { - d += encodeBlock(dst[d:], p) - } - } - return dst[:d] -} - -// inputMargin is the minimum number of extra input bytes to keep, inside -// encodeBlock's inner loop. On some architectures, this margin lets us -// implement a fast path for emitLiteral, where the copy of short (<= 16 byte) -// literals can be implemented as a single load to and store from a 16-byte -// register. That literal's actual length can be as short as 1 byte, so this -// can copy up to 15 bytes too much, but that's OK as subsequent iterations of -// the encoding loop will fix up the copy overrun, and this inputMargin ensures -// that we don't overrun the dst and src buffers. -const inputMargin = 16 - 1 - -// minNonLiteralBlockSize is the minimum size of the input to encodeBlock that -// could be encoded with a copy tag. This is the minimum with respect to the -// algorithm used by encodeBlock, not a minimum enforced by the file format. -// -// The encoded output must start with at least a 1 byte literal, as there are -// no previous bytes to copy. A minimal (1 byte) copy after that, generated -// from an emitCopy call in encodeBlock's main loop, would require at least -// another inputMargin bytes, for the reason above: we want any emitLiteral -// calls inside encodeBlock's main loop to use the fast path if possible, which -// requires being able to overrun by inputMargin bytes. Thus, -// minNonLiteralBlockSize equals 1 + 1 + inputMargin. -// -// The C++ code doesn't use this exact threshold, but it could, as discussed at -// https://groups.google.com/d/topic/snappy-compression/oGbhsdIJSJ8/discussion -// The difference between Go (2+inputMargin) and C++ (inputMargin) is purely an -// optimization. It should not affect the encoded form. This is tested by -// TestSameEncodingAsCppShortCopies. -const minNonLiteralBlockSize = 1 + 1 + inputMargin - -// MaxEncodedLen returns the maximum length of a snappy block, given its -// uncompressed length. -// -// It will return a negative value if srcLen is too large to encode. -func MaxEncodedLen(srcLen int) int { - n := uint64(srcLen) - if n > 0xffffffff { - return -1 - } - // Compressed data can be defined as: - // compressed := item* literal* - // item := literal* copy - // - // The trailing literal sequence has a space blowup of at most 62/60 - // since a literal of length 60 needs one tag byte + one extra byte - // for length information. - // - // Item blowup is trickier to measure. Suppose the "copy" op copies - // 4 bytes of data. Because of a special check in the encoding code, - // we produce a 4-byte copy only if the offset is < 65536. Therefore - // the copy op takes 3 bytes to encode, and this type of item leads - // to at most the 62/60 blowup for representing literals. - // - // Suppose the "copy" op copies 5 bytes of data. If the offset is big - // enough, it will take 5 bytes to encode the copy op. Therefore the - // worst case here is a one-byte literal followed by a five-byte copy. - // That is, 6 bytes of input turn into 7 bytes of "compressed" data. - // - // This last factor dominates the blowup, so the final estimate is: - n = 32 + n + n/6 - if n > 0xffffffff { - return -1 - } - return int(n) -} - -var errClosed = errors.New("snappy: Writer is closed") - -// NewWriter returns a new Writer that compresses to w. -// -// The Writer returned does not buffer writes. There is no need to Flush or -// Close such a Writer. -// -// Deprecated: the Writer returned is not suitable for many small writes, only -// for few large writes. Use NewBufferedWriter instead, which is efficient -// regardless of the frequency and shape of the writes, and remember to Close -// that Writer when done. -func NewWriter(w io.Writer) *Writer { - return &Writer{ - w: w, - obuf: make([]byte, obufLen), - } -} - -// NewBufferedWriter returns a new Writer that compresses to w, using the -// framing format described at -// https://github.com/google/snappy/blob/master/framing_format.txt -// -// The Writer returned buffers writes. Users must call Close to guarantee all -// data has been forwarded to the underlying io.Writer. They may also call -// Flush zero or more times before calling Close. -func NewBufferedWriter(w io.Writer) *Writer { - return &Writer{ - w: w, - ibuf: make([]byte, 0, maxBlockSize), - obuf: make([]byte, obufLen), - } -} - -// Writer is an io.Writer that can write Snappy-compressed bytes. -type Writer struct { - w io.Writer - err error - - // ibuf is a buffer for the incoming (uncompressed) bytes. - // - // Its use is optional. For backwards compatibility, Writers created by the - // NewWriter function have ibuf == nil, do not buffer incoming bytes, and - // therefore do not need to be Flush'ed or Close'd. - ibuf []byte - - // obuf is a buffer for the outgoing (compressed) bytes. - obuf []byte - - // wroteStreamHeader is whether we have written the stream header. - wroteStreamHeader bool -} - -// Reset discards the writer's state and switches the Snappy writer to write to -// w. This permits reusing a Writer rather than allocating a new one. -func (w *Writer) Reset(writer io.Writer) { - w.w = writer - w.err = nil - if w.ibuf != nil { - w.ibuf = w.ibuf[:0] - } - w.wroteStreamHeader = false -} - -// Write satisfies the io.Writer interface. -func (w *Writer) Write(p []byte) (nRet int, errRet error) { - if w.ibuf == nil { - // Do not buffer incoming bytes. This does not perform or compress well - // if the caller of Writer.Write writes many small slices. This - // behavior is therefore deprecated, but still supported for backwards - // compatibility with code that doesn't explicitly Flush or Close. - return w.write(p) - } - - // The remainder of this method is based on bufio.Writer.Write from the - // standard library. - - for len(p) > (cap(w.ibuf)-len(w.ibuf)) && w.err == nil { - var n int - if len(w.ibuf) == 0 { - // Large write, empty buffer. - // Write directly from p to avoid copy. - n, _ = w.write(p) - } else { - n = copy(w.ibuf[len(w.ibuf):cap(w.ibuf)], p) - w.ibuf = w.ibuf[:len(w.ibuf)+n] - w.Flush() - } - nRet += n - p = p[n:] - } - if w.err != nil { - return nRet, w.err - } - n := copy(w.ibuf[len(w.ibuf):cap(w.ibuf)], p) - w.ibuf = w.ibuf[:len(w.ibuf)+n] - nRet += n - return nRet, nil -} - -func (w *Writer) write(p []byte) (nRet int, errRet error) { - if w.err != nil { - return 0, w.err - } - for len(p) > 0 { - obufStart := len(magicChunk) - if !w.wroteStreamHeader { - w.wroteStreamHeader = true - copy(w.obuf, magicChunk) - obufStart = 0 - } - - var uncompressed []byte - if len(p) > maxBlockSize { - uncompressed, p = p[:maxBlockSize], p[maxBlockSize:] - } else { - uncompressed, p = p, nil - } - checksum := crc(uncompressed) - - // Compress the buffer, discarding the result if the improvement - // isn't at least 12.5%. - compressed := Encode(w.obuf[obufHeaderLen:], uncompressed) - chunkType := uint8(chunkTypeCompressedData) - chunkLen := 4 + len(compressed) - obufEnd := obufHeaderLen + len(compressed) - if len(compressed) >= len(uncompressed)-len(uncompressed)/8 { - chunkType = chunkTypeUncompressedData - chunkLen = 4 + len(uncompressed) - obufEnd = obufHeaderLen - } - - // Fill in the per-chunk header that comes before the body. - w.obuf[len(magicChunk)+0] = chunkType - w.obuf[len(magicChunk)+1] = uint8(chunkLen >> 0) - w.obuf[len(magicChunk)+2] = uint8(chunkLen >> 8) - w.obuf[len(magicChunk)+3] = uint8(chunkLen >> 16) - w.obuf[len(magicChunk)+4] = uint8(checksum >> 0) - w.obuf[len(magicChunk)+5] = uint8(checksum >> 8) - w.obuf[len(magicChunk)+6] = uint8(checksum >> 16) - w.obuf[len(magicChunk)+7] = uint8(checksum >> 24) - - if _, err := w.w.Write(w.obuf[obufStart:obufEnd]); err != nil { - w.err = err - return nRet, err - } - if chunkType == chunkTypeUncompressedData { - if _, err := w.w.Write(uncompressed); err != nil { - w.err = err - return nRet, err - } - } - nRet += len(uncompressed) - } - return nRet, nil -} - -// Flush flushes the Writer to its underlying io.Writer. -func (w *Writer) Flush() error { - if w.err != nil { - return w.err - } - if len(w.ibuf) == 0 { - return nil - } - w.write(w.ibuf) - w.ibuf = w.ibuf[:0] - return w.err -} - -// Close calls Flush and then closes the Writer. -func (w *Writer) Close() error { - w.Flush() - ret := w.err - if w.err == nil { - w.err = errClosed - } - return ret -} diff --git a/vendor/github.com/golang/snappy/encode_amd64.go b/vendor/github.com/golang/snappy/encode_amd64.go deleted file mode 100644 index 150d91b..0000000 --- a/vendor/github.com/golang/snappy/encode_amd64.go +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright 2016 The Snappy-Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build !appengine -// +build gc -// +build !noasm - -package snappy - -// emitLiteral has the same semantics as in encode_other.go. -// -//go:noescape -func emitLiteral(dst, lit []byte) int - -// emitCopy has the same semantics as in encode_other.go. -// -//go:noescape -func emitCopy(dst []byte, offset, length int) int - -// extendMatch has the same semantics as in encode_other.go. -// -//go:noescape -func extendMatch(src []byte, i, j int) int - -// encodeBlock has the same semantics as in encode_other.go. -// -//go:noescape -func encodeBlock(dst, src []byte) (d int) diff --git a/vendor/github.com/golang/snappy/encode_amd64.s b/vendor/github.com/golang/snappy/encode_amd64.s deleted file mode 100644 index adfd979..0000000 --- a/vendor/github.com/golang/snappy/encode_amd64.s +++ /dev/null @@ -1,730 +0,0 @@ -// Copyright 2016 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build !appengine -// +build gc -// +build !noasm - -#include "textflag.h" - -// The XXX lines assemble on Go 1.4, 1.5 and 1.7, but not 1.6, due to a -// Go toolchain regression. See https://github.com/golang/go/issues/15426 and -// https://github.com/golang/snappy/issues/29 -// -// As a workaround, the package was built with a known good assembler, and -// those instructions were disassembled by "objdump -d" to yield the -// 4e 0f b7 7c 5c 78 movzwq 0x78(%rsp,%r11,2),%r15 -// style comments, in AT&T asm syntax. Note that rsp here is a physical -// register, not Go/asm's SP pseudo-register (see https://golang.org/doc/asm). -// The instructions were then encoded as "BYTE $0x.." sequences, which assemble -// fine on Go 1.6. - -// The asm code generally follows the pure Go code in encode_other.go, except -// where marked with a "!!!". - -// ---------------------------------------------------------------------------- - -// func emitLiteral(dst, lit []byte) int -// -// All local variables fit into registers. The register allocation: -// - AX len(lit) -// - BX n -// - DX return value -// - DI &dst[i] -// - R10 &lit[0] -// -// The 24 bytes of stack space is to call runtime·memmove. -// -// The unusual register allocation of local variables, such as R10 for the -// source pointer, matches the allocation used at the call site in encodeBlock, -// which makes it easier to manually inline this function. -TEXT ·emitLiteral(SB), NOSPLIT, $24-56 - MOVQ dst_base+0(FP), DI - MOVQ lit_base+24(FP), R10 - MOVQ lit_len+32(FP), AX - MOVQ AX, DX - MOVL AX, BX - SUBL $1, BX - - CMPL BX, $60 - JLT oneByte - CMPL BX, $256 - JLT twoBytes - -threeBytes: - MOVB $0xf4, 0(DI) - MOVW BX, 1(DI) - ADDQ $3, DI - ADDQ $3, DX - JMP memmove - -twoBytes: - MOVB $0xf0, 0(DI) - MOVB BX, 1(DI) - ADDQ $2, DI - ADDQ $2, DX - JMP memmove - -oneByte: - SHLB $2, BX - MOVB BX, 0(DI) - ADDQ $1, DI - ADDQ $1, DX - -memmove: - MOVQ DX, ret+48(FP) - - // copy(dst[i:], lit) - // - // This means calling runtime·memmove(&dst[i], &lit[0], len(lit)), so we push - // DI, R10 and AX as arguments. - MOVQ DI, 0(SP) - MOVQ R10, 8(SP) - MOVQ AX, 16(SP) - CALL runtime·memmove(SB) - RET - -// ---------------------------------------------------------------------------- - -// func emitCopy(dst []byte, offset, length int) int -// -// All local variables fit into registers. The register allocation: -// - AX length -// - SI &dst[0] -// - DI &dst[i] -// - R11 offset -// -// The unusual register allocation of local variables, such as R11 for the -// offset, matches the allocation used at the call site in encodeBlock, which -// makes it easier to manually inline this function. -TEXT ·emitCopy(SB), NOSPLIT, $0-48 - MOVQ dst_base+0(FP), DI - MOVQ DI, SI - MOVQ offset+24(FP), R11 - MOVQ length+32(FP), AX - -loop0: - // for length >= 68 { etc } - CMPL AX, $68 - JLT step1 - - // Emit a length 64 copy, encoded as 3 bytes. - MOVB $0xfe, 0(DI) - MOVW R11, 1(DI) - ADDQ $3, DI - SUBL $64, AX - JMP loop0 - -step1: - // if length > 64 { etc } - CMPL AX, $64 - JLE step2 - - // Emit a length 60 copy, encoded as 3 bytes. - MOVB $0xee, 0(DI) - MOVW R11, 1(DI) - ADDQ $3, DI - SUBL $60, AX - -step2: - // if length >= 12 || offset >= 2048 { goto step3 } - CMPL AX, $12 - JGE step3 - CMPL R11, $2048 - JGE step3 - - // Emit the remaining copy, encoded as 2 bytes. - MOVB R11, 1(DI) - SHRL $8, R11 - SHLB $5, R11 - SUBB $4, AX - SHLB $2, AX - ORB AX, R11 - ORB $1, R11 - MOVB R11, 0(DI) - ADDQ $2, DI - - // Return the number of bytes written. - SUBQ SI, DI - MOVQ DI, ret+40(FP) - RET - -step3: - // Emit the remaining copy, encoded as 3 bytes. - SUBL $1, AX - SHLB $2, AX - ORB $2, AX - MOVB AX, 0(DI) - MOVW R11, 1(DI) - ADDQ $3, DI - - // Return the number of bytes written. - SUBQ SI, DI - MOVQ DI, ret+40(FP) - RET - -// ---------------------------------------------------------------------------- - -// func extendMatch(src []byte, i, j int) int -// -// All local variables fit into registers. The register allocation: -// - DX &src[0] -// - SI &src[j] -// - R13 &src[len(src) - 8] -// - R14 &src[len(src)] -// - R15 &src[i] -// -// The unusual register allocation of local variables, such as R15 for a source -// pointer, matches the allocation used at the call site in encodeBlock, which -// makes it easier to manually inline this function. -TEXT ·extendMatch(SB), NOSPLIT, $0-48 - MOVQ src_base+0(FP), DX - MOVQ src_len+8(FP), R14 - MOVQ i+24(FP), R15 - MOVQ j+32(FP), SI - ADDQ DX, R14 - ADDQ DX, R15 - ADDQ DX, SI - MOVQ R14, R13 - SUBQ $8, R13 - -cmp8: - // As long as we are 8 or more bytes before the end of src, we can load and - // compare 8 bytes at a time. If those 8 bytes are equal, repeat. - CMPQ SI, R13 - JA cmp1 - MOVQ (R15), AX - MOVQ (SI), BX - CMPQ AX, BX - JNE bsf - ADDQ $8, R15 - ADDQ $8, SI - JMP cmp8 - -bsf: - // If those 8 bytes were not equal, XOR the two 8 byte values, and return - // the index of the first byte that differs. The BSF instruction finds the - // least significant 1 bit, the amd64 architecture is little-endian, and - // the shift by 3 converts a bit index to a byte index. - XORQ AX, BX - BSFQ BX, BX - SHRQ $3, BX - ADDQ BX, SI - - // Convert from &src[ret] to ret. - SUBQ DX, SI - MOVQ SI, ret+40(FP) - RET - -cmp1: - // In src's tail, compare 1 byte at a time. - CMPQ SI, R14 - JAE extendMatchEnd - MOVB (R15), AX - MOVB (SI), BX - CMPB AX, BX - JNE extendMatchEnd - ADDQ $1, R15 - ADDQ $1, SI - JMP cmp1 - -extendMatchEnd: - // Convert from &src[ret] to ret. - SUBQ DX, SI - MOVQ SI, ret+40(FP) - RET - -// ---------------------------------------------------------------------------- - -// func encodeBlock(dst, src []byte) (d int) -// -// All local variables fit into registers, other than "var table". The register -// allocation: -// - AX . . -// - BX . . -// - CX 56 shift (note that amd64 shifts by non-immediates must use CX). -// - DX 64 &src[0], tableSize -// - SI 72 &src[s] -// - DI 80 &dst[d] -// - R9 88 sLimit -// - R10 . &src[nextEmit] -// - R11 96 prevHash, currHash, nextHash, offset -// - R12 104 &src[base], skip -// - R13 . &src[nextS], &src[len(src) - 8] -// - R14 . len(src), bytesBetweenHashLookups, &src[len(src)], x -// - R15 112 candidate -// -// The second column (56, 64, etc) is the stack offset to spill the registers -// when calling other functions. We could pack this slightly tighter, but it's -// simpler to have a dedicated spill map independent of the function called. -// -// "var table [maxTableSize]uint16" takes up 32768 bytes of stack space. An -// extra 56 bytes, to call other functions, and an extra 64 bytes, to spill -// local variables (registers) during calls gives 32768 + 56 + 64 = 32888. -TEXT ·encodeBlock(SB), 0, $32888-56 - MOVQ dst_base+0(FP), DI - MOVQ src_base+24(FP), SI - MOVQ src_len+32(FP), R14 - - // shift, tableSize := uint32(32-8), 1<<8 - MOVQ $24, CX - MOVQ $256, DX - -calcShift: - // for ; tableSize < maxTableSize && tableSize < len(src); tableSize *= 2 { - // shift-- - // } - CMPQ DX, $16384 - JGE varTable - CMPQ DX, R14 - JGE varTable - SUBQ $1, CX - SHLQ $1, DX - JMP calcShift - -varTable: - // var table [maxTableSize]uint16 - // - // In the asm code, unlike the Go code, we can zero-initialize only the - // first tableSize elements. Each uint16 element is 2 bytes and each MOVOU - // writes 16 bytes, so we can do only tableSize/8 writes instead of the - // 2048 writes that would zero-initialize all of table's 32768 bytes. - SHRQ $3, DX - LEAQ table-32768(SP), BX - PXOR X0, X0 - -memclr: - MOVOU X0, 0(BX) - ADDQ $16, BX - SUBQ $1, DX - JNZ memclr - - // !!! DX = &src[0] - MOVQ SI, DX - - // sLimit := len(src) - inputMargin - MOVQ R14, R9 - SUBQ $15, R9 - - // !!! Pre-emptively spill CX, DX and R9 to the stack. Their values don't - // change for the rest of the function. - MOVQ CX, 56(SP) - MOVQ DX, 64(SP) - MOVQ R9, 88(SP) - - // nextEmit := 0 - MOVQ DX, R10 - - // s := 1 - ADDQ $1, SI - - // nextHash := hash(load32(src, s), shift) - MOVL 0(SI), R11 - IMULL $0x1e35a7bd, R11 - SHRL CX, R11 - -outer: - // for { etc } - - // skip := 32 - MOVQ $32, R12 - - // nextS := s - MOVQ SI, R13 - - // candidate := 0 - MOVQ $0, R15 - -inner0: - // for { etc } - - // s := nextS - MOVQ R13, SI - - // bytesBetweenHashLookups := skip >> 5 - MOVQ R12, R14 - SHRQ $5, R14 - - // nextS = s + bytesBetweenHashLookups - ADDQ R14, R13 - - // skip += bytesBetweenHashLookups - ADDQ R14, R12 - - // if nextS > sLimit { goto emitRemainder } - MOVQ R13, AX - SUBQ DX, AX - CMPQ AX, R9 - JA emitRemainder - - // candidate = int(table[nextHash]) - // XXX: MOVWQZX table-32768(SP)(R11*2), R15 - // XXX: 4e 0f b7 7c 5c 78 movzwq 0x78(%rsp,%r11,2),%r15 - BYTE $0x4e - BYTE $0x0f - BYTE $0xb7 - BYTE $0x7c - BYTE $0x5c - BYTE $0x78 - - // table[nextHash] = uint16(s) - MOVQ SI, AX - SUBQ DX, AX - - // XXX: MOVW AX, table-32768(SP)(R11*2) - // XXX: 66 42 89 44 5c 78 mov %ax,0x78(%rsp,%r11,2) - BYTE $0x66 - BYTE $0x42 - BYTE $0x89 - BYTE $0x44 - BYTE $0x5c - BYTE $0x78 - - // nextHash = hash(load32(src, nextS), shift) - MOVL 0(R13), R11 - IMULL $0x1e35a7bd, R11 - SHRL CX, R11 - - // if load32(src, s) != load32(src, candidate) { continue } break - MOVL 0(SI), AX - MOVL (DX)(R15*1), BX - CMPL AX, BX - JNE inner0 - -fourByteMatch: - // As per the encode_other.go code: - // - // A 4-byte match has been found. We'll later see etc. - - // !!! Jump to a fast path for short (<= 16 byte) literals. See the comment - // on inputMargin in encode.go. - MOVQ SI, AX - SUBQ R10, AX - CMPQ AX, $16 - JLE emitLiteralFastPath - - // ---------------------------------------- - // Begin inline of the emitLiteral call. - // - // d += emitLiteral(dst[d:], src[nextEmit:s]) - - MOVL AX, BX - SUBL $1, BX - - CMPL BX, $60 - JLT inlineEmitLiteralOneByte - CMPL BX, $256 - JLT inlineEmitLiteralTwoBytes - -inlineEmitLiteralThreeBytes: - MOVB $0xf4, 0(DI) - MOVW BX, 1(DI) - ADDQ $3, DI - JMP inlineEmitLiteralMemmove - -inlineEmitLiteralTwoBytes: - MOVB $0xf0, 0(DI) - MOVB BX, 1(DI) - ADDQ $2, DI - JMP inlineEmitLiteralMemmove - -inlineEmitLiteralOneByte: - SHLB $2, BX - MOVB BX, 0(DI) - ADDQ $1, DI - -inlineEmitLiteralMemmove: - // Spill local variables (registers) onto the stack; call; unspill. - // - // copy(dst[i:], lit) - // - // This means calling runtime·memmove(&dst[i], &lit[0], len(lit)), so we push - // DI, R10 and AX as arguments. - MOVQ DI, 0(SP) - MOVQ R10, 8(SP) - MOVQ AX, 16(SP) - ADDQ AX, DI // Finish the "d +=" part of "d += emitLiteral(etc)". - MOVQ SI, 72(SP) - MOVQ DI, 80(SP) - MOVQ R15, 112(SP) - CALL runtime·memmove(SB) - MOVQ 56(SP), CX - MOVQ 64(SP), DX - MOVQ 72(SP), SI - MOVQ 80(SP), DI - MOVQ 88(SP), R9 - MOVQ 112(SP), R15 - JMP inner1 - -inlineEmitLiteralEnd: - // End inline of the emitLiteral call. - // ---------------------------------------- - -emitLiteralFastPath: - // !!! Emit the 1-byte encoding "uint8(len(lit)-1)<<2". - MOVB AX, BX - SUBB $1, BX - SHLB $2, BX - MOVB BX, (DI) - ADDQ $1, DI - - // !!! Implement the copy from lit to dst as a 16-byte load and store. - // (Encode's documentation says that dst and src must not overlap.) - // - // This always copies 16 bytes, instead of only len(lit) bytes, but that's - // OK. Subsequent iterations will fix up the overrun. - // - // Note that on amd64, it is legal and cheap to issue unaligned 8-byte or - // 16-byte loads and stores. This technique probably wouldn't be as - // effective on architectures that are fussier about alignment. - MOVOU 0(R10), X0 - MOVOU X0, 0(DI) - ADDQ AX, DI - -inner1: - // for { etc } - - // base := s - MOVQ SI, R12 - - // !!! offset := base - candidate - MOVQ R12, R11 - SUBQ R15, R11 - SUBQ DX, R11 - - // ---------------------------------------- - // Begin inline of the extendMatch call. - // - // s = extendMatch(src, candidate+4, s+4) - - // !!! R14 = &src[len(src)] - MOVQ src_len+32(FP), R14 - ADDQ DX, R14 - - // !!! R13 = &src[len(src) - 8] - MOVQ R14, R13 - SUBQ $8, R13 - - // !!! R15 = &src[candidate + 4] - ADDQ $4, R15 - ADDQ DX, R15 - - // !!! s += 4 - ADDQ $4, SI - -inlineExtendMatchCmp8: - // As long as we are 8 or more bytes before the end of src, we can load and - // compare 8 bytes at a time. If those 8 bytes are equal, repeat. - CMPQ SI, R13 - JA inlineExtendMatchCmp1 - MOVQ (R15), AX - MOVQ (SI), BX - CMPQ AX, BX - JNE inlineExtendMatchBSF - ADDQ $8, R15 - ADDQ $8, SI - JMP inlineExtendMatchCmp8 - -inlineExtendMatchBSF: - // If those 8 bytes were not equal, XOR the two 8 byte values, and return - // the index of the first byte that differs. The BSF instruction finds the - // least significant 1 bit, the amd64 architecture is little-endian, and - // the shift by 3 converts a bit index to a byte index. - XORQ AX, BX - BSFQ BX, BX - SHRQ $3, BX - ADDQ BX, SI - JMP inlineExtendMatchEnd - -inlineExtendMatchCmp1: - // In src's tail, compare 1 byte at a time. - CMPQ SI, R14 - JAE inlineExtendMatchEnd - MOVB (R15), AX - MOVB (SI), BX - CMPB AX, BX - JNE inlineExtendMatchEnd - ADDQ $1, R15 - ADDQ $1, SI - JMP inlineExtendMatchCmp1 - -inlineExtendMatchEnd: - // End inline of the extendMatch call. - // ---------------------------------------- - - // ---------------------------------------- - // Begin inline of the emitCopy call. - // - // d += emitCopy(dst[d:], base-candidate, s-base) - - // !!! length := s - base - MOVQ SI, AX - SUBQ R12, AX - -inlineEmitCopyLoop0: - // for length >= 68 { etc } - CMPL AX, $68 - JLT inlineEmitCopyStep1 - - // Emit a length 64 copy, encoded as 3 bytes. - MOVB $0xfe, 0(DI) - MOVW R11, 1(DI) - ADDQ $3, DI - SUBL $64, AX - JMP inlineEmitCopyLoop0 - -inlineEmitCopyStep1: - // if length > 64 { etc } - CMPL AX, $64 - JLE inlineEmitCopyStep2 - - // Emit a length 60 copy, encoded as 3 bytes. - MOVB $0xee, 0(DI) - MOVW R11, 1(DI) - ADDQ $3, DI - SUBL $60, AX - -inlineEmitCopyStep2: - // if length >= 12 || offset >= 2048 { goto inlineEmitCopyStep3 } - CMPL AX, $12 - JGE inlineEmitCopyStep3 - CMPL R11, $2048 - JGE inlineEmitCopyStep3 - - // Emit the remaining copy, encoded as 2 bytes. - MOVB R11, 1(DI) - SHRL $8, R11 - SHLB $5, R11 - SUBB $4, AX - SHLB $2, AX - ORB AX, R11 - ORB $1, R11 - MOVB R11, 0(DI) - ADDQ $2, DI - JMP inlineEmitCopyEnd - -inlineEmitCopyStep3: - // Emit the remaining copy, encoded as 3 bytes. - SUBL $1, AX - SHLB $2, AX - ORB $2, AX - MOVB AX, 0(DI) - MOVW R11, 1(DI) - ADDQ $3, DI - -inlineEmitCopyEnd: - // End inline of the emitCopy call. - // ---------------------------------------- - - // nextEmit = s - MOVQ SI, R10 - - // if s >= sLimit { goto emitRemainder } - MOVQ SI, AX - SUBQ DX, AX - CMPQ AX, R9 - JAE emitRemainder - - // As per the encode_other.go code: - // - // We could immediately etc. - - // x := load64(src, s-1) - MOVQ -1(SI), R14 - - // prevHash := hash(uint32(x>>0), shift) - MOVL R14, R11 - IMULL $0x1e35a7bd, R11 - SHRL CX, R11 - - // table[prevHash] = uint16(s-1) - MOVQ SI, AX - SUBQ DX, AX - SUBQ $1, AX - - // XXX: MOVW AX, table-32768(SP)(R11*2) - // XXX: 66 42 89 44 5c 78 mov %ax,0x78(%rsp,%r11,2) - BYTE $0x66 - BYTE $0x42 - BYTE $0x89 - BYTE $0x44 - BYTE $0x5c - BYTE $0x78 - - // currHash := hash(uint32(x>>8), shift) - SHRQ $8, R14 - MOVL R14, R11 - IMULL $0x1e35a7bd, R11 - SHRL CX, R11 - - // candidate = int(table[currHash]) - // XXX: MOVWQZX table-32768(SP)(R11*2), R15 - // XXX: 4e 0f b7 7c 5c 78 movzwq 0x78(%rsp,%r11,2),%r15 - BYTE $0x4e - BYTE $0x0f - BYTE $0xb7 - BYTE $0x7c - BYTE $0x5c - BYTE $0x78 - - // table[currHash] = uint16(s) - ADDQ $1, AX - - // XXX: MOVW AX, table-32768(SP)(R11*2) - // XXX: 66 42 89 44 5c 78 mov %ax,0x78(%rsp,%r11,2) - BYTE $0x66 - BYTE $0x42 - BYTE $0x89 - BYTE $0x44 - BYTE $0x5c - BYTE $0x78 - - // if uint32(x>>8) == load32(src, candidate) { continue } - MOVL (DX)(R15*1), BX - CMPL R14, BX - JEQ inner1 - - // nextHash = hash(uint32(x>>16), shift) - SHRQ $8, R14 - MOVL R14, R11 - IMULL $0x1e35a7bd, R11 - SHRL CX, R11 - - // s++ - ADDQ $1, SI - - // break out of the inner1 for loop, i.e. continue the outer loop. - JMP outer - -emitRemainder: - // if nextEmit < len(src) { etc } - MOVQ src_len+32(FP), AX - ADDQ DX, AX - CMPQ R10, AX - JEQ encodeBlockEnd - - // d += emitLiteral(dst[d:], src[nextEmit:]) - // - // Push args. - MOVQ DI, 0(SP) - MOVQ $0, 8(SP) // Unnecessary, as the callee ignores it, but conservative. - MOVQ $0, 16(SP) // Unnecessary, as the callee ignores it, but conservative. - MOVQ R10, 24(SP) - SUBQ R10, AX - MOVQ AX, 32(SP) - MOVQ AX, 40(SP) // Unnecessary, as the callee ignores it, but conservative. - - // Spill local variables (registers) onto the stack; call; unspill. - MOVQ DI, 80(SP) - CALL ·emitLiteral(SB) - MOVQ 80(SP), DI - - // Finish the "d +=" part of "d += emitLiteral(etc)". - ADDQ 48(SP), DI - -encodeBlockEnd: - MOVQ dst_base+0(FP), AX - SUBQ AX, DI - MOVQ DI, d+48(FP) - RET diff --git a/vendor/github.com/golang/snappy/encode_other.go b/vendor/github.com/golang/snappy/encode_other.go deleted file mode 100644 index dbcae90..0000000 --- a/vendor/github.com/golang/snappy/encode_other.go +++ /dev/null @@ -1,238 +0,0 @@ -// Copyright 2016 The Snappy-Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// +build !amd64 appengine !gc noasm - -package snappy - -func load32(b []byte, i int) uint32 { - b = b[i : i+4 : len(b)] // Help the compiler eliminate bounds checks on the next line. - return uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24 -} - -func load64(b []byte, i int) uint64 { - b = b[i : i+8 : len(b)] // Help the compiler eliminate bounds checks on the next line. - return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | - uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56 -} - -// emitLiteral writes a literal chunk and returns the number of bytes written. -// -// It assumes that: -// dst is long enough to hold the encoded bytes -// 1 <= len(lit) && len(lit) <= 65536 -func emitLiteral(dst, lit []byte) int { - i, n := 0, uint(len(lit)-1) - switch { - case n < 60: - dst[0] = uint8(n)<<2 | tagLiteral - i = 1 - case n < 1<<8: - dst[0] = 60<<2 | tagLiteral - dst[1] = uint8(n) - i = 2 - default: - dst[0] = 61<<2 | tagLiteral - dst[1] = uint8(n) - dst[2] = uint8(n >> 8) - i = 3 - } - return i + copy(dst[i:], lit) -} - -// emitCopy writes a copy chunk and returns the number of bytes written. -// -// It assumes that: -// dst is long enough to hold the encoded bytes -// 1 <= offset && offset <= 65535 -// 4 <= length && length <= 65535 -func emitCopy(dst []byte, offset, length int) int { - i := 0 - // The maximum length for a single tagCopy1 or tagCopy2 op is 64 bytes. The - // threshold for this loop is a little higher (at 68 = 64 + 4), and the - // length emitted down below is is a little lower (at 60 = 64 - 4), because - // it's shorter to encode a length 67 copy as a length 60 tagCopy2 followed - // by a length 7 tagCopy1 (which encodes as 3+2 bytes) than to encode it as - // a length 64 tagCopy2 followed by a length 3 tagCopy2 (which encodes as - // 3+3 bytes). The magic 4 in the 64±4 is because the minimum length for a - // tagCopy1 op is 4 bytes, which is why a length 3 copy has to be an - // encodes-as-3-bytes tagCopy2 instead of an encodes-as-2-bytes tagCopy1. - for length >= 68 { - // Emit a length 64 copy, encoded as 3 bytes. - dst[i+0] = 63<<2 | tagCopy2 - dst[i+1] = uint8(offset) - dst[i+2] = uint8(offset >> 8) - i += 3 - length -= 64 - } - if length > 64 { - // Emit a length 60 copy, encoded as 3 bytes. - dst[i+0] = 59<<2 | tagCopy2 - dst[i+1] = uint8(offset) - dst[i+2] = uint8(offset >> 8) - i += 3 - length -= 60 - } - if length >= 12 || offset >= 2048 { - // Emit the remaining copy, encoded as 3 bytes. - dst[i+0] = uint8(length-1)<<2 | tagCopy2 - dst[i+1] = uint8(offset) - dst[i+2] = uint8(offset >> 8) - return i + 3 - } - // Emit the remaining copy, encoded as 2 bytes. - dst[i+0] = uint8(offset>>8)<<5 | uint8(length-4)<<2 | tagCopy1 - dst[i+1] = uint8(offset) - return i + 2 -} - -// extendMatch returns the largest k such that k <= len(src) and that -// src[i:i+k-j] and src[j:k] have the same contents. -// -// It assumes that: -// 0 <= i && i < j && j <= len(src) -func extendMatch(src []byte, i, j int) int { - for ; j < len(src) && src[i] == src[j]; i, j = i+1, j+1 { - } - return j -} - -func hash(u, shift uint32) uint32 { - return (u * 0x1e35a7bd) >> shift -} - -// encodeBlock encodes a non-empty src to a guaranteed-large-enough dst. It -// assumes that the varint-encoded length of the decompressed bytes has already -// been written. -// -// It also assumes that: -// len(dst) >= MaxEncodedLen(len(src)) && -// minNonLiteralBlockSize <= len(src) && len(src) <= maxBlockSize -func encodeBlock(dst, src []byte) (d int) { - // Initialize the hash table. Its size ranges from 1<<8 to 1<<14 inclusive. - // The table element type is uint16, as s < sLimit and sLimit < len(src) - // and len(src) <= maxBlockSize and maxBlockSize == 65536. - const ( - maxTableSize = 1 << 14 - // tableMask is redundant, but helps the compiler eliminate bounds - // checks. - tableMask = maxTableSize - 1 - ) - shift := uint32(32 - 8) - for tableSize := 1 << 8; tableSize < maxTableSize && tableSize < len(src); tableSize *= 2 { - shift-- - } - // In Go, all array elements are zero-initialized, so there is no advantage - // to a smaller tableSize per se. However, it matches the C++ algorithm, - // and in the asm versions of this code, we can get away with zeroing only - // the first tableSize elements. - var table [maxTableSize]uint16 - - // sLimit is when to stop looking for offset/length copies. The inputMargin - // lets us use a fast path for emitLiteral in the main loop, while we are - // looking for copies. - sLimit := len(src) - inputMargin - - // nextEmit is where in src the next emitLiteral should start from. - nextEmit := 0 - - // The encoded form must start with a literal, as there are no previous - // bytes to copy, so we start looking for hash matches at s == 1. - s := 1 - nextHash := hash(load32(src, s), shift) - - for { - // Copied from the C++ snappy implementation: - // - // Heuristic match skipping: If 32 bytes are scanned with no matches - // found, start looking only at every other byte. If 32 more bytes are - // scanned (or skipped), look at every third byte, etc.. When a match - // is found, immediately go back to looking at every byte. This is a - // small loss (~5% performance, ~0.1% density) for compressible data - // due to more bookkeeping, but for non-compressible data (such as - // JPEG) it's a huge win since the compressor quickly "realizes" the - // data is incompressible and doesn't bother looking for matches - // everywhere. - // - // The "skip" variable keeps track of how many bytes there are since - // the last match; dividing it by 32 (ie. right-shifting by five) gives - // the number of bytes to move ahead for each iteration. - skip := 32 - - nextS := s - candidate := 0 - for { - s = nextS - bytesBetweenHashLookups := skip >> 5 - nextS = s + bytesBetweenHashLookups - skip += bytesBetweenHashLookups - if nextS > sLimit { - goto emitRemainder - } - candidate = int(table[nextHash&tableMask]) - table[nextHash&tableMask] = uint16(s) - nextHash = hash(load32(src, nextS), shift) - if load32(src, s) == load32(src, candidate) { - break - } - } - - // A 4-byte match has been found. We'll later see if more than 4 bytes - // match. But, prior to the match, src[nextEmit:s] are unmatched. Emit - // them as literal bytes. - d += emitLiteral(dst[d:], src[nextEmit:s]) - - // Call emitCopy, and then see if another emitCopy could be our next - // move. Repeat until we find no match for the input immediately after - // what was consumed by the last emitCopy call. - // - // If we exit this loop normally then we need to call emitLiteral next, - // though we don't yet know how big the literal will be. We handle that - // by proceeding to the next iteration of the main loop. We also can - // exit this loop via goto if we get close to exhausting the input. - for { - // Invariant: we have a 4-byte match at s, and no need to emit any - // literal bytes prior to s. - base := s - - // Extend the 4-byte match as long as possible. - // - // This is an inlined version of: - // s = extendMatch(src, candidate+4, s+4) - s += 4 - for i := candidate + 4; s < len(src) && src[i] == src[s]; i, s = i+1, s+1 { - } - - d += emitCopy(dst[d:], base-candidate, s-base) - nextEmit = s - if s >= sLimit { - goto emitRemainder - } - - // We could immediately start working at s now, but to improve - // compression we first update the hash table at s-1 and at s. If - // another emitCopy is not our next move, also calculate nextHash - // at s+1. At least on GOARCH=amd64, these three hash calculations - // are faster as one load64 call (with some shifts) instead of - // three load32 calls. - x := load64(src, s-1) - prevHash := hash(uint32(x>>0), shift) - table[prevHash&tableMask] = uint16(s - 1) - currHash := hash(uint32(x>>8), shift) - candidate = int(table[currHash&tableMask]) - table[currHash&tableMask] = uint16(s) - if uint32(x>>8) != load32(src, candidate) { - nextHash = hash(uint32(x>>16), shift) - s++ - break - } - } - } - -emitRemainder: - if nextEmit < len(src) { - d += emitLiteral(dst[d:], src[nextEmit:]) - } - return d -} diff --git a/vendor/github.com/golang/snappy/golden_test.go b/vendor/github.com/golang/snappy/golden_test.go deleted file mode 100644 index e4496f9..0000000 --- a/vendor/github.com/golang/snappy/golden_test.go +++ /dev/null @@ -1,1965 +0,0 @@ -// Copyright 2016 The Snappy-Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package snappy - -// extendMatchGoldenTestCases is the i and j arguments, and the returned value, -// for every extendMatch call issued when encoding the -// testdata/Mark.Twain-Tom.Sawyer.txt file. It is used to benchmark the -// extendMatch implementation. -// -// It was generated manually by adding some print statements to the (pure Go) -// extendMatch implementation: -// -// func extendMatch(src []byte, i, j int) int { -// i0, j0 := i, j -// for ; j < len(src) && src[i] == src[j]; i, j = i+1, j+1 { -// } -// println("{", i0, ",", j0, ",", j, "},") -// return j -// } -// -// and running "go test -test.run=EncodeGoldenInput -tags=noasm". -var extendMatchGoldenTestCases = []struct { - i, j, want int -}{ - {11, 61, 62}, - {80, 81, 82}, - {86, 87, 101}, - {85, 133, 149}, - {152, 153, 162}, - {133, 168, 193}, - {168, 207, 225}, - {81, 255, 275}, - {278, 279, 283}, - {306, 417, 417}, - {373, 428, 430}, - {389, 444, 447}, - {474, 510, 512}, - {465, 533, 533}, - {47, 547, 547}, - {307, 551, 554}, - {420, 582, 587}, - {309, 604, 604}, - {604, 625, 625}, - {538, 629, 629}, - {328, 640, 640}, - {573, 645, 645}, - {319, 657, 657}, - {30, 664, 664}, - {45, 679, 680}, - {621, 684, 684}, - {376, 700, 700}, - {33, 707, 708}, - {601, 733, 733}, - {334, 744, 745}, - {625, 758, 759}, - {382, 763, 763}, - {550, 769, 771}, - {533, 789, 789}, - {804, 813, 813}, - {342, 841, 842}, - {742, 847, 847}, - {74, 852, 852}, - {810, 864, 864}, - {758, 868, 869}, - {714, 883, 883}, - {582, 889, 891}, - {61, 934, 935}, - {894, 942, 942}, - {939, 949, 949}, - {785, 956, 957}, - {886, 978, 978}, - {792, 998, 998}, - {998, 1005, 1005}, - {572, 1032, 1032}, - {698, 1051, 1053}, - {599, 1067, 1069}, - {1056, 1079, 1079}, - {942, 1089, 1090}, - {831, 1094, 1096}, - {1088, 1100, 1103}, - {732, 1113, 1114}, - {1037, 1118, 1118}, - {872, 1128, 1130}, - {1079, 1140, 1142}, - {332, 1162, 1162}, - {207, 1168, 1186}, - {1189, 1190, 1225}, - {105, 1229, 1230}, - {79, 1256, 1257}, - {1190, 1261, 1283}, - {255, 1306, 1306}, - {1319, 1339, 1358}, - {364, 1370, 1370}, - {955, 1378, 1380}, - {122, 1403, 1403}, - {1325, 1407, 1419}, - {664, 1423, 1424}, - {941, 1461, 1463}, - {867, 1477, 1478}, - {757, 1488, 1489}, - {1140, 1499, 1499}, - {31, 1506, 1506}, - {1487, 1510, 1512}, - {1089, 1520, 1521}, - {1467, 1525, 1529}, - {1394, 1537, 1537}, - {1499, 1541, 1541}, - {367, 1558, 1558}, - {1475, 1564, 1564}, - {1525, 1568, 1571}, - {1541, 1582, 1583}, - {864, 1587, 1588}, - {704, 1597, 1597}, - {336, 1602, 1602}, - {1383, 1613, 1613}, - {1498, 1617, 1618}, - {1051, 1623, 1625}, - {401, 1643, 1645}, - {1072, 1654, 1655}, - {1067, 1667, 1669}, - {699, 1673, 1674}, - {1587, 1683, 1684}, - {920, 1696, 1696}, - {1505, 1710, 1710}, - {1550, 1723, 1723}, - {996, 1727, 1727}, - {833, 1733, 1734}, - {1638, 1739, 1740}, - {1654, 1744, 1744}, - {753, 1761, 1761}, - {1548, 1773, 1773}, - {1568, 1777, 1780}, - {1683, 1793, 1794}, - {948, 1801, 1801}, - {1666, 1805, 1808}, - {1502, 1814, 1814}, - {1696, 1822, 1822}, - {502, 1836, 1837}, - {917, 1843, 1843}, - {1733, 1854, 1855}, - {970, 1859, 1859}, - {310, 1863, 1863}, - {657, 1872, 1872}, - {1005, 1876, 1876}, - {1662, 1880, 1880}, - {904, 1892, 1892}, - {1427, 1910, 1910}, - {1772, 1929, 1930}, - {1822, 1937, 1940}, - {1858, 1949, 1950}, - {1602, 1956, 1956}, - {1150, 1962, 1962}, - {1504, 1966, 1967}, - {51, 1971, 1971}, - {1605, 1979, 1979}, - {1458, 1983, 1988}, - {1536, 2001, 2006}, - {1373, 2014, 2018}, - {1494, 2025, 2025}, - {1667, 2029, 2031}, - {1592, 2035, 2035}, - {330, 2045, 2045}, - {1376, 2053, 2053}, - {1991, 2058, 2059}, - {1635, 2065, 2065}, - {1992, 2073, 2074}, - {2014, 2080, 2081}, - {1546, 2085, 2087}, - {59, 2099, 2099}, - {1996, 2106, 2106}, - {1836, 2110, 2110}, - {2068, 2114, 2114}, - {1338, 2122, 2122}, - {1562, 2128, 2130}, - {1934, 2134, 2134}, - {2114, 2141, 2142}, - {977, 2149, 2150}, - {956, 2154, 2155}, - {1407, 2162, 2162}, - {1773, 2166, 2166}, - {883, 2171, 2171}, - {623, 2175, 2178}, - {1520, 2191, 2192}, - {1162, 2200, 2200}, - {912, 2204, 2204}, - {733, 2208, 2208}, - {1777, 2212, 2215}, - {1532, 2219, 2219}, - {718, 2223, 2225}, - {2069, 2229, 2229}, - {2207, 2245, 2246}, - {1139, 2264, 2264}, - {677, 2274, 2274}, - {2099, 2279, 2279}, - {1863, 2283, 2283}, - {1966, 2305, 2306}, - {2279, 2313, 2313}, - {1628, 2319, 2319}, - {755, 2329, 2329}, - {1461, 2334, 2334}, - {2117, 2340, 2340}, - {2313, 2349, 2349}, - {1859, 2353, 2353}, - {1048, 2362, 2362}, - {895, 2366, 2366}, - {2278, 2373, 2373}, - {1884, 2377, 2377}, - {1402, 2387, 2392}, - {700, 2398, 2398}, - {1971, 2402, 2402}, - {2009, 2419, 2419}, - {1441, 2426, 2428}, - {2208, 2432, 2432}, - {2038, 2436, 2436}, - {932, 2443, 2443}, - {1759, 2447, 2448}, - {744, 2452, 2452}, - {1875, 2458, 2458}, - {2405, 2468, 2468}, - {1596, 2472, 2473}, - {1953, 2480, 2482}, - {736, 2487, 2487}, - {1913, 2493, 2493}, - {774, 2497, 2497}, - {1484, 2506, 2508}, - {2432, 2512, 2512}, - {752, 2519, 2519}, - {2497, 2523, 2523}, - {2409, 2528, 2529}, - {2122, 2533, 2533}, - {2396, 2537, 2538}, - {2410, 2547, 2548}, - {1093, 2555, 2560}, - {551, 2564, 2565}, - {2268, 2569, 2569}, - {1362, 2580, 2580}, - {1916, 2584, 2585}, - {994, 2589, 2590}, - {1979, 2596, 2596}, - {1041, 2602, 2602}, - {2104, 2614, 2616}, - {2609, 2621, 2628}, - {2329, 2638, 2638}, - {2211, 2657, 2658}, - {2638, 2662, 2667}, - {2578, 2676, 2679}, - {2153, 2685, 2686}, - {2608, 2696, 2697}, - {598, 2712, 2712}, - {2620, 2719, 2720}, - {1888, 2724, 2728}, - {2709, 2732, 2732}, - {1365, 2739, 2739}, - {784, 2747, 2748}, - {424, 2753, 2753}, - {2204, 2759, 2759}, - {812, 2768, 2769}, - {2455, 2773, 2773}, - {1722, 2781, 2781}, - {1917, 2792, 2792}, - {2705, 2799, 2799}, - {2685, 2806, 2807}, - {2742, 2811, 2811}, - {1370, 2818, 2818}, - {2641, 2830, 2830}, - {2512, 2837, 2837}, - {2457, 2841, 2841}, - {2756, 2845, 2845}, - {2719, 2855, 2855}, - {1423, 2859, 2859}, - {2849, 2863, 2865}, - {1474, 2871, 2871}, - {1161, 2875, 2876}, - {2282, 2880, 2881}, - {2746, 2888, 2888}, - {1783, 2893, 2893}, - {2401, 2899, 2900}, - {2632, 2920, 2923}, - {2422, 2928, 2930}, - {2715, 2939, 2939}, - {2162, 2943, 2943}, - {2859, 2947, 2947}, - {1910, 2951, 2951}, - {1431, 2955, 2956}, - {1439, 2964, 2964}, - {2501, 2968, 2969}, - {2029, 2973, 2976}, - {689, 2983, 2984}, - {1658, 2988, 2988}, - {1031, 2996, 2996}, - {2149, 3001, 3002}, - {25, 3009, 3013}, - {2964, 3023, 3023}, - {953, 3027, 3028}, - {2359, 3036, 3036}, - {3023, 3049, 3049}, - {2880, 3055, 3056}, - {2973, 3076, 3077}, - {2874, 3090, 3090}, - {2871, 3094, 3094}, - {2532, 3100, 3100}, - {2938, 3107, 3108}, - {350, 3115, 3115}, - {2196, 3119, 3121}, - {1133, 3127, 3129}, - {1797, 3134, 3150}, - {3032, 3158, 3158}, - {3016, 3172, 3172}, - {2533, 3179, 3179}, - {3055, 3187, 3188}, - {1384, 3192, 3193}, - {2799, 3199, 3199}, - {2126, 3203, 3207}, - {2334, 3215, 3215}, - {2105, 3220, 3221}, - {3199, 3229, 3229}, - {2891, 3233, 3233}, - {855, 3240, 3240}, - {1852, 3253, 3256}, - {2140, 3263, 3263}, - {1682, 3268, 3270}, - {3243, 3274, 3274}, - {924, 3279, 3279}, - {2212, 3283, 3283}, - {2596, 3287, 3287}, - {2999, 3291, 3291}, - {2353, 3295, 3295}, - {2480, 3302, 3304}, - {1959, 3308, 3311}, - {3000, 3318, 3318}, - {845, 3330, 3330}, - {2283, 3334, 3334}, - {2519, 3342, 3342}, - {3325, 3346, 3348}, - {2397, 3353, 3354}, - {2763, 3358, 3358}, - {3198, 3363, 3364}, - {3211, 3368, 3372}, - {2950, 3376, 3377}, - {3245, 3388, 3391}, - {2264, 3398, 3398}, - {795, 3403, 3403}, - {3287, 3407, 3407}, - {3358, 3411, 3411}, - {3317, 3415, 3415}, - {3232, 3431, 3431}, - {2128, 3435, 3437}, - {3236, 3441, 3441}, - {3398, 3445, 3446}, - {2814, 3450, 3450}, - {3394, 3466, 3466}, - {2425, 3470, 3470}, - {3330, 3476, 3476}, - {1612, 3480, 3480}, - {1004, 3485, 3486}, - {2732, 3490, 3490}, - {1117, 3494, 3495}, - {629, 3501, 3501}, - {3087, 3514, 3514}, - {684, 3518, 3518}, - {3489, 3522, 3524}, - {1760, 3529, 3529}, - {617, 3537, 3537}, - {3431, 3541, 3541}, - {997, 3547, 3547}, - {882, 3552, 3553}, - {2419, 3558, 3558}, - {610, 3562, 3563}, - {1903, 3567, 3569}, - {3005, 3575, 3575}, - {3076, 3585, 3586}, - {3541, 3590, 3590}, - {3490, 3594, 3594}, - {1899, 3599, 3599}, - {3545, 3606, 3606}, - {3290, 3614, 3615}, - {2056, 3619, 3620}, - {3556, 3625, 3625}, - {3294, 3632, 3633}, - {637, 3643, 3644}, - {3609, 3648, 3650}, - {3175, 3658, 3658}, - {3498, 3665, 3665}, - {1597, 3669, 3669}, - {1983, 3673, 3673}, - {3215, 3682, 3682}, - {3544, 3689, 3689}, - {3694, 3698, 3698}, - {3228, 3715, 3716}, - {2594, 3720, 3722}, - {3573, 3726, 3726}, - {2479, 3732, 3735}, - {3191, 3741, 3742}, - {1113, 3746, 3747}, - {2844, 3751, 3751}, - {3445, 3756, 3757}, - {3755, 3766, 3766}, - {3421, 3775, 3780}, - {3593, 3784, 3786}, - {3263, 3796, 3796}, - {3469, 3806, 3806}, - {2602, 3815, 3815}, - {723, 3819, 3821}, - {1608, 3826, 3826}, - {3334, 3830, 3830}, - {2198, 3835, 3835}, - {2635, 3840, 3840}, - {3702, 3852, 3853}, - {3406, 3858, 3859}, - {3681, 3867, 3870}, - {3407, 3880, 3880}, - {340, 3889, 3889}, - {3772, 3893, 3893}, - {593, 3897, 3897}, - {2563, 3914, 3916}, - {2981, 3929, 3929}, - {1835, 3933, 3934}, - {3906, 3951, 3951}, - {1459, 3958, 3958}, - {3889, 3974, 3974}, - {2188, 3982, 3982}, - {3220, 3986, 3987}, - {3585, 3991, 3993}, - {3712, 3997, 4001}, - {2805, 4007, 4007}, - {1879, 4012, 4013}, - {3618, 4018, 4018}, - {1145, 4031, 4032}, - {3901, 4037, 4037}, - {2772, 4046, 4047}, - {2802, 4053, 4054}, - {3299, 4058, 4058}, - {3725, 4066, 4066}, - {2271, 4070, 4070}, - {385, 4075, 4076}, - {3624, 4089, 4090}, - {3745, 4096, 4098}, - {1563, 4102, 4102}, - {4045, 4106, 4111}, - {3696, 4115, 4119}, - {3376, 4125, 4126}, - {1880, 4130, 4130}, - {2048, 4140, 4141}, - {2724, 4149, 4149}, - {1767, 4156, 4156}, - {2601, 4164, 4164}, - {2757, 4168, 4168}, - {3974, 4172, 4172}, - {3914, 4178, 4178}, - {516, 4185, 4185}, - {1032, 4189, 4190}, - {3462, 4197, 4198}, - {3805, 4202, 4203}, - {3910, 4207, 4212}, - {3075, 4221, 4221}, - {3756, 4225, 4226}, - {1872, 4236, 4237}, - {3844, 4241, 4241}, - {3991, 4245, 4249}, - {2203, 4258, 4258}, - {3903, 4267, 4268}, - {705, 4272, 4272}, - {1896, 4276, 4276}, - {1955, 4285, 4288}, - {3746, 4302, 4303}, - {2672, 4311, 4311}, - {3969, 4317, 4317}, - {3883, 4322, 4322}, - {1920, 4339, 4340}, - {3527, 4344, 4346}, - {1160, 4358, 4358}, - {3648, 4364, 4366}, - {2711, 4387, 4387}, - {3619, 4391, 4392}, - {1944, 4396, 4396}, - {4369, 4400, 4400}, - {2736, 4404, 4407}, - {2546, 4411, 4412}, - {4390, 4422, 4422}, - {3610, 4426, 4427}, - {4058, 4431, 4431}, - {4374, 4435, 4435}, - {3463, 4445, 4446}, - {1813, 4452, 4452}, - {3669, 4456, 4456}, - {3830, 4460, 4460}, - {421, 4464, 4465}, - {1719, 4471, 4471}, - {3880, 4475, 4475}, - {1834, 4485, 4487}, - {3590, 4491, 4491}, - {442, 4496, 4497}, - {4435, 4501, 4501}, - {3814, 4509, 4509}, - {987, 4513, 4513}, - {4494, 4518, 4521}, - {3218, 4526, 4529}, - {4221, 4537, 4537}, - {2778, 4543, 4545}, - {4422, 4552, 4552}, - {4031, 4558, 4559}, - {4178, 4563, 4563}, - {3726, 4567, 4574}, - {4027, 4578, 4578}, - {4339, 4585, 4587}, - {3796, 4592, 4595}, - {543, 4600, 4613}, - {2855, 4620, 4621}, - {2795, 4627, 4627}, - {3440, 4631, 4632}, - {4279, 4636, 4639}, - {4245, 4643, 4645}, - {4516, 4649, 4650}, - {3133, 4654, 4654}, - {4042, 4658, 4659}, - {3422, 4663, 4663}, - {4046, 4667, 4668}, - {4267, 4672, 4672}, - {4004, 4676, 4677}, - {2490, 4682, 4682}, - {2451, 4697, 4697}, - {3027, 4705, 4705}, - {4028, 4717, 4717}, - {4460, 4721, 4721}, - {2471, 4725, 4727}, - {3090, 4735, 4735}, - {3192, 4739, 4740}, - {3835, 4760, 4760}, - {4540, 4764, 4764}, - {4007, 4772, 4774}, - {619, 4784, 4784}, - {3561, 4789, 4791}, - {3367, 4805, 4805}, - {4490, 4810, 4811}, - {2402, 4815, 4815}, - {3352, 4819, 4822}, - {2773, 4828, 4828}, - {4552, 4832, 4832}, - {2522, 4840, 4841}, - {316, 4847, 4852}, - {4715, 4858, 4858}, - {2959, 4862, 4862}, - {4858, 4868, 4869}, - {2134, 4873, 4873}, - {578, 4878, 4878}, - {4189, 4889, 4890}, - {2229, 4894, 4894}, - {4501, 4898, 4898}, - {2297, 4903, 4903}, - {2933, 4909, 4909}, - {3008, 4913, 4913}, - {3153, 4917, 4917}, - {4819, 4921, 4921}, - {4921, 4932, 4933}, - {4920, 4944, 4945}, - {4814, 4954, 4955}, - {576, 4966, 4966}, - {1854, 4970, 4971}, - {1374, 4975, 4976}, - {3307, 4980, 4980}, - {974, 4984, 4988}, - {4721, 4992, 4992}, - {4898, 4996, 4996}, - {4475, 5006, 5006}, - {3819, 5012, 5012}, - {1948, 5019, 5021}, - {4954, 5027, 5029}, - {3740, 5038, 5040}, - {4763, 5044, 5045}, - {1936, 5051, 5051}, - {4844, 5055, 5060}, - {4215, 5069, 5072}, - {1146, 5076, 5076}, - {3845, 5082, 5082}, - {4865, 5090, 5090}, - {4624, 5094, 5094}, - {4815, 5098, 5098}, - {5006, 5105, 5105}, - {4980, 5109, 5109}, - {4795, 5113, 5115}, - {5043, 5119, 5121}, - {4782, 5129, 5129}, - {3826, 5139, 5139}, - {3876, 5156, 5156}, - {3111, 5167, 5171}, - {1470, 5177, 5177}, - {4431, 5181, 5181}, - {546, 5189, 5189}, - {4225, 5193, 5193}, - {1672, 5199, 5201}, - {4207, 5205, 5209}, - {4220, 5216, 5217}, - {4658, 5224, 5225}, - {3295, 5235, 5235}, - {2436, 5239, 5239}, - {2349, 5246, 5246}, - {2175, 5250, 5250}, - {5180, 5257, 5258}, - {3161, 5263, 5263}, - {5105, 5272, 5272}, - {3552, 5282, 5282}, - {4944, 5299, 5300}, - {4130, 5312, 5313}, - {902, 5323, 5323}, - {913, 5327, 5327}, - {2987, 5333, 5334}, - {5150, 5344, 5344}, - {5249, 5348, 5348}, - {1965, 5358, 5359}, - {5330, 5364, 5364}, - {2012, 5373, 5377}, - {712, 5384, 5386}, - {5235, 5390, 5390}, - {5044, 5398, 5399}, - {564, 5406, 5406}, - {39, 5410, 5410}, - {4642, 5422, 5425}, - {4421, 5437, 5438}, - {2347, 5449, 5449}, - {5333, 5453, 5454}, - {4136, 5458, 5459}, - {3793, 5468, 5468}, - {2243, 5480, 5480}, - {4889, 5492, 5493}, - {4295, 5504, 5504}, - {2785, 5511, 5511}, - {2377, 5518, 5518}, - {3662, 5525, 5525}, - {5097, 5529, 5530}, - {4781, 5537, 5538}, - {4697, 5547, 5548}, - {436, 5552, 5553}, - {5542, 5558, 5558}, - {3692, 5562, 5562}, - {2696, 5568, 5569}, - {4620, 5578, 5578}, - {2898, 5590, 5590}, - {5557, 5596, 5618}, - {2797, 5623, 5625}, - {2792, 5629, 5629}, - {5243, 5633, 5633}, - {5348, 5637, 5637}, - {5547, 5643, 5643}, - {4296, 5654, 5655}, - {5568, 5662, 5662}, - {3001, 5670, 5671}, - {3794, 5679, 5679}, - {4006, 5685, 5686}, - {4969, 5690, 5692}, - {687, 5704, 5704}, - {4563, 5708, 5708}, - {1723, 5738, 5738}, - {649, 5742, 5742}, - {5163, 5748, 5755}, - {3907, 5759, 5759}, - {3074, 5764, 5764}, - {5326, 5771, 5771}, - {2951, 5776, 5776}, - {5181, 5780, 5780}, - {2614, 5785, 5788}, - {4709, 5794, 5794}, - {2784, 5799, 5799}, - {5518, 5803, 5803}, - {4155, 5812, 5815}, - {921, 5819, 5819}, - {5224, 5823, 5824}, - {2853, 5830, 5836}, - {5776, 5840, 5840}, - {2955, 5844, 5845}, - {5745, 5853, 5853}, - {3291, 5857, 5857}, - {2988, 5861, 5861}, - {2647, 5865, 5865}, - {5398, 5869, 5870}, - {1085, 5874, 5875}, - {4906, 5881, 5881}, - {802, 5886, 5886}, - {5119, 5890, 5893}, - {5802, 5899, 5900}, - {3415, 5904, 5904}, - {5629, 5908, 5908}, - {3714, 5912, 5914}, - {5558, 5921, 5921}, - {2710, 5927, 5928}, - {1094, 5932, 5934}, - {2653, 5940, 5941}, - {4735, 5954, 5954}, - {5861, 5958, 5958}, - {1040, 5971, 5971}, - {5514, 5977, 5977}, - {5048, 5981, 5982}, - {5953, 5992, 5993}, - {3751, 5997, 5997}, - {4991, 6001, 6002}, - {5885, 6006, 6007}, - {5529, 6011, 6012}, - {4974, 6019, 6020}, - {5857, 6024, 6024}, - {3483, 6032, 6032}, - {3594, 6036, 6036}, - {1997, 6040, 6040}, - {5997, 6044, 6047}, - {5197, 6051, 6051}, - {1764, 6055, 6055}, - {6050, 6059, 6059}, - {5239, 6063, 6063}, - {5049, 6067, 6067}, - {5957, 6073, 6074}, - {1022, 6078, 6078}, - {3414, 6083, 6084}, - {3809, 6090, 6090}, - {4562, 6095, 6096}, - {5878, 6104, 6104}, - {594, 6108, 6109}, - {3353, 6115, 6116}, - {4992, 6120, 6121}, - {2424, 6125, 6125}, - {4484, 6130, 6130}, - {3900, 6134, 6135}, - {5793, 6139, 6141}, - {3562, 6145, 6145}, - {1438, 6152, 6153}, - {6058, 6157, 6158}, - {4411, 6162, 6163}, - {4590, 6167, 6171}, - {4748, 6175, 6175}, - {5517, 6183, 6184}, - {6095, 6191, 6192}, - {1471, 6203, 6203}, - {2643, 6209, 6210}, - {450, 6220, 6220}, - {5266, 6226, 6226}, - {2576, 6233, 6233}, - {2607, 6239, 6240}, - {5164, 6244, 6251}, - {6054, 6255, 6255}, - {1789, 6260, 6261}, - {5250, 6265, 6265}, - {6062, 6273, 6278}, - {5990, 6282, 6282}, - {3283, 6286, 6286}, - {5436, 6290, 6290}, - {6059, 6294, 6294}, - {5668, 6298, 6300}, - {3072, 6324, 6329}, - {3132, 6338, 6339}, - {3246, 6343, 6344}, - {28, 6348, 6349}, - {1503, 6353, 6355}, - {6067, 6359, 6359}, - {3384, 6364, 6364}, - {545, 6375, 6376}, - {5803, 6380, 6380}, - {5522, 6384, 6385}, - {5908, 6389, 6389}, - {2796, 6393, 6396}, - {4831, 6403, 6404}, - {6388, 6412, 6412}, - {6005, 6417, 6420}, - {4450, 6430, 6430}, - {4050, 6435, 6435}, - {5372, 6441, 6441}, - {4378, 6447, 6447}, - {6199, 6452, 6452}, - {3026, 6456, 6456}, - {2642, 6460, 6462}, - {6392, 6470, 6470}, - {6459, 6474, 6474}, - {2829, 6487, 6488}, - {2942, 6499, 6504}, - {5069, 6508, 6511}, - {5341, 6515, 6516}, - {5853, 6521, 6525}, - {6104, 6531, 6531}, - {5759, 6535, 6538}, - {4672, 6542, 6543}, - {2443, 6550, 6550}, - {5109, 6554, 6554}, - {6494, 6558, 6560}, - {6006, 6570, 6572}, - {6424, 6576, 6580}, - {4693, 6591, 6592}, - {6439, 6596, 6597}, - {3179, 6601, 6601}, - {5299, 6606, 6607}, - {4148, 6612, 6613}, - {3774, 6617, 6617}, - {3537, 6623, 6624}, - {4975, 6628, 6629}, - {3848, 6636, 6636}, - {856, 6640, 6640}, - {5724, 6645, 6645}, - {6632, 6651, 6651}, - {4630, 6656, 6658}, - {1440, 6662, 6662}, - {4281, 6666, 6667}, - {4302, 6671, 6672}, - {2589, 6676, 6677}, - {5647, 6681, 6687}, - {6082, 6691, 6693}, - {6144, 6698, 6698}, - {6103, 6709, 6710}, - {3710, 6714, 6714}, - {4253, 6718, 6721}, - {2467, 6730, 6730}, - {4778, 6734, 6734}, - {6528, 6738, 6738}, - {4358, 6747, 6747}, - {5889, 6753, 6753}, - {5193, 6757, 6757}, - {5797, 6761, 6761}, - {3858, 6765, 6766}, - {5951, 6776, 6776}, - {6487, 6781, 6782}, - {3282, 6786, 6787}, - {4667, 6797, 6799}, - {1927, 6803, 6806}, - {6583, 6810, 6810}, - {4937, 6814, 6814}, - {6099, 6824, 6824}, - {4415, 6835, 6836}, - {6332, 6840, 6841}, - {5160, 6850, 6850}, - {4764, 6854, 6854}, - {6814, 6858, 6859}, - {3018, 6864, 6864}, - {6293, 6868, 6869}, - {6359, 6877, 6877}, - {3047, 6884, 6886}, - {5262, 6890, 6891}, - {5471, 6900, 6900}, - {3268, 6910, 6912}, - {1047, 6916, 6916}, - {5904, 6923, 6923}, - {5798, 6933, 6938}, - {4149, 6942, 6942}, - {1821, 6946, 6946}, - {3599, 6952, 6952}, - {6470, 6957, 6957}, - {5562, 6961, 6961}, - {6268, 6965, 6967}, - {6389, 6971, 6971}, - {6596, 6975, 6976}, - {6553, 6980, 6981}, - {6576, 6985, 6989}, - {1375, 6993, 6993}, - {652, 6998, 6998}, - {4876, 7002, 7003}, - {5768, 7011, 7013}, - {3973, 7017, 7017}, - {6802, 7025, 7025}, - {6955, 7034, 7036}, - {6974, 7040, 7040}, - {5944, 7044, 7044}, - {6992, 7048, 7054}, - {6872, 7059, 7059}, - {2943, 7063, 7063}, - {6923, 7067, 7067}, - {5094, 7071, 7071}, - {4873, 7075, 7075}, - {5819, 7079, 7079}, - {5945, 7085, 7085}, - {1540, 7090, 7091}, - {2090, 7095, 7095}, - {5024, 7104, 7105}, - {6900, 7109, 7109}, - {6024, 7113, 7114}, - {6000, 7118, 7120}, - {2187, 7124, 7125}, - {6760, 7129, 7130}, - {5898, 7134, 7136}, - {7032, 7144, 7144}, - {4271, 7148, 7148}, - {3706, 7152, 7152}, - {6970, 7156, 7157}, - {7088, 7161, 7163}, - {2718, 7168, 7169}, - {5674, 7175, 7175}, - {4631, 7182, 7182}, - {7070, 7188, 7189}, - {6220, 7196, 7196}, - {3458, 7201, 7202}, - {2041, 7211, 7212}, - {1454, 7216, 7216}, - {5199, 7225, 7227}, - {3529, 7234, 7234}, - {6890, 7238, 7238}, - {3815, 7242, 7243}, - {5490, 7250, 7253}, - {6554, 7257, 7263}, - {5890, 7267, 7269}, - {6877, 7273, 7273}, - {4877, 7277, 7277}, - {2502, 7285, 7285}, - {1483, 7289, 7295}, - {7210, 7304, 7308}, - {6845, 7313, 7316}, - {7219, 7320, 7320}, - {7001, 7325, 7329}, - {6853, 7333, 7334}, - {6120, 7338, 7338}, - {6606, 7342, 7343}, - {7020, 7348, 7350}, - {3509, 7354, 7354}, - {7133, 7359, 7363}, - {3434, 7371, 7374}, - {2787, 7384, 7384}, - {7044, 7388, 7388}, - {6960, 7394, 7395}, - {6676, 7399, 7400}, - {7161, 7404, 7404}, - {7285, 7417, 7418}, - {4558, 7425, 7426}, - {4828, 7430, 7430}, - {6063, 7436, 7436}, - {3597, 7442, 7442}, - {914, 7446, 7446}, - {7320, 7452, 7454}, - {7267, 7458, 7460}, - {5076, 7464, 7464}, - {7430, 7468, 7469}, - {6273, 7473, 7474}, - {7440, 7478, 7487}, - {7348, 7491, 7494}, - {1021, 7510, 7510}, - {7473, 7515, 7515}, - {2823, 7519, 7519}, - {6264, 7527, 7527}, - {7302, 7531, 7531}, - {7089, 7535, 7535}, - {7342, 7540, 7541}, - {3688, 7547, 7551}, - {3054, 7558, 7560}, - {4177, 7566, 7567}, - {6691, 7574, 7575}, - {7156, 7585, 7586}, - {7147, 7590, 7592}, - {7407, 7598, 7598}, - {7403, 7602, 7603}, - {6868, 7607, 7607}, - {6636, 7611, 7611}, - {4805, 7617, 7617}, - {5779, 7623, 7623}, - {7063, 7627, 7627}, - {5079, 7632, 7632}, - {7377, 7637, 7637}, - {7337, 7641, 7642}, - {6738, 7655, 7655}, - {7338, 7659, 7659}, - {6541, 7669, 7671}, - {595, 7675, 7675}, - {7658, 7679, 7680}, - {7647, 7685, 7686}, - {2477, 7690, 7690}, - {5823, 7694, 7694}, - {4156, 7699, 7699}, - {5931, 7703, 7706}, - {6854, 7712, 7712}, - {4931, 7718, 7718}, - {6979, 7722, 7722}, - {5085, 7727, 7727}, - {6965, 7732, 7732}, - {7201, 7736, 7737}, - {3639, 7741, 7743}, - {7534, 7749, 7749}, - {4292, 7753, 7753}, - {3427, 7759, 7763}, - {7273, 7767, 7767}, - {940, 7778, 7778}, - {4838, 7782, 7785}, - {4216, 7790, 7792}, - {922, 7800, 7801}, - {7256, 7810, 7811}, - {7789, 7815, 7819}, - {7225, 7823, 7825}, - {7531, 7829, 7829}, - {6997, 7833, 7833}, - {7757, 7837, 7838}, - {4129, 7842, 7842}, - {7333, 7848, 7849}, - {6776, 7855, 7855}, - {7527, 7859, 7859}, - {4370, 7863, 7863}, - {4512, 7868, 7868}, - {5679, 7880, 7880}, - {3162, 7884, 7885}, - {3933, 7892, 7894}, - {7804, 7899, 7902}, - {6363, 7906, 7907}, - {7848, 7911, 7912}, - {5584, 7917, 7921}, - {874, 7926, 7926}, - {3342, 7930, 7930}, - {4507, 7935, 7937}, - {3672, 7943, 7944}, - {7911, 7948, 7949}, - {6402, 7956, 7956}, - {7940, 7960, 7960}, - {7113, 7964, 7964}, - {1073, 7968, 7968}, - {7740, 7974, 7974}, - {7601, 7978, 7982}, - {6797, 7987, 7988}, - {3528, 7994, 7995}, - {5483, 7999, 7999}, - {5717, 8011, 8011}, - {5480, 8017, 8017}, - {7770, 8023, 8030}, - {2452, 8034, 8034}, - {5282, 8047, 8047}, - {7967, 8051, 8051}, - {1128, 8058, 8066}, - {6348, 8070, 8070}, - {8055, 8077, 8077}, - {7925, 8081, 8086}, - {6810, 8090, 8090}, - {5051, 8101, 8101}, - {4696, 8109, 8110}, - {5129, 8119, 8119}, - {4449, 8123, 8123}, - {7222, 8127, 8127}, - {4649, 8131, 8134}, - {7994, 8138, 8138}, - {5954, 8148, 8148}, - {475, 8152, 8153}, - {7906, 8157, 8157}, - {7458, 8164, 8166}, - {7632, 8171, 8173}, - {3874, 8177, 8183}, - {4391, 8187, 8187}, - {561, 8191, 8191}, - {2417, 8195, 8195}, - {2357, 8204, 8204}, - {2269, 8216, 8218}, - {3968, 8222, 8222}, - {2200, 8226, 8227}, - {3453, 8247, 8247}, - {2439, 8251, 8252}, - {7175, 8257, 8257}, - {976, 8262, 8264}, - {4953, 8273, 8273}, - {4219, 8278, 8278}, - {6, 8285, 8291}, - {5703, 8295, 8296}, - {5272, 8300, 8300}, - {8037, 8304, 8304}, - {8186, 8314, 8314}, - {8304, 8318, 8318}, - {8051, 8326, 8326}, - {8318, 8330, 8330}, - {2671, 8334, 8335}, - {2662, 8339, 8339}, - {8081, 8349, 8350}, - {3328, 8356, 8356}, - {2879, 8360, 8362}, - {8050, 8370, 8371}, - {8330, 8375, 8376}, - {8375, 8386, 8386}, - {4961, 8390, 8390}, - {1017, 8403, 8405}, - {3533, 8416, 8416}, - {4555, 8422, 8422}, - {6445, 8426, 8426}, - {8169, 8432, 8432}, - {990, 8436, 8436}, - {4102, 8440, 8440}, - {7398, 8444, 8446}, - {3480, 8450, 8450}, - {6324, 8462, 8462}, - {7948, 8466, 8467}, - {5950, 8471, 8471}, - {5189, 8476, 8476}, - {4026, 8490, 8490}, - {8374, 8494, 8495}, - {4682, 8501, 8501}, - {7387, 8506, 8506}, - {8164, 8510, 8515}, - {4079, 8524, 8524}, - {8360, 8529, 8531}, - {7446, 8540, 8543}, - {7971, 8547, 8548}, - {4311, 8552, 8552}, - {5204, 8556, 8557}, - {7968, 8562, 8562}, - {7847, 8571, 8573}, - {8547, 8577, 8577}, - {5320, 8581, 8581}, - {8556, 8585, 8586}, - {8504, 8590, 8590}, - {7669, 8602, 8604}, - {5874, 8608, 8609}, - {5828, 8613, 8613}, - {7998, 8617, 8617}, - {8519, 8625, 8625}, - {7250, 8637, 8637}, - {426, 8641, 8641}, - {8436, 8645, 8645}, - {5986, 8649, 8656}, - {8157, 8660, 8660}, - {7182, 8665, 8665}, - {8421, 8675, 8675}, - {8509, 8681, 8681}, - {5137, 8688, 8689}, - {8625, 8694, 8695}, - {5228, 8701, 8702}, - {6661, 8714, 8714}, - {1010, 8719, 8719}, - {6648, 8723, 8723}, - {3500, 8728, 8728}, - {2442, 8735, 8735}, - {8494, 8740, 8741}, - {8171, 8753, 8755}, - {7242, 8763, 8764}, - {4739, 8768, 8769}, - {7079, 8773, 8773}, - {8386, 8777, 8777}, - {8624, 8781, 8787}, - {661, 8791, 8794}, - {8631, 8801, 8801}, - {7753, 8805, 8805}, - {4783, 8809, 8810}, - {1673, 8814, 8815}, - {6623, 8819, 8819}, - {4404, 8823, 8823}, - {8089, 8827, 8828}, - {8773, 8832, 8832}, - {5394, 8836, 8836}, - {6231, 8841, 8843}, - {1015, 8852, 8853}, - {6873, 8857, 8857}, - {6289, 8865, 8865}, - {8577, 8869, 8869}, - {8114, 8873, 8875}, - {8534, 8883, 8883}, - {3007, 8887, 8888}, - {8827, 8892, 8893}, - {4788, 8897, 8900}, - {5698, 8906, 8907}, - {7690, 8911, 8911}, - {6643, 8919, 8919}, - {7206, 8923, 8924}, - {7866, 8929, 8931}, - {8880, 8942, 8942}, - {8630, 8951, 8952}, - {6027, 8958, 8958}, - {7749, 8966, 8967}, - {4932, 8972, 8973}, - {8892, 8980, 8981}, - {634, 9003, 9003}, - {8109, 9007, 9008}, - {8777, 9012, 9012}, - {3981, 9016, 9017}, - {5723, 9025, 9025}, - {7662, 9034, 9038}, - {8955, 9042, 9042}, - {8070, 9060, 9062}, - {8910, 9066, 9066}, - {5363, 9070, 9071}, - {7699, 9075, 9076}, - {8991, 9081, 9081}, - {6850, 9085, 9085}, - {5811, 9092, 9094}, - {9079, 9098, 9102}, - {6456, 9106, 9106}, - {2259, 9111, 9111}, - {4752, 9116, 9116}, - {9060, 9120, 9123}, - {8090, 9127, 9127}, - {5305, 9131, 9132}, - {8623, 9137, 9137}, - {7417, 9141, 9141}, - {6564, 9148, 9149}, - {9126, 9157, 9158}, - {4285, 9169, 9170}, - {8698, 9174, 9174}, - {8869, 9178, 9178}, - {2572, 9182, 9183}, - {6482, 9188, 9190}, - {9181, 9201, 9201}, - {2968, 9208, 9209}, - {2506, 9213, 9215}, - {9127, 9219, 9219}, - {7910, 9225, 9227}, - {5422, 9235, 9239}, - {8813, 9244, 9246}, - {9178, 9250, 9250}, - {8748, 9255, 9255}, - {7354, 9265, 9265}, - {7767, 9269, 9269}, - {7710, 9281, 9283}, - {8826, 9288, 9290}, - {861, 9295, 9295}, - {4482, 9301, 9301}, - {9264, 9305, 9306}, - {8805, 9310, 9310}, - {4995, 9314, 9314}, - {6730, 9318, 9318}, - {7457, 9328, 9328}, - {2547, 9335, 9336}, - {6298, 9340, 9343}, - {9305, 9353, 9354}, - {9269, 9358, 9358}, - {6338, 9370, 9370}, - {7289, 9376, 9379}, - {5780, 9383, 9383}, - {7607, 9387, 9387}, - {2065, 9392, 9392}, - {7238, 9396, 9396}, - {8856, 9400, 9400}, - {8069, 9412, 9413}, - {611, 9420, 9420}, - {7071, 9424, 9424}, - {3089, 9430, 9431}, - {7117, 9435, 9438}, - {1976, 9445, 9445}, - {6640, 9449, 9449}, - {5488, 9453, 9453}, - {8739, 9457, 9459}, - {5958, 9466, 9466}, - {7985, 9470, 9470}, - {8735, 9475, 9475}, - {5009, 9479, 9479}, - {8073, 9483, 9484}, - {2328, 9490, 9491}, - {9250, 9495, 9495}, - {4043, 9502, 9502}, - {7712, 9506, 9506}, - {9012, 9510, 9510}, - {9028, 9514, 9515}, - {2190, 9521, 9524}, - {9029, 9528, 9528}, - {9519, 9532, 9532}, - {9495, 9536, 9536}, - {8527, 9540, 9540}, - {2137, 9550, 9550}, - {8419, 9557, 9557}, - {9383, 9561, 9562}, - {8970, 9575, 9578}, - {8911, 9582, 9582}, - {7828, 9595, 9596}, - {6180, 9600, 9600}, - {8738, 9604, 9607}, - {7540, 9611, 9612}, - {9599, 9616, 9618}, - {9187, 9623, 9623}, - {9294, 9628, 9629}, - {4536, 9639, 9639}, - {3867, 9643, 9643}, - {6305, 9648, 9648}, - {1617, 9654, 9657}, - {5762, 9666, 9666}, - {8314, 9670, 9670}, - {9666, 9674, 9675}, - {9506, 9679, 9679}, - {9669, 9685, 9686}, - {9683, 9690, 9690}, - {8763, 9697, 9698}, - {7468, 9702, 9702}, - {460, 9707, 9707}, - {3115, 9712, 9712}, - {9424, 9716, 9717}, - {7359, 9721, 9724}, - {7547, 9728, 9729}, - {7151, 9733, 9738}, - {7627, 9742, 9742}, - {2822, 9747, 9747}, - {8247, 9751, 9753}, - {9550, 9758, 9758}, - {7585, 9762, 9763}, - {1002, 9767, 9767}, - {7168, 9772, 9773}, - {6941, 9777, 9780}, - {9728, 9784, 9786}, - {9770, 9792, 9796}, - {6411, 9801, 9802}, - {3689, 9806, 9808}, - {9575, 9814, 9816}, - {7025, 9820, 9821}, - {2776, 9826, 9826}, - {9806, 9830, 9830}, - {9820, 9834, 9835}, - {9800, 9839, 9847}, - {9834, 9851, 9852}, - {9829, 9856, 9862}, - {1400, 9866, 9866}, - {3197, 9870, 9871}, - {9851, 9875, 9876}, - {9742, 9883, 9884}, - {3362, 9888, 9889}, - {9883, 9893, 9893}, - {5711, 9899, 9910}, - {7806, 9915, 9915}, - {9120, 9919, 9919}, - {9715, 9925, 9934}, - {2580, 9938, 9938}, - {4907, 9942, 9944}, - {6239, 9953, 9954}, - {6961, 9963, 9963}, - {5295, 9967, 9968}, - {1915, 9972, 9973}, - {3426, 9983, 9985}, - {9875, 9994, 9995}, - {6942, 9999, 9999}, - {6621, 10005, 10005}, - {7589, 10010, 10012}, - {9286, 10020, 10020}, - {838, 10024, 10024}, - {9980, 10028, 10031}, - {9994, 10035, 10041}, - {2702, 10048, 10051}, - {2621, 10059, 10059}, - {10054, 10065, 10065}, - {8612, 10073, 10074}, - {7033, 10078, 10078}, - {916, 10082, 10082}, - {10035, 10086, 10087}, - {8613, 10097, 10097}, - {9919, 10107, 10108}, - {6133, 10114, 10115}, - {10059, 10119, 10119}, - {10065, 10126, 10127}, - {7732, 10131, 10131}, - {7155, 10135, 10136}, - {6728, 10140, 10140}, - {6162, 10144, 10145}, - {4724, 10150, 10150}, - {1665, 10154, 10154}, - {10126, 10163, 10163}, - {9783, 10168, 10168}, - {1715, 10172, 10173}, - {7152, 10177, 10182}, - {8760, 10187, 10187}, - {7829, 10191, 10191}, - {9679, 10196, 10196}, - {9369, 10201, 10201}, - {2928, 10206, 10208}, - {6951, 10214, 10217}, - {5633, 10221, 10221}, - {7199, 10225, 10225}, - {10118, 10230, 10231}, - {9999, 10235, 10236}, - {10045, 10240, 10249}, - {5565, 10256, 10256}, - {9866, 10261, 10261}, - {10163, 10268, 10268}, - {9869, 10272, 10272}, - {9789, 10276, 10283}, - {10235, 10287, 10288}, - {10214, 10298, 10299}, - {6971, 10303, 10303}, - {3346, 10307, 10307}, - {10185, 10311, 10312}, - {9993, 10318, 10320}, - {2779, 10332, 10334}, - {1726, 10338, 10338}, - {741, 10354, 10360}, - {10230, 10372, 10373}, - {10260, 10384, 10385}, - {10131, 10389, 10398}, - {6946, 10406, 10409}, - {10158, 10413, 10420}, - {10123, 10424, 10424}, - {6157, 10428, 10429}, - {4518, 10434, 10434}, - {9893, 10438, 10438}, - {9865, 10442, 10446}, - {7558, 10454, 10454}, - {10434, 10460, 10460}, - {10064, 10466, 10468}, - {2703, 10472, 10474}, - {9751, 10478, 10479}, - {6714, 10485, 10485}, - {8020, 10490, 10490}, - {10303, 10494, 10494}, - {3521, 10499, 10500}, - {9281, 10513, 10515}, - {6028, 10519, 10523}, - {9387, 10527, 10527}, - {7614, 10531, 10531}, - {3611, 10536, 10536}, - {9162, 10540, 10540}, - {10081, 10546, 10547}, - {10034, 10560, 10562}, - {6726, 10567, 10571}, - {8237, 10575, 10575}, - {10438, 10579, 10583}, - {10140, 10587, 10587}, - {5784, 10592, 10592}, - {9819, 10597, 10600}, - {10567, 10604, 10608}, - {9335, 10613, 10613}, - {8300, 10617, 10617}, - {10575, 10621, 10621}, - {9678, 10625, 10626}, - {9962, 10632, 10633}, - {10535, 10637, 10638}, - {8199, 10642, 10642}, - {10372, 10647, 10648}, - {10637, 10656, 10657}, - {10579, 10667, 10668}, - {10465, 10677, 10680}, - {6702, 10684, 10685}, - {10073, 10691, 10692}, - {4505, 10696, 10697}, - {9042, 10701, 10701}, - {6460, 10705, 10706}, - {10010, 10714, 10716}, - {10656, 10720, 10722}, - {7282, 10727, 10729}, - {2327, 10733, 10733}, - {2491, 10740, 10741}, - {10704, 10748, 10750}, - {6465, 10754, 10754}, - {10647, 10758, 10759}, - {10424, 10763, 10763}, - {10748, 10776, 10776}, - {10546, 10780, 10781}, - {10758, 10785, 10786}, - {10287, 10790, 10797}, - {10785, 10801, 10807}, - {10240, 10811, 10826}, - {9509, 10830, 10830}, - {2579, 10836, 10838}, - {9801, 10843, 10845}, - {7555, 10849, 10850}, - {10776, 10860, 10865}, - {8023, 10869, 10869}, - {10046, 10876, 10884}, - {10253, 10888, 10892}, - {9941, 10897, 10897}, - {7898, 10901, 10905}, - {6725, 10909, 10913}, - {10757, 10921, 10923}, - {10160, 10931, 10931}, - {10916, 10935, 10942}, - {10261, 10946, 10946}, - {10318, 10952, 10954}, - {5911, 10959, 10961}, - {10801, 10965, 10966}, - {10946, 10970, 10977}, - {10592, 10982, 10984}, - {9913, 10988, 10990}, - {8510, 10994, 10996}, - {9419, 11000, 11001}, - {6765, 11006, 11007}, - {10725, 11011, 11011}, - {5537, 11017, 11019}, - {9208, 11024, 11025}, - {5850, 11030, 11030}, - {9610, 11034, 11036}, - {8846, 11041, 11047}, - {9697, 11051, 11051}, - {1622, 11055, 11058}, - {2370, 11062, 11062}, - {8393, 11067, 11067}, - {9756, 11071, 11071}, - {10172, 11076, 11076}, - {27, 11081, 11081}, - {7357, 11087, 11092}, - {8151, 11104, 11106}, - {6115, 11110, 11110}, - {10667, 11114, 11115}, - {11099, 11121, 11123}, - {10705, 11127, 11127}, - {8938, 11131, 11131}, - {11114, 11135, 11136}, - {1390, 11140, 11141}, - {10964, 11146, 11148}, - {11140, 11152, 11155}, - {9813, 11159, 11166}, - {624, 11171, 11172}, - {3118, 11177, 11179}, - {11029, 11184, 11186}, - {10186, 11190, 11190}, - {10306, 11196, 11196}, - {8665, 11201, 11201}, - {7382, 11205, 11205}, - {1100, 11210, 11210}, - {2337, 11216, 11217}, - {1609, 11221, 11223}, - {5763, 11228, 11229}, - {5220, 11233, 11233}, - {11061, 11241, 11241}, - {10617, 11246, 11246}, - {11190, 11250, 11251}, - {10144, 11255, 11256}, - {11232, 11260, 11260}, - {857, 11264, 11265}, - {10994, 11269, 11271}, - {3879, 11280, 11281}, - {11184, 11287, 11289}, - {9611, 11293, 11295}, - {11250, 11299, 11299}, - {4495, 11304, 11304}, - {7574, 11308, 11309}, - {9814, 11315, 11317}, - {1713, 11321, 11324}, - {1905, 11328, 11328}, - {8745, 11335, 11340}, - {8883, 11351, 11351}, - {8119, 11358, 11358}, - {1842, 11363, 11364}, - {11237, 11368, 11368}, - {8814, 11373, 11374}, - {5684, 11378, 11378}, - {11011, 11382, 11382}, - {6520, 11389, 11389}, - {11183, 11393, 11396}, - {1790, 11404, 11404}, - {9536, 11408, 11408}, - {11298, 11418, 11419}, - {3929, 11425, 11425}, - {5588, 11429, 11429}, - {8476, 11436, 11436}, - {4096, 11440, 11442}, - {11084, 11446, 11454}, - {10603, 11458, 11463}, - {7332, 11472, 11474}, - {7611, 11483, 11486}, - {4836, 11490, 11491}, - {10024, 11495, 11495}, - {4917, 11501, 11506}, - {6486, 11510, 11512}, - {11269, 11516, 11518}, - {3603, 11522, 11525}, - {11126, 11535, 11535}, - {11418, 11539, 11541}, - {11408, 11545, 11545}, - {9021, 11549, 11552}, - {6745, 11557, 11557}, - {5118, 11561, 11564}, - {7590, 11568, 11569}, - {4426, 11573, 11578}, - {9790, 11582, 11583}, - {6447, 11587, 11587}, - {10229, 11591, 11594}, - {10457, 11598, 11598}, - {10168, 11604, 11604}, - {10543, 11608, 11608}, - {7404, 11612, 11612}, - {11127, 11616, 11616}, - {3337, 11620, 11620}, - {11501, 11624, 11628}, - {4543, 11633, 11635}, - {8449, 11642, 11642}, - {4943, 11646, 11648}, - {10526, 11652, 11654}, - {11620, 11659, 11659}, - {8927, 11664, 11669}, - {532, 11673, 11673}, - {10513, 11677, 11679}, - {10428, 11683, 11683}, - {10999, 11689, 11690}, - {9469, 11695, 11695}, - {3606, 11699, 11699}, - {9560, 11708, 11709}, - {1564, 11714, 11714}, - {10527, 11718, 11718}, - {3071, 11723, 11726}, - {11590, 11731, 11732}, - {6605, 11737, 11737}, - {11624, 11741, 11745}, - {7822, 11749, 11752}, - {5269, 11757, 11758}, - {1339, 11767, 11767}, - {1363, 11771, 11773}, - {3704, 11777, 11777}, - {10952, 11781, 11783}, - {6764, 11793, 11795}, - {8675, 11800, 11800}, - {9963, 11804, 11804}, - {11573, 11808, 11809}, - {9548, 11813, 11813}, - {11591, 11817, 11818}, - {11446, 11822, 11822}, - {9224, 11828, 11828}, - {3158, 11836, 11836}, - {10830, 11840, 11840}, - {7234, 11846, 11846}, - {11299, 11850, 11850}, - {11544, 11854, 11855}, - {11498, 11859, 11859}, - {10993, 11865, 11868}, - {9720, 11872, 11878}, - {10489, 11882, 11890}, - {11712, 11898, 11904}, - {11516, 11908, 11910}, - {11568, 11914, 11915}, - {10177, 11919, 11924}, - {11363, 11928, 11929}, - {10494, 11933, 11933}, - {9870, 11937, 11938}, - {9427, 11942, 11942}, - {11481, 11949, 11949}, - {6030, 11955, 11957}, - {11718, 11961, 11961}, - {10531, 11965, 11983}, - {5126, 11987, 11987}, - {7515, 11991, 11991}, - {10646, 11996, 11997}, - {2947, 12001, 12001}, - {9582, 12009, 12010}, - {6202, 12017, 12018}, - {11714, 12022, 12022}, - {9235, 12033, 12037}, - {9721, 12041, 12044}, - {11932, 12051, 12052}, - {12040, 12056, 12056}, - {12051, 12060, 12060}, - {11601, 12066, 12066}, - {8426, 12070, 12070}, - {4053, 12077, 12077}, - {4262, 12081, 12081}, - {9761, 12086, 12088}, - {11582, 12092, 12093}, - {10965, 12097, 12098}, - {11803, 12103, 12104}, - {11933, 12108, 12109}, - {10688, 12117, 12117}, - {12107, 12125, 12126}, - {6774, 12130, 12132}, - {6286, 12137, 12137}, - {9543, 12141, 12141}, - {12097, 12145, 12146}, - {10790, 12150, 12150}, - {10125, 12154, 12156}, - {12125, 12164, 12164}, - {12064, 12168, 12172}, - {10811, 12178, 12188}, - {12092, 12192, 12193}, - {10058, 12197, 12198}, - {11611, 12211, 12212}, - {3459, 12216, 12216}, - {10291, 12225, 12228}, - {12191, 12232, 12234}, - {12145, 12238, 12238}, - {12001, 12242, 12250}, - {3840, 12255, 12255}, - {12216, 12259, 12259}, - {674, 12272, 12272}, - {12141, 12276, 12276}, - {10766, 12280, 12280}, - {11545, 12284, 12284}, - {6496, 12290, 12290}, - {11381, 12294, 12295}, - {603, 12302, 12303}, - {12276, 12308, 12308}, - {11850, 12313, 12314}, - {565, 12319, 12319}, - {9351, 12324, 12324}, - {11822, 12328, 12328}, - {2691, 12333, 12334}, - {11840, 12338, 12338}, - {11070, 12343, 12343}, - {9510, 12347, 12347}, - {11024, 12352, 12353}, - {7173, 12359, 12359}, - {517, 12363, 12363}, - {6311, 12367, 12368}, - {11367, 12372, 12373}, - {12008, 12377, 12377}, - {11372, 12382, 12384}, - {11358, 12391, 12392}, - {11382, 12396, 12396}, - {6882, 12400, 12401}, - {11246, 12405, 12405}, - {8359, 12409, 12412}, - {10154, 12418, 12418}, - {12016, 12425, 12426}, - {8972, 12434, 12435}, - {10478, 12439, 12440}, - {12395, 12449, 12449}, - {11612, 12454, 12454}, - {12347, 12458, 12458}, - {10700, 12466, 12467}, - {3637, 12471, 12476}, - {1042, 12480, 12481}, - {6747, 12488, 12488}, - {12396, 12492, 12493}, - {9420, 12497, 12497}, - {11285, 12501, 12510}, - {4470, 12515, 12515}, - {9374, 12519, 12519}, - {11293, 12528, 12528}, - {2058, 12534, 12535}, - {6521, 12539, 12539}, - {12492, 12543, 12543}, - {3043, 12547, 12547}, - {2982, 12551, 12553}, - {11030, 12557, 12563}, - {7636, 12568, 12568}, - {9639, 12572, 12572}, - {12543, 12576, 12576}, - {5989, 12580, 12583}, - {11051, 12587, 12587}, - {1061, 12592, 12594}, - {12313, 12599, 12601}, - {11846, 12605, 12605}, - {12576, 12609, 12609}, - {11040, 12618, 12625}, - {12479, 12629, 12629}, - {6903, 12633, 12633}, - {12322, 12639, 12639}, - {12253, 12643, 12645}, - {5594, 12651, 12651}, - {12522, 12655, 12655}, - {11703, 12659, 12659}, - {1377, 12665, 12665}, - {8022, 12669, 12669}, - {12280, 12674, 12674}, - {9023, 12680, 12681}, - {12328, 12685, 12685}, - {3085, 12689, 12693}, - {4700, 12698, 12698}, - {10224, 12702, 12702}, - {8781, 12706, 12706}, - {1651, 12710, 12710}, - {12458, 12714, 12714}, - {12005, 12718, 12721}, - {11908, 12725, 12726}, - {8202, 12733, 12733}, - {11708, 12739, 12740}, - {12599, 12744, 12745}, - {12284, 12749, 12749}, - {5285, 12756, 12756}, - {12055, 12775, 12777}, - {6919, 12782, 12782}, - {12242, 12786, 12786}, - {12009, 12790, 12790}, - {9628, 12794, 12796}, - {11354, 12801, 12802}, - {10225, 12806, 12807}, - {579, 12813, 12813}, - {8935, 12817, 12822}, - {8753, 12827, 12829}, - {11006, 12835, 12835}, - {858, 12841, 12845}, - {476, 12849, 12849}, - {7667, 12854, 12854}, - {12760, 12860, 12871}, - {11677, 12875, 12877}, - {12714, 12881, 12881}, - {12731, 12885, 12890}, - {7108, 12894, 12896}, - {1165, 12900, 12900}, - {4021, 12906, 12906}, - {10829, 12910, 12911}, - {12331, 12915, 12915}, - {8887, 12919, 12921}, - {11639, 12925, 12925}, - {7964, 12929, 12929}, - {12528, 12937, 12937}, - {8148, 12941, 12941}, - {12770, 12948, 12950}, - {12609, 12954, 12954}, - {12685, 12958, 12958}, - {2803, 12962, 12962}, - {9561, 12966, 12966}, - {6671, 12972, 12973}, - {12056, 12977, 12977}, - {6380, 12981, 12981}, - {12048, 12985, 12985}, - {11961, 12989, 12993}, - {3368, 12997, 12999}, - {6634, 13004, 13004}, - {6775, 13009, 13010}, - {12136, 13014, 13019}, - {10341, 13023, 13023}, - {13002, 13027, 13027}, - {10587, 13031, 13031}, - {10307, 13035, 13035}, - {12736, 13039, 13039}, - {12744, 13043, 13044}, - {6175, 13048, 13048}, - {9702, 13053, 13054}, - {662, 13059, 13061}, - {12718, 13065, 13068}, - {12893, 13072, 13075}, - {8299, 13086, 13091}, - {12604, 13095, 13096}, - {12848, 13100, 13101}, - {12749, 13105, 13105}, - {12526, 13109, 13114}, - {9173, 13122, 13122}, - {12769, 13128, 13128}, - {13038, 13132, 13132}, - {12725, 13136, 13137}, - {12639, 13146, 13146}, - {9711, 13150, 13151}, - {12137, 13155, 13155}, - {13039, 13159, 13159}, - {4681, 13163, 13164}, - {12954, 13168, 13168}, - {13158, 13175, 13176}, - {13105, 13180, 13180}, - {10754, 13184, 13184}, - {13167, 13188, 13188}, - {12658, 13192, 13192}, - {4294, 13199, 13200}, - {11682, 13204, 13205}, - {11695, 13209, 13209}, - {11076, 13214, 13214}, - {12232, 13218, 13218}, - {9399, 13223, 13224}, - {12880, 13228, 13229}, - {13048, 13234, 13234}, - {9701, 13238, 13239}, - {13209, 13243, 13243}, - {3658, 13248, 13248}, - {3698, 13252, 13254}, - {12237, 13260, 13260}, - {8872, 13266, 13266}, - {12957, 13272, 13273}, - {1393, 13281, 13281}, - {2013, 13285, 13288}, - {4244, 13296, 13299}, - {9428, 13303, 13303}, - {12702, 13307, 13307}, - {13078, 13311, 13311}, - {6071, 13315, 13315}, - {3061, 13319, 13319}, - {2051, 13324, 13324}, - {11560, 13328, 13331}, - {6584, 13336, 13336}, - {8482, 13340, 13340}, - {5331, 13344, 13344}, - {4171, 13348, 13348}, - {8501, 13352, 13352}, - {9219, 13356, 13356}, - {9473, 13360, 13363}, - {12881, 13367, 13367}, - {13065, 13371, 13375}, - {2979, 13379, 13384}, - {1518, 13388, 13388}, - {11177, 13392, 13392}, - {9457, 13398, 13398}, - {12293, 13407, 13410}, - {3697, 13414, 13417}, - {10338, 13425, 13425}, - {13367, 13429, 13429}, - {11074, 13433, 13437}, - {4201, 13441, 13443}, - {1812, 13447, 13448}, - {13360, 13452, 13456}, - {13188, 13463, 13463}, - {9732, 13470, 13470}, - {11332, 13477, 13477}, - {9918, 13487, 13487}, - {6337, 13497, 13497}, - {13429, 13501, 13501}, - {11413, 13505, 13505}, - {4685, 13512, 13513}, - {13136, 13517, 13519}, - {7416, 13528, 13530}, - {12929, 13534, 13534}, - {11110, 13539, 13539}, - {11521, 13543, 13543}, - {12825, 13553, 13553}, - {13447, 13557, 13558}, - {12299, 13562, 13563}, - {9003, 13570, 13570}, - {12500, 13577, 13577}, - {13501, 13581, 13581}, - {9392, 13586, 13586}, - {12454, 13590, 13590}, - {6189, 13595, 13595}, - {13053, 13599, 13599}, - {11881, 13604, 13604}, - {13159, 13608, 13608}, - {4894, 13612, 13612}, - {13221, 13621, 13621}, - {8950, 13625, 13625}, - {13533, 13629, 13629}, - {9633, 13633, 13633}, - {7892, 13637, 13639}, - {13581, 13643, 13643}, - {13616, 13647, 13649}, - {12794, 13653, 13654}, - {8919, 13659, 13659}, - {9674, 13663, 13663}, - {13577, 13668, 13668}, - {12966, 13672, 13672}, - {12659, 13676, 13683}, - {6124, 13688, 13688}, - {9225, 13693, 13695}, - {11833, 13702, 13702}, - {12904, 13709, 13717}, - {13647, 13721, 13722}, - {11687, 13726, 13727}, - {12434, 13731, 13732}, - {12689, 13736, 13742}, - {13168, 13746, 13746}, - {6151, 13751, 13752}, - {11821, 13756, 13757}, - {6467, 13764, 13764}, - {5730, 13769, 13769}, - {5136, 13780, 13780}, - {724, 13784, 13785}, - {13517, 13789, 13791}, - {640, 13795, 13796}, - {7721, 13800, 13802}, - {11121, 13806, 13807}, - {5791, 13811, 13815}, - {12894, 13819, 13819}, - {11100, 13824, 13824}, - {7011, 13830, 13830}, - {7129, 13834, 13837}, - {13833, 13841, 13841}, - {11276, 13847, 13847}, - {13621, 13853, 13853}, - {13589, 13862, 13863}, - {12989, 13867, 13867}, - {12789, 13871, 13871}, - {1239, 13875, 13875}, - {4675, 13879, 13881}, - {4686, 13885, 13885}, - {707, 13889, 13889}, - {5449, 13897, 13898}, - {13867, 13902, 13903}, - {10613, 13908, 13908}, - {13789, 13912, 13914}, - {4451, 13918, 13919}, - {9200, 13924, 13924}, - {2011, 13930, 13930}, - {11433, 13934, 13936}, - {4695, 13942, 13943}, - {9435, 13948, 13951}, - {13688, 13955, 13957}, - {11694, 13961, 13962}, - {5712, 13966, 13966}, - {5991, 13970, 13972}, - {13477, 13976, 13976}, - {10213, 13987, 13987}, - {11839, 13991, 13993}, - {12272, 13997, 13997}, - {6206, 14001, 14001}, - {13179, 14006, 14007}, - {2939, 14011, 14011}, - {12972, 14016, 14017}, - {13918, 14021, 14022}, - {7436, 14026, 14027}, - {7678, 14032, 14034}, - {13586, 14040, 14040}, - {13347, 14044, 14044}, - {13109, 14048, 14051}, - {9244, 14055, 14057}, - {13315, 14061, 14061}, - {13276, 14067, 14067}, - {11435, 14073, 14074}, - {13853, 14078, 14078}, - {13452, 14082, 14082}, - {14044, 14087, 14087}, - {4440, 14091, 14095}, - {4479, 14100, 14103}, - {9395, 14107, 14109}, - {6834, 14119, 14119}, - {10458, 14123, 14124}, - {1429, 14129, 14129}, - {8443, 14135, 14135}, - {10365, 14140, 14140}, - {5267, 14145, 14145}, - {11834, 14151, 14153}, -} diff --git a/vendor/github.com/golang/snappy/snappy.go b/vendor/github.com/golang/snappy/snappy.go deleted file mode 100644 index 0cf5e37..0000000 --- a/vendor/github.com/golang/snappy/snappy.go +++ /dev/null @@ -1,87 +0,0 @@ -// Copyright 2011 The Snappy-Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Package snappy implements the snappy block-based compression format. -// It aims for very high speeds and reasonable compression. -// -// The C++ snappy implementation is at https://github.com/google/snappy -package snappy // import "github.com/golang/snappy" - -import ( - "hash/crc32" -) - -/* -Each encoded block begins with the varint-encoded length of the decoded data, -followed by a sequence of chunks. Chunks begin and end on byte boundaries. The -first byte of each chunk is broken into its 2 least and 6 most significant bits -called l and m: l ranges in [0, 4) and m ranges in [0, 64). l is the chunk tag. -Zero means a literal tag. All other values mean a copy tag. - -For literal tags: - - If m < 60, the next 1 + m bytes are literal bytes. - - Otherwise, let n be the little-endian unsigned integer denoted by the next - m - 59 bytes. The next 1 + n bytes after that are literal bytes. - -For copy tags, length bytes are copied from offset bytes ago, in the style of -Lempel-Ziv compression algorithms. In particular: - - For l == 1, the offset ranges in [0, 1<<11) and the length in [4, 12). - The length is 4 + the low 3 bits of m. The high 3 bits of m form bits 8-10 - of the offset. The next byte is bits 0-7 of the offset. - - For l == 2, the offset ranges in [0, 1<<16) and the length in [1, 65). - The length is 1 + m. The offset is the little-endian unsigned integer - denoted by the next 2 bytes. - - For l == 3, this tag is a legacy format that is no longer issued by most - encoders. Nonetheless, the offset ranges in [0, 1<<32) and the length in - [1, 65). The length is 1 + m. The offset is the little-endian unsigned - integer denoted by the next 4 bytes. -*/ -const ( - tagLiteral = 0x00 - tagCopy1 = 0x01 - tagCopy2 = 0x02 - tagCopy4 = 0x03 -) - -const ( - checksumSize = 4 - chunkHeaderSize = 4 - magicChunk = "\xff\x06\x00\x00" + magicBody - magicBody = "sNaPpY" - - // maxBlockSize is the maximum size of the input to encodeBlock. It is not - // part of the wire format per se, but some parts of the encoder assume - // that an offset fits into a uint16. - // - // Also, for the framing format (Writer type instead of Encode function), - // https://github.com/google/snappy/blob/master/framing_format.txt says - // that "the uncompressed data in a chunk must be no longer than 65536 - // bytes". - maxBlockSize = 65536 - - // maxEncodedLenOfMaxBlockSize equals MaxEncodedLen(maxBlockSize), but is - // hard coded to be a const instead of a variable, so that obufLen can also - // be a const. Their equivalence is confirmed by - // TestMaxEncodedLenOfMaxBlockSize. - maxEncodedLenOfMaxBlockSize = 76490 - - obufHeaderLen = len(magicChunk) + checksumSize + chunkHeaderSize - obufLen = obufHeaderLen + maxEncodedLenOfMaxBlockSize -) - -const ( - chunkTypeCompressedData = 0x00 - chunkTypeUncompressedData = 0x01 - chunkTypePadding = 0xfe - chunkTypeStreamIdentifier = 0xff -) - -var crcTable = crc32.MakeTable(crc32.Castagnoli) - -// crc implements the checksum specified in section 3 of -// https://github.com/google/snappy/blob/master/framing_format.txt -func crc(b []byte) uint32 { - c := crc32.Update(0, crcTable, b) - return uint32(c>>15|c<<17) + 0xa282ead8 -} diff --git a/vendor/github.com/golang/snappy/snappy_test.go b/vendor/github.com/golang/snappy/snappy_test.go deleted file mode 100644 index 2712710..0000000 --- a/vendor/github.com/golang/snappy/snappy_test.go +++ /dev/null @@ -1,1353 +0,0 @@ -// Copyright 2011 The Snappy-Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package snappy - -import ( - "bytes" - "encoding/binary" - "flag" - "fmt" - "io" - "io/ioutil" - "math/rand" - "net/http" - "os" - "os/exec" - "path/filepath" - "runtime" - "strings" - "testing" -) - -var ( - download = flag.Bool("download", false, "If true, download any missing files before running benchmarks") - testdataDir = flag.String("testdataDir", "testdata", "Directory containing the test data") - benchdataDir = flag.String("benchdataDir", "testdata/bench", "Directory containing the benchmark data") -) - -// goEncoderShouldMatchCppEncoder is whether to test that the algorithm used by -// Go's encoder matches byte-for-byte what the C++ snappy encoder produces, on -// this GOARCH. There is more than one valid encoding of any given input, and -// there is more than one good algorithm along the frontier of trading off -// throughput for output size. Nonetheless, we presume that the C++ encoder's -// algorithm is a good one and has been tested on a wide range of inputs, so -// matching that exactly should mean that the Go encoder's algorithm is also -// good, without needing to gather our own corpus of test data. -// -// The exact algorithm used by the C++ code is potentially endian dependent, as -// it puns a byte pointer to a uint32 pointer to load, hash and compare 4 bytes -// at a time. The Go implementation is endian agnostic, in that its output is -// the same (as little-endian C++ code), regardless of the CPU's endianness. -// -// Thus, when comparing Go's output to C++ output generated beforehand, such as -// the "testdata/pi.txt.rawsnappy" file generated by C++ code on a little- -// endian system, we can run that test regardless of the runtime.GOARCH value. -// -// When comparing Go's output to dynamically generated C++ output, i.e. the -// result of fork/exec'ing a C++ program, we can run that test only on -// little-endian systems, because the C++ output might be different on -// big-endian systems. The runtime package doesn't export endianness per se, -// but we can restrict this match-C++ test to common little-endian systems. -const goEncoderShouldMatchCppEncoder = runtime.GOARCH == "386" || runtime.GOARCH == "amd64" || runtime.GOARCH == "arm" - -func TestMaxEncodedLenOfMaxBlockSize(t *testing.T) { - got := maxEncodedLenOfMaxBlockSize - want := MaxEncodedLen(maxBlockSize) - if got != want { - t.Fatalf("got %d, want %d", got, want) - } -} - -func cmp(a, b []byte) error { - if bytes.Equal(a, b) { - return nil - } - if len(a) != len(b) { - return fmt.Errorf("got %d bytes, want %d", len(a), len(b)) - } - for i := range a { - if a[i] != b[i] { - return fmt.Errorf("byte #%d: got 0x%02x, want 0x%02x", i, a[i], b[i]) - } - } - return nil -} - -func roundtrip(b, ebuf, dbuf []byte) error { - d, err := Decode(dbuf, Encode(ebuf, b)) - if err != nil { - return fmt.Errorf("decoding error: %v", err) - } - if err := cmp(d, b); err != nil { - return fmt.Errorf("roundtrip mismatch: %v", err) - } - return nil -} - -func TestEmpty(t *testing.T) { - if err := roundtrip(nil, nil, nil); err != nil { - t.Fatal(err) - } -} - -func TestSmallCopy(t *testing.T) { - for _, ebuf := range [][]byte{nil, make([]byte, 20), make([]byte, 64)} { - for _, dbuf := range [][]byte{nil, make([]byte, 20), make([]byte, 64)} { - for i := 0; i < 32; i++ { - s := "aaaa" + strings.Repeat("b", i) + "aaaabbbb" - if err := roundtrip([]byte(s), ebuf, dbuf); err != nil { - t.Errorf("len(ebuf)=%d, len(dbuf)=%d, i=%d: %v", len(ebuf), len(dbuf), i, err) - } - } - } - } -} - -func TestSmallRand(t *testing.T) { - rng := rand.New(rand.NewSource(1)) - for n := 1; n < 20000; n += 23 { - b := make([]byte, n) - for i := range b { - b[i] = uint8(rng.Intn(256)) - } - if err := roundtrip(b, nil, nil); err != nil { - t.Fatal(err) - } - } -} - -func TestSmallRegular(t *testing.T) { - for n := 1; n < 20000; n += 23 { - b := make([]byte, n) - for i := range b { - b[i] = uint8(i%10 + 'a') - } - if err := roundtrip(b, nil, nil); err != nil { - t.Fatal(err) - } - } -} - -func TestInvalidVarint(t *testing.T) { - testCases := []struct { - desc string - input string - }{{ - "invalid varint, final byte has continuation bit set", - "\xff", - }, { - "invalid varint, value overflows uint64", - "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\x00", - }, { - // https://github.com/google/snappy/blob/master/format_description.txt - // says that "the stream starts with the uncompressed length [as a - // varint] (up to a maximum of 2^32 - 1)". - "valid varint (as uint64), but value overflows uint32", - "\x80\x80\x80\x80\x10", - }} - - for _, tc := range testCases { - input := []byte(tc.input) - if _, err := DecodedLen(input); err != ErrCorrupt { - t.Errorf("%s: DecodedLen: got %v, want ErrCorrupt", tc.desc, err) - } - if _, err := Decode(nil, input); err != ErrCorrupt { - t.Errorf("%s: Decode: got %v, want ErrCorrupt", tc.desc, err) - } - } -} - -func TestDecode(t *testing.T) { - lit40Bytes := make([]byte, 40) - for i := range lit40Bytes { - lit40Bytes[i] = byte(i) - } - lit40 := string(lit40Bytes) - - testCases := []struct { - desc string - input string - want string - wantErr error - }{{ - `decodedLen=0; valid input`, - "\x00", - "", - nil, - }, { - `decodedLen=3; tagLiteral, 0-byte length; length=3; valid input`, - "\x03" + "\x08\xff\xff\xff", - "\xff\xff\xff", - nil, - }, { - `decodedLen=2; tagLiteral, 0-byte length; length=3; not enough dst bytes`, - "\x02" + "\x08\xff\xff\xff", - "", - ErrCorrupt, - }, { - `decodedLen=3; tagLiteral, 0-byte length; length=3; not enough src bytes`, - "\x03" + "\x08\xff\xff", - "", - ErrCorrupt, - }, { - `decodedLen=40; tagLiteral, 0-byte length; length=40; valid input`, - "\x28" + "\x9c" + lit40, - lit40, - nil, - }, { - `decodedLen=1; tagLiteral, 1-byte length; not enough length bytes`, - "\x01" + "\xf0", - "", - ErrCorrupt, - }, { - `decodedLen=3; tagLiteral, 1-byte length; length=3; valid input`, - "\x03" + "\xf0\x02\xff\xff\xff", - "\xff\xff\xff", - nil, - }, { - `decodedLen=1; tagLiteral, 2-byte length; not enough length bytes`, - "\x01" + "\xf4\x00", - "", - ErrCorrupt, - }, { - `decodedLen=3; tagLiteral, 2-byte length; length=3; valid input`, - "\x03" + "\xf4\x02\x00\xff\xff\xff", - "\xff\xff\xff", - nil, - }, { - `decodedLen=1; tagLiteral, 3-byte length; not enough length bytes`, - "\x01" + "\xf8\x00\x00", - "", - ErrCorrupt, - }, { - `decodedLen=3; tagLiteral, 3-byte length; length=3; valid input`, - "\x03" + "\xf8\x02\x00\x00\xff\xff\xff", - "\xff\xff\xff", - nil, - }, { - `decodedLen=1; tagLiteral, 4-byte length; not enough length bytes`, - "\x01" + "\xfc\x00\x00\x00", - "", - ErrCorrupt, - }, { - `decodedLen=1; tagLiteral, 4-byte length; length=3; not enough dst bytes`, - "\x01" + "\xfc\x02\x00\x00\x00\xff\xff\xff", - "", - ErrCorrupt, - }, { - `decodedLen=4; tagLiteral, 4-byte length; length=3; not enough src bytes`, - "\x04" + "\xfc\x02\x00\x00\x00\xff", - "", - ErrCorrupt, - }, { - `decodedLen=3; tagLiteral, 4-byte length; length=3; valid input`, - "\x03" + "\xfc\x02\x00\x00\x00\xff\xff\xff", - "\xff\xff\xff", - nil, - }, { - `decodedLen=4; tagCopy1, 1 extra length|offset byte; not enough extra bytes`, - "\x04" + "\x01", - "", - ErrCorrupt, - }, { - `decodedLen=4; tagCopy2, 2 extra length|offset bytes; not enough extra bytes`, - "\x04" + "\x02\x00", - "", - ErrCorrupt, - }, { - `decodedLen=4; tagCopy4, 4 extra length|offset bytes; not enough extra bytes`, - "\x04" + "\x03\x00\x00\x00", - "", - ErrCorrupt, - }, { - `decodedLen=4; tagLiteral (4 bytes "abcd"); valid input`, - "\x04" + "\x0cabcd", - "abcd", - nil, - }, { - `decodedLen=13; tagLiteral (4 bytes "abcd"); tagCopy1; length=9 offset=4; valid input`, - "\x0d" + "\x0cabcd" + "\x15\x04", - "abcdabcdabcda", - nil, - }, { - `decodedLen=8; tagLiteral (4 bytes "abcd"); tagCopy1; length=4 offset=4; valid input`, - "\x08" + "\x0cabcd" + "\x01\x04", - "abcdabcd", - nil, - }, { - `decodedLen=8; tagLiteral (4 bytes "abcd"); tagCopy1; length=4 offset=2; valid input`, - "\x08" + "\x0cabcd" + "\x01\x02", - "abcdcdcd", - nil, - }, { - `decodedLen=8; tagLiteral (4 bytes "abcd"); tagCopy1; length=4 offset=1; valid input`, - "\x08" + "\x0cabcd" + "\x01\x01", - "abcddddd", - nil, - }, { - `decodedLen=8; tagLiteral (4 bytes "abcd"); tagCopy1; length=4 offset=0; zero offset`, - "\x08" + "\x0cabcd" + "\x01\x00", - "", - ErrCorrupt, - }, { - `decodedLen=9; tagLiteral (4 bytes "abcd"); tagCopy1; length=4 offset=4; inconsistent dLen`, - "\x09" + "\x0cabcd" + "\x01\x04", - "", - ErrCorrupt, - }, { - `decodedLen=8; tagLiteral (4 bytes "abcd"); tagCopy1; length=4 offset=5; offset too large`, - "\x08" + "\x0cabcd" + "\x01\x05", - "", - ErrCorrupt, - }, { - `decodedLen=7; tagLiteral (4 bytes "abcd"); tagCopy1; length=4 offset=4; length too large`, - "\x07" + "\x0cabcd" + "\x01\x04", - "", - ErrCorrupt, - }, { - `decodedLen=6; tagLiteral (4 bytes "abcd"); tagCopy2; length=2 offset=3; valid input`, - "\x06" + "\x0cabcd" + "\x06\x03\x00", - "abcdbc", - nil, - }, { - `decodedLen=6; tagLiteral (4 bytes "abcd"); tagCopy4; length=2 offset=3; valid input`, - "\x06" + "\x0cabcd" + "\x07\x03\x00\x00\x00", - "abcdbc", - nil, - }} - - const ( - // notPresentXxx defines a range of byte values [0xa0, 0xc5) that are - // not present in either the input or the output. It is written to dBuf - // to check that Decode does not write bytes past the end of - // dBuf[:dLen]. - // - // The magic number 37 was chosen because it is prime. A more 'natural' - // number like 32 might lead to a false negative if, for example, a - // byte was incorrectly copied 4*8 bytes later. - notPresentBase = 0xa0 - notPresentLen = 37 - ) - - var dBuf [100]byte -loop: - for i, tc := range testCases { - input := []byte(tc.input) - for _, x := range input { - if notPresentBase <= x && x < notPresentBase+notPresentLen { - t.Errorf("#%d (%s): input shouldn't contain %#02x\ninput: % x", i, tc.desc, x, input) - continue loop - } - } - - dLen, n := binary.Uvarint(input) - if n <= 0 { - t.Errorf("#%d (%s): invalid varint-encoded dLen", i, tc.desc) - continue - } - if dLen > uint64(len(dBuf)) { - t.Errorf("#%d (%s): dLen %d is too large", i, tc.desc, dLen) - continue - } - - for j := range dBuf { - dBuf[j] = byte(notPresentBase + j%notPresentLen) - } - g, gotErr := Decode(dBuf[:], input) - if got := string(g); got != tc.want || gotErr != tc.wantErr { - t.Errorf("#%d (%s):\ngot %q, %v\nwant %q, %v", - i, tc.desc, got, gotErr, tc.want, tc.wantErr) - continue - } - for j, x := range dBuf { - if uint64(j) < dLen { - continue - } - if w := byte(notPresentBase + j%notPresentLen); x != w { - t.Errorf("#%d (%s): Decode overrun: dBuf[%d] was modified: got %#02x, want %#02x\ndBuf: % x", - i, tc.desc, j, x, w, dBuf) - continue loop - } - } - } -} - -func TestDecodeCopy4(t *testing.T) { - dots := strings.Repeat(".", 65536) - - input := strings.Join([]string{ - "\x89\x80\x04", // decodedLen = 65545. - "\x0cpqrs", // 4-byte literal "pqrs". - "\xf4\xff\xff" + dots, // 65536-byte literal dots. - "\x13\x04\x00\x01\x00", // tagCopy4; length=5 offset=65540. - }, "") - - gotBytes, err := Decode(nil, []byte(input)) - if err != nil { - t.Fatal(err) - } - got := string(gotBytes) - want := "pqrs" + dots + "pqrs." - if len(got) != len(want) { - t.Fatalf("got %d bytes, want %d", len(got), len(want)) - } - if got != want { - for i := 0; i < len(got); i++ { - if g, w := got[i], want[i]; g != w { - t.Fatalf("byte #%d: got %#02x, want %#02x", i, g, w) - } - } - } -} - -// TestDecodeLengthOffset tests decoding an encoding of the form literal + -// copy-length-offset + literal. For example: "abcdefghijkl" + "efghij" + "AB". -func TestDecodeLengthOffset(t *testing.T) { - const ( - prefix = "abcdefghijklmnopqr" - suffix = "ABCDEFGHIJKLMNOPQR" - - // notPresentXxx defines a range of byte values [0xa0, 0xc5) that are - // not present in either the input or the output. It is written to - // gotBuf to check that Decode does not write bytes past the end of - // gotBuf[:totalLen]. - // - // The magic number 37 was chosen because it is prime. A more 'natural' - // number like 32 might lead to a false negative if, for example, a - // byte was incorrectly copied 4*8 bytes later. - notPresentBase = 0xa0 - notPresentLen = 37 - ) - var gotBuf, wantBuf, inputBuf [128]byte - for length := 1; length <= 18; length++ { - for offset := 1; offset <= 18; offset++ { - loop: - for suffixLen := 0; suffixLen <= 18; suffixLen++ { - totalLen := len(prefix) + length + suffixLen - - inputLen := binary.PutUvarint(inputBuf[:], uint64(totalLen)) - inputBuf[inputLen] = tagLiteral + 4*byte(len(prefix)-1) - inputLen++ - inputLen += copy(inputBuf[inputLen:], prefix) - inputBuf[inputLen+0] = tagCopy2 + 4*byte(length-1) - inputBuf[inputLen+1] = byte(offset) - inputBuf[inputLen+2] = 0x00 - inputLen += 3 - if suffixLen > 0 { - inputBuf[inputLen] = tagLiteral + 4*byte(suffixLen-1) - inputLen++ - inputLen += copy(inputBuf[inputLen:], suffix[:suffixLen]) - } - input := inputBuf[:inputLen] - - for i := range gotBuf { - gotBuf[i] = byte(notPresentBase + i%notPresentLen) - } - got, err := Decode(gotBuf[:], input) - if err != nil { - t.Errorf("length=%d, offset=%d; suffixLen=%d: %v", length, offset, suffixLen, err) - continue - } - - wantLen := 0 - wantLen += copy(wantBuf[wantLen:], prefix) - for i := 0; i < length; i++ { - wantBuf[wantLen] = wantBuf[wantLen-offset] - wantLen++ - } - wantLen += copy(wantBuf[wantLen:], suffix[:suffixLen]) - want := wantBuf[:wantLen] - - for _, x := range input { - if notPresentBase <= x && x < notPresentBase+notPresentLen { - t.Errorf("length=%d, offset=%d; suffixLen=%d: input shouldn't contain %#02x\ninput: % x", - length, offset, suffixLen, x, input) - continue loop - } - } - for i, x := range gotBuf { - if i < totalLen { - continue - } - if w := byte(notPresentBase + i%notPresentLen); x != w { - t.Errorf("length=%d, offset=%d; suffixLen=%d; totalLen=%d: "+ - "Decode overrun: gotBuf[%d] was modified: got %#02x, want %#02x\ngotBuf: % x", - length, offset, suffixLen, totalLen, i, x, w, gotBuf) - continue loop - } - } - for _, x := range want { - if notPresentBase <= x && x < notPresentBase+notPresentLen { - t.Errorf("length=%d, offset=%d; suffixLen=%d: want shouldn't contain %#02x\nwant: % x", - length, offset, suffixLen, x, want) - continue loop - } - } - - if !bytes.Equal(got, want) { - t.Errorf("length=%d, offset=%d; suffixLen=%d:\ninput % x\ngot % x\nwant % x", - length, offset, suffixLen, input, got, want) - continue - } - } - } - } -} - -const ( - goldenText = "Mark.Twain-Tom.Sawyer.txt" - goldenCompressed = goldenText + ".rawsnappy" -) - -func TestDecodeGoldenInput(t *testing.T) { - tDir := filepath.FromSlash(*testdataDir) - src, err := ioutil.ReadFile(filepath.Join(tDir, goldenCompressed)) - if err != nil { - t.Fatalf("ReadFile: %v", err) - } - got, err := Decode(nil, src) - if err != nil { - t.Fatalf("Decode: %v", err) - } - want, err := ioutil.ReadFile(filepath.Join(tDir, goldenText)) - if err != nil { - t.Fatalf("ReadFile: %v", err) - } - if err := cmp(got, want); err != nil { - t.Fatal(err) - } -} - -func TestEncodeGoldenInput(t *testing.T) { - tDir := filepath.FromSlash(*testdataDir) - src, err := ioutil.ReadFile(filepath.Join(tDir, goldenText)) - if err != nil { - t.Fatalf("ReadFile: %v", err) - } - got := Encode(nil, src) - want, err := ioutil.ReadFile(filepath.Join(tDir, goldenCompressed)) - if err != nil { - t.Fatalf("ReadFile: %v", err) - } - if err := cmp(got, want); err != nil { - t.Fatal(err) - } -} - -func TestExtendMatchGoldenInput(t *testing.T) { - tDir := filepath.FromSlash(*testdataDir) - src, err := ioutil.ReadFile(filepath.Join(tDir, goldenText)) - if err != nil { - t.Fatalf("ReadFile: %v", err) - } - for i, tc := range extendMatchGoldenTestCases { - got := extendMatch(src, tc.i, tc.j) - if got != tc.want { - t.Errorf("test #%d: i, j = %5d, %5d: got %5d (= j + %6d), want %5d (= j + %6d)", - i, tc.i, tc.j, got, got-tc.j, tc.want, tc.want-tc.j) - } - } -} - -func TestExtendMatch(t *testing.T) { - // ref is a simple, reference implementation of extendMatch. - ref := func(src []byte, i, j int) int { - for ; j < len(src) && src[i] == src[j]; i, j = i+1, j+1 { - } - return j - } - - nums := []int{0, 1, 2, 7, 8, 9, 29, 30, 31, 32, 33, 34, 38, 39, 40} - for yIndex := 40; yIndex > 30; yIndex-- { - xxx := bytes.Repeat([]byte("x"), 40) - if yIndex < len(xxx) { - xxx[yIndex] = 'y' - } - for _, i := range nums { - for _, j := range nums { - if i >= j { - continue - } - got := extendMatch(xxx, i, j) - want := ref(xxx, i, j) - if got != want { - t.Errorf("yIndex=%d, i=%d, j=%d: got %d, want %d", yIndex, i, j, got, want) - } - } - } - } -} - -const snappytoolCmdName = "cmd/snappytool/snappytool" - -func skipTestSameEncodingAsCpp() (msg string) { - if !goEncoderShouldMatchCppEncoder { - return fmt.Sprintf("skipping testing that the encoding is byte-for-byte identical to C++: GOARCH=%s", runtime.GOARCH) - } - if _, err := os.Stat(snappytoolCmdName); err != nil { - return fmt.Sprintf("could not find snappytool: %v", err) - } - return "" -} - -func runTestSameEncodingAsCpp(src []byte) error { - got := Encode(nil, src) - - cmd := exec.Command(snappytoolCmdName, "-e") - cmd.Stdin = bytes.NewReader(src) - want, err := cmd.Output() - if err != nil { - return fmt.Errorf("could not run snappytool: %v", err) - } - return cmp(got, want) -} - -func TestSameEncodingAsCppShortCopies(t *testing.T) { - if msg := skipTestSameEncodingAsCpp(); msg != "" { - t.Skip(msg) - } - src := bytes.Repeat([]byte{'a'}, 20) - for i := 0; i <= len(src); i++ { - if err := runTestSameEncodingAsCpp(src[:i]); err != nil { - t.Errorf("i=%d: %v", i, err) - } - } -} - -func TestSameEncodingAsCppLongFiles(t *testing.T) { - if msg := skipTestSameEncodingAsCpp(); msg != "" { - t.Skip(msg) - } - bDir := filepath.FromSlash(*benchdataDir) - failed := false - for i, tf := range testFiles { - if err := downloadBenchmarkFiles(t, tf.filename); err != nil { - t.Fatalf("failed to download testdata: %s", err) - } - data := readFile(t, filepath.Join(bDir, tf.filename)) - if n := tf.sizeLimit; 0 < n && n < len(data) { - data = data[:n] - } - if err := runTestSameEncodingAsCpp(data); err != nil { - t.Errorf("i=%d: %v", i, err) - failed = true - } - } - if failed { - t.Errorf("was the snappytool program built against the C++ snappy library version " + - "d53de187 or later, commited on 2016-04-05? See " + - "https://github.com/google/snappy/commit/d53de18799418e113e44444252a39b12a0e4e0cc") - } -} - -// TestSlowForwardCopyOverrun tests the "expand the pattern" algorithm -// described in decode_amd64.s and its claim of a 10 byte overrun worst case. -func TestSlowForwardCopyOverrun(t *testing.T) { - const base = 100 - - for length := 1; length < 18; length++ { - for offset := 1; offset < 18; offset++ { - highWaterMark := base - d := base - l := length - o := offset - - // makeOffsetAtLeast8 - for o < 8 { - if end := d + 8; highWaterMark < end { - highWaterMark = end - } - l -= o - d += o - o += o - } - - // fixUpSlowForwardCopy - a := d - d += l - - // finishSlowForwardCopy - for l > 0 { - if end := a + 8; highWaterMark < end { - highWaterMark = end - } - a += 8 - l -= 8 - } - - dWant := base + length - overrun := highWaterMark - dWant - if d != dWant || overrun < 0 || 10 < overrun { - t.Errorf("length=%d, offset=%d: d and overrun: got (%d, %d), want (%d, something in [0, 10])", - length, offset, d, overrun, dWant) - } - } - } -} - -// TestEncodeNoiseThenRepeats encodes input for which the first half is very -// incompressible and the second half is very compressible. The encoded form's -// length should be closer to 50% of the original length than 100%. -func TestEncodeNoiseThenRepeats(t *testing.T) { - for _, origLen := range []int{256 * 1024, 2048 * 1024} { - src := make([]byte, origLen) - rng := rand.New(rand.NewSource(1)) - firstHalf, secondHalf := src[:origLen/2], src[origLen/2:] - for i := range firstHalf { - firstHalf[i] = uint8(rng.Intn(256)) - } - for i := range secondHalf { - secondHalf[i] = uint8(i >> 8) - } - dst := Encode(nil, src) - if got, want := len(dst), origLen*3/4; got >= want { - t.Errorf("origLen=%d: got %d encoded bytes, want less than %d", origLen, got, want) - } - } -} - -func TestFramingFormat(t *testing.T) { - // src is comprised of alternating 1e5-sized sequences of random - // (incompressible) bytes and repeated (compressible) bytes. 1e5 was chosen - // because it is larger than maxBlockSize (64k). - src := make([]byte, 1e6) - rng := rand.New(rand.NewSource(1)) - for i := 0; i < 10; i++ { - if i%2 == 0 { - for j := 0; j < 1e5; j++ { - src[1e5*i+j] = uint8(rng.Intn(256)) - } - } else { - for j := 0; j < 1e5; j++ { - src[1e5*i+j] = uint8(i) - } - } - } - - buf := new(bytes.Buffer) - if _, err := NewWriter(buf).Write(src); err != nil { - t.Fatalf("Write: encoding: %v", err) - } - dst, err := ioutil.ReadAll(NewReader(buf)) - if err != nil { - t.Fatalf("ReadAll: decoding: %v", err) - } - if err := cmp(dst, src); err != nil { - t.Fatal(err) - } -} - -func TestWriterGoldenOutput(t *testing.T) { - buf := new(bytes.Buffer) - w := NewBufferedWriter(buf) - defer w.Close() - w.Write([]byte("abcd")) // Not compressible. - w.Flush() - w.Write(bytes.Repeat([]byte{'A'}, 150)) // Compressible. - w.Flush() - // The next chunk is also compressible, but a naive, greedy encoding of the - // overall length 67 copy as a length 64 copy (the longest expressible as a - // tagCopy1 or tagCopy2) plus a length 3 remainder would be two 3-byte - // tagCopy2 tags (6 bytes), since the minimum length for a tagCopy1 is 4 - // bytes. Instead, we could do it shorter, in 5 bytes: a 3-byte tagCopy2 - // (of length 60) and a 2-byte tagCopy1 (of length 7). - w.Write(bytes.Repeat([]byte{'B'}, 68)) - w.Write([]byte("efC")) // Not compressible. - w.Write(bytes.Repeat([]byte{'C'}, 20)) // Compressible. - w.Write(bytes.Repeat([]byte{'B'}, 20)) // Compressible. - w.Write([]byte("g")) // Not compressible. - w.Flush() - - got := buf.String() - want := strings.Join([]string{ - magicChunk, - "\x01\x08\x00\x00", // Uncompressed chunk, 8 bytes long (including 4 byte checksum). - "\x68\x10\xe6\xb6", // Checksum. - "\x61\x62\x63\x64", // Uncompressed payload: "abcd". - "\x00\x11\x00\x00", // Compressed chunk, 17 bytes long (including 4 byte checksum). - "\x5f\xeb\xf2\x10", // Checksum. - "\x96\x01", // Compressed payload: Uncompressed length (varint encoded): 150. - "\x00\x41", // Compressed payload: tagLiteral, length=1, "A". - "\xfe\x01\x00", // Compressed payload: tagCopy2, length=64, offset=1. - "\xfe\x01\x00", // Compressed payload: tagCopy2, length=64, offset=1. - "\x52\x01\x00", // Compressed payload: tagCopy2, length=21, offset=1. - "\x00\x18\x00\x00", // Compressed chunk, 24 bytes long (including 4 byte checksum). - "\x30\x85\x69\xeb", // Checksum. - "\x70", // Compressed payload: Uncompressed length (varint encoded): 112. - "\x00\x42", // Compressed payload: tagLiteral, length=1, "B". - "\xee\x01\x00", // Compressed payload: tagCopy2, length=60, offset=1. - "\x0d\x01", // Compressed payload: tagCopy1, length=7, offset=1. - "\x08\x65\x66\x43", // Compressed payload: tagLiteral, length=3, "efC". - "\x4e\x01\x00", // Compressed payload: tagCopy2, length=20, offset=1. - "\x4e\x5a\x00", // Compressed payload: tagCopy2, length=20, offset=90. - "\x00\x67", // Compressed payload: tagLiteral, length=1, "g". - }, "") - if got != want { - t.Fatalf("\ngot: % x\nwant: % x", got, want) - } -} - -func TestEmitLiteral(t *testing.T) { - testCases := []struct { - length int - want string - }{ - {1, "\x00"}, - {2, "\x04"}, - {59, "\xe8"}, - {60, "\xec"}, - {61, "\xf0\x3c"}, - {62, "\xf0\x3d"}, - {254, "\xf0\xfd"}, - {255, "\xf0\xfe"}, - {256, "\xf0\xff"}, - {257, "\xf4\x00\x01"}, - {65534, "\xf4\xfd\xff"}, - {65535, "\xf4\xfe\xff"}, - {65536, "\xf4\xff\xff"}, - } - - dst := make([]byte, 70000) - nines := bytes.Repeat([]byte{0x99}, 65536) - for _, tc := range testCases { - lit := nines[:tc.length] - n := emitLiteral(dst, lit) - if !bytes.HasSuffix(dst[:n], lit) { - t.Errorf("length=%d: did not end with that many literal bytes", tc.length) - continue - } - got := string(dst[:n-tc.length]) - if got != tc.want { - t.Errorf("length=%d:\ngot % x\nwant % x", tc.length, got, tc.want) - continue - } - } -} - -func TestEmitCopy(t *testing.T) { - testCases := []struct { - offset int - length int - want string - }{ - {8, 04, "\x01\x08"}, - {8, 11, "\x1d\x08"}, - {8, 12, "\x2e\x08\x00"}, - {8, 13, "\x32\x08\x00"}, - {8, 59, "\xea\x08\x00"}, - {8, 60, "\xee\x08\x00"}, - {8, 61, "\xf2\x08\x00"}, - {8, 62, "\xf6\x08\x00"}, - {8, 63, "\xfa\x08\x00"}, - {8, 64, "\xfe\x08\x00"}, - {8, 65, "\xee\x08\x00\x05\x08"}, - {8, 66, "\xee\x08\x00\x09\x08"}, - {8, 67, "\xee\x08\x00\x0d\x08"}, - {8, 68, "\xfe\x08\x00\x01\x08"}, - {8, 69, "\xfe\x08\x00\x05\x08"}, - {8, 80, "\xfe\x08\x00\x3e\x08\x00"}, - - {256, 04, "\x21\x00"}, - {256, 11, "\x3d\x00"}, - {256, 12, "\x2e\x00\x01"}, - {256, 13, "\x32\x00\x01"}, - {256, 59, "\xea\x00\x01"}, - {256, 60, "\xee\x00\x01"}, - {256, 61, "\xf2\x00\x01"}, - {256, 62, "\xf6\x00\x01"}, - {256, 63, "\xfa\x00\x01"}, - {256, 64, "\xfe\x00\x01"}, - {256, 65, "\xee\x00\x01\x25\x00"}, - {256, 66, "\xee\x00\x01\x29\x00"}, - {256, 67, "\xee\x00\x01\x2d\x00"}, - {256, 68, "\xfe\x00\x01\x21\x00"}, - {256, 69, "\xfe\x00\x01\x25\x00"}, - {256, 80, "\xfe\x00\x01\x3e\x00\x01"}, - - {2048, 04, "\x0e\x00\x08"}, - {2048, 11, "\x2a\x00\x08"}, - {2048, 12, "\x2e\x00\x08"}, - {2048, 13, "\x32\x00\x08"}, - {2048, 59, "\xea\x00\x08"}, - {2048, 60, "\xee\x00\x08"}, - {2048, 61, "\xf2\x00\x08"}, - {2048, 62, "\xf6\x00\x08"}, - {2048, 63, "\xfa\x00\x08"}, - {2048, 64, "\xfe\x00\x08"}, - {2048, 65, "\xee\x00\x08\x12\x00\x08"}, - {2048, 66, "\xee\x00\x08\x16\x00\x08"}, - {2048, 67, "\xee\x00\x08\x1a\x00\x08"}, - {2048, 68, "\xfe\x00\x08\x0e\x00\x08"}, - {2048, 69, "\xfe\x00\x08\x12\x00\x08"}, - {2048, 80, "\xfe\x00\x08\x3e\x00\x08"}, - } - - dst := make([]byte, 1024) - for _, tc := range testCases { - n := emitCopy(dst, tc.offset, tc.length) - got := string(dst[:n]) - if got != tc.want { - t.Errorf("offset=%d, length=%d:\ngot % x\nwant % x", tc.offset, tc.length, got, tc.want) - } - } -} - -func TestNewBufferedWriter(t *testing.T) { - // Test all 32 possible sub-sequences of these 5 input slices. - // - // Their lengths sum to 400,000, which is over 6 times the Writer ibuf - // capacity: 6 * maxBlockSize is 393,216. - inputs := [][]byte{ - bytes.Repeat([]byte{'a'}, 40000), - bytes.Repeat([]byte{'b'}, 150000), - bytes.Repeat([]byte{'c'}, 60000), - bytes.Repeat([]byte{'d'}, 120000), - bytes.Repeat([]byte{'e'}, 30000), - } -loop: - for i := 0; i < 1< 0; { - i := copy(x, src) - x = x[i:] - } - return dst -} - -func benchWords(b *testing.B, n int, decode bool) { - // Note: the file is OS-language dependent so the resulting values are not - // directly comparable for non-US-English OS installations. - data := expand(readFile(b, "/usr/share/dict/words"), n) - if decode { - benchDecode(b, data) - } else { - benchEncode(b, data) - } -} - -func BenchmarkWordsDecode1e1(b *testing.B) { benchWords(b, 1e1, true) } -func BenchmarkWordsDecode1e2(b *testing.B) { benchWords(b, 1e2, true) } -func BenchmarkWordsDecode1e3(b *testing.B) { benchWords(b, 1e3, true) } -func BenchmarkWordsDecode1e4(b *testing.B) { benchWords(b, 1e4, true) } -func BenchmarkWordsDecode1e5(b *testing.B) { benchWords(b, 1e5, true) } -func BenchmarkWordsDecode1e6(b *testing.B) { benchWords(b, 1e6, true) } -func BenchmarkWordsEncode1e1(b *testing.B) { benchWords(b, 1e1, false) } -func BenchmarkWordsEncode1e2(b *testing.B) { benchWords(b, 1e2, false) } -func BenchmarkWordsEncode1e3(b *testing.B) { benchWords(b, 1e3, false) } -func BenchmarkWordsEncode1e4(b *testing.B) { benchWords(b, 1e4, false) } -func BenchmarkWordsEncode1e5(b *testing.B) { benchWords(b, 1e5, false) } -func BenchmarkWordsEncode1e6(b *testing.B) { benchWords(b, 1e6, false) } - -func BenchmarkRandomEncode(b *testing.B) { - rng := rand.New(rand.NewSource(1)) - data := make([]byte, 1<<20) - for i := range data { - data[i] = uint8(rng.Intn(256)) - } - benchEncode(b, data) -} - -// testFiles' values are copied directly from -// https://raw.githubusercontent.com/google/snappy/master/snappy_unittest.cc -// The label field is unused in snappy-go. -var testFiles = []struct { - label string - filename string - sizeLimit int -}{ - {"html", "html", 0}, - {"urls", "urls.10K", 0}, - {"jpg", "fireworks.jpeg", 0}, - {"jpg_200", "fireworks.jpeg", 200}, - {"pdf", "paper-100k.pdf", 0}, - {"html4", "html_x_4", 0}, - {"txt1", "alice29.txt", 0}, - {"txt2", "asyoulik.txt", 0}, - {"txt3", "lcet10.txt", 0}, - {"txt4", "plrabn12.txt", 0}, - {"pb", "geo.protodata", 0}, - {"gaviota", "kppkn.gtb", 0}, -} - -const ( - // The benchmark data files are at this canonical URL. - benchURL = "https://raw.githubusercontent.com/google/snappy/master/testdata/" -) - -func downloadBenchmarkFiles(b testing.TB, basename string) (errRet error) { - bDir := filepath.FromSlash(*benchdataDir) - filename := filepath.Join(bDir, basename) - if stat, err := os.Stat(filename); err == nil && stat.Size() != 0 { - return nil - } - - if !*download { - b.Skipf("test data not found; skipping %s without the -download flag", testOrBenchmark(b)) - } - // Download the official snappy C++ implementation reference test data - // files for benchmarking. - if err := os.MkdirAll(bDir, 0777); err != nil && !os.IsExist(err) { - return fmt.Errorf("failed to create %s: %s", bDir, err) - } - - f, err := os.Create(filename) - if err != nil { - return fmt.Errorf("failed to create %s: %s", filename, err) - } - defer f.Close() - defer func() { - if errRet != nil { - os.Remove(filename) - } - }() - url := benchURL + basename - resp, err := http.Get(url) - if err != nil { - return fmt.Errorf("failed to download %s: %s", url, err) - } - defer resp.Body.Close() - if s := resp.StatusCode; s != http.StatusOK { - return fmt.Errorf("downloading %s: HTTP status code %d (%s)", url, s, http.StatusText(s)) - } - _, err = io.Copy(f, resp.Body) - if err != nil { - return fmt.Errorf("failed to download %s to %s: %s", url, filename, err) - } - return nil -} - -func benchFile(b *testing.B, i int, decode bool) { - if err := downloadBenchmarkFiles(b, testFiles[i].filename); err != nil { - b.Fatalf("failed to download testdata: %s", err) - } - bDir := filepath.FromSlash(*benchdataDir) - data := readFile(b, filepath.Join(bDir, testFiles[i].filename)) - if n := testFiles[i].sizeLimit; 0 < n && n < len(data) { - data = data[:n] - } - if decode { - benchDecode(b, data) - } else { - benchEncode(b, data) - } -} - -// Naming convention is kept similar to what snappy's C++ implementation uses. -func Benchmark_UFlat0(b *testing.B) { benchFile(b, 0, true) } -func Benchmark_UFlat1(b *testing.B) { benchFile(b, 1, true) } -func Benchmark_UFlat2(b *testing.B) { benchFile(b, 2, true) } -func Benchmark_UFlat3(b *testing.B) { benchFile(b, 3, true) } -func Benchmark_UFlat4(b *testing.B) { benchFile(b, 4, true) } -func Benchmark_UFlat5(b *testing.B) { benchFile(b, 5, true) } -func Benchmark_UFlat6(b *testing.B) { benchFile(b, 6, true) } -func Benchmark_UFlat7(b *testing.B) { benchFile(b, 7, true) } -func Benchmark_UFlat8(b *testing.B) { benchFile(b, 8, true) } -func Benchmark_UFlat9(b *testing.B) { benchFile(b, 9, true) } -func Benchmark_UFlat10(b *testing.B) { benchFile(b, 10, true) } -func Benchmark_UFlat11(b *testing.B) { benchFile(b, 11, true) } -func Benchmark_ZFlat0(b *testing.B) { benchFile(b, 0, false) } -func Benchmark_ZFlat1(b *testing.B) { benchFile(b, 1, false) } -func Benchmark_ZFlat2(b *testing.B) { benchFile(b, 2, false) } -func Benchmark_ZFlat3(b *testing.B) { benchFile(b, 3, false) } -func Benchmark_ZFlat4(b *testing.B) { benchFile(b, 4, false) } -func Benchmark_ZFlat5(b *testing.B) { benchFile(b, 5, false) } -func Benchmark_ZFlat6(b *testing.B) { benchFile(b, 6, false) } -func Benchmark_ZFlat7(b *testing.B) { benchFile(b, 7, false) } -func Benchmark_ZFlat8(b *testing.B) { benchFile(b, 8, false) } -func Benchmark_ZFlat9(b *testing.B) { benchFile(b, 9, false) } -func Benchmark_ZFlat10(b *testing.B) { benchFile(b, 10, false) } -func Benchmark_ZFlat11(b *testing.B) { benchFile(b, 11, false) } - -func BenchmarkExtendMatch(b *testing.B) { - tDir := filepath.FromSlash(*testdataDir) - src, err := ioutil.ReadFile(filepath.Join(tDir, goldenText)) - if err != nil { - b.Fatalf("ReadFile: %v", err) - } - b.ResetTimer() - for i := 0; i < b.N; i++ { - for _, tc := range extendMatchGoldenTestCases { - extendMatch(src, tc.i, tc.j) - } - } -} diff --git a/vendor/github.com/golang/snappy/testdata/Mark.Twain-Tom.Sawyer.txt b/vendor/github.com/golang/snappy/testdata/Mark.Twain-Tom.Sawyer.txt deleted file mode 100644 index 86a1875..0000000 --- a/vendor/github.com/golang/snappy/testdata/Mark.Twain-Tom.Sawyer.txt +++ /dev/null @@ -1,396 +0,0 @@ -Produced by David Widger. The previous edition was updated by Jose -Menendez. - - - - - - THE ADVENTURES OF TOM SAWYER - BY - MARK TWAIN - (Samuel Langhorne Clemens) - - - - - P R E F A C E - -MOST of the adventures recorded in this book really occurred; one or -two were experiences of my own, the rest those of boys who were -schoolmates of mine. Huck Finn is drawn from life; Tom Sawyer also, but -not from an individual--he is a combination of the characteristics of -three boys whom I knew, and therefore belongs to the composite order of -architecture. - -The odd superstitions touched upon were all prevalent among children -and slaves in the West at the period of this story--that is to say, -thirty or forty years ago. - -Although my book is intended mainly for the entertainment of boys and -girls, I hope it will not be shunned by men and women on that account, -for part of my plan has been to try to pleasantly remind adults of what -they once were themselves, and of how they felt and thought and talked, -and what queer enterprises they sometimes engaged in. - - THE AUTHOR. - -HARTFORD, 1876. - - - - T O M S A W Y E R - - - -CHAPTER I - -"TOM!" - -No answer. - -"TOM!" - -No answer. - -"What's gone with that boy, I wonder? You TOM!" - -No answer. - -The old lady pulled her spectacles down and looked over them about the -room; then she put them up and looked out under them. She seldom or -never looked THROUGH them for so small a thing as a boy; they were her -state pair, the pride of her heart, and were built for "style," not -service--she could have seen through a pair of stove-lids just as well. -She looked perplexed for a moment, and then said, not fiercely, but -still loud enough for the furniture to hear: - -"Well, I lay if I get hold of you I'll--" - -She did not finish, for by this time she was bending down and punching -under the bed with the broom, and so she needed breath to punctuate the -punches with. She resurrected nothing but the cat. - -"I never did see the beat of that boy!" - -She went to the open door and stood in it and looked out among the -tomato vines and "jimpson" weeds that constituted the garden. No Tom. -So she lifted up her voice at an angle calculated for distance and -shouted: - -"Y-o-u-u TOM!" - -There was a slight noise behind her and she turned just in time to -seize a small boy by the slack of his roundabout and arrest his flight. - -"There! I might 'a' thought of that closet. What you been doing in -there?" - -"Nothing." - -"Nothing! Look at your hands. And look at your mouth. What IS that -truck?" - -"I don't know, aunt." - -"Well, I know. It's jam--that's what it is. Forty times I've said if -you didn't let that jam alone I'd skin you. Hand me that switch." - -The switch hovered in the air--the peril was desperate-- - -"My! Look behind you, aunt!" - -The old lady whirled round, and snatched her skirts out of danger. The -lad fled on the instant, scrambled up the high board-fence, and -disappeared over it. - -His aunt Polly stood surprised a moment, and then broke into a gentle -laugh. - -"Hang the boy, can't I never learn anything? Ain't he played me tricks -enough like that for me to be looking out for him by this time? But old -fools is the biggest fools there is. Can't learn an old dog new tricks, -as the saying is. But my goodness, he never plays them alike, two days, -and how is a body to know what's coming? He 'pears to know just how -long he can torment me before I get my dander up, and he knows if he -can make out to put me off for a minute or make me laugh, it's all down -again and I can't hit him a lick. I ain't doing my duty by that boy, -and that's the Lord's truth, goodness knows. Spare the rod and spile -the child, as the Good Book says. I'm a laying up sin and suffering for -us both, I know. He's full of the Old Scratch, but laws-a-me! he's my -own dead sister's boy, poor thing, and I ain't got the heart to lash -him, somehow. Every time I let him off, my conscience does hurt me so, -and every time I hit him my old heart most breaks. Well-a-well, man -that is born of woman is of few days and full of trouble, as the -Scripture says, and I reckon it's so. He'll play hookey this evening, * -and [* Southwestern for "afternoon"] I'll just be obleeged to make him -work, to-morrow, to punish him. It's mighty hard to make him work -Saturdays, when all the boys is having holiday, but he hates work more -than he hates anything else, and I've GOT to do some of my duty by him, -or I'll be the ruination of the child." - -Tom did play hookey, and he had a very good time. He got back home -barely in season to help Jim, the small colored boy, saw next-day's -wood and split the kindlings before supper--at least he was there in -time to tell his adventures to Jim while Jim did three-fourths of the -work. Tom's younger brother (or rather half-brother) Sid was already -through with his part of the work (picking up chips), for he was a -quiet boy, and had no adventurous, troublesome ways. - -While Tom was eating his supper, and stealing sugar as opportunity -offered, Aunt Polly asked him questions that were full of guile, and -very deep--for she wanted to trap him into damaging revealments. Like -many other simple-hearted souls, it was her pet vanity to believe she -was endowed with a talent for dark and mysterious diplomacy, and she -loved to contemplate her most transparent devices as marvels of low -cunning. Said she: - -"Tom, it was middling warm in school, warn't it?" - -"Yes'm." - -"Powerful warm, warn't it?" - -"Yes'm." - -"Didn't you want to go in a-swimming, Tom?" - -A bit of a scare shot through Tom--a touch of uncomfortable suspicion. -He searched Aunt Polly's face, but it told him nothing. So he said: - -"No'm--well, not very much." - -The old lady reached out her hand and felt Tom's shirt, and said: - -"But you ain't too warm now, though." And it flattered her to reflect -that she had discovered that the shirt was dry without anybody knowing -that that was what she had in her mind. But in spite of her, Tom knew -where the wind lay, now. So he forestalled what might be the next move: - -"Some of us pumped on our heads--mine's damp yet. See?" - -Aunt Polly was vexed to think she had overlooked that bit of -circumstantial evidence, and missed a trick. Then she had a new -inspiration: - -"Tom, you didn't have to undo your shirt collar where I sewed it, to -pump on your head, did you? Unbutton your jacket!" - -The trouble vanished out of Tom's face. He opened his jacket. His -shirt collar was securely sewed. - -"Bother! Well, go 'long with you. I'd made sure you'd played hookey -and been a-swimming. But I forgive ye, Tom. I reckon you're a kind of a -singed cat, as the saying is--better'n you look. THIS time." - -She was half sorry her sagacity had miscarried, and half glad that Tom -had stumbled into obedient conduct for once. - -But Sidney said: - -"Well, now, if I didn't think you sewed his collar with white thread, -but it's black." - -"Why, I did sew it with white! Tom!" - -But Tom did not wait for the rest. As he went out at the door he said: - -"Siddy, I'll lick you for that." - -In a safe place Tom examined two large needles which were thrust into -the lapels of his jacket, and had thread bound about them--one needle -carried white thread and the other black. He said: - -"She'd never noticed if it hadn't been for Sid. Confound it! sometimes -she sews it with white, and sometimes she sews it with black. I wish to -geeminy she'd stick to one or t'other--I can't keep the run of 'em. But -I bet you I'll lam Sid for that. I'll learn him!" - -He was not the Model Boy of the village. He knew the model boy very -well though--and loathed him. - -Within two minutes, or even less, he had forgotten all his troubles. -Not because his troubles were one whit less heavy and bitter to him -than a man's are to a man, but because a new and powerful interest bore -them down and drove them out of his mind for the time--just as men's -misfortunes are forgotten in the excitement of new enterprises. This -new interest was a valued novelty in whistling, which he had just -acquired from a negro, and he was suffering to practise it undisturbed. -It consisted in a peculiar bird-like turn, a sort of liquid warble, -produced by touching the tongue to the roof of the mouth at short -intervals in the midst of the music--the reader probably remembers how -to do it, if he has ever been a boy. Diligence and attention soon gave -him the knack of it, and he strode down the street with his mouth full -of harmony and his soul full of gratitude. He felt much as an -astronomer feels who has discovered a new planet--no doubt, as far as -strong, deep, unalloyed pleasure is concerned, the advantage was with -the boy, not the astronomer. - -The summer evenings were long. It was not dark, yet. Presently Tom -checked his whistle. A stranger was before him--a boy a shade larger -than himself. A new-comer of any age or either sex was an impressive -curiosity in the poor little shabby village of St. Petersburg. This boy -was well dressed, too--well dressed on a week-day. This was simply -astounding. His cap was a dainty thing, his close-buttoned blue cloth -roundabout was new and natty, and so were his pantaloons. He had shoes -on--and it was only Friday. He even wore a necktie, a bright bit of -ribbon. He had a citified air about him that ate into Tom's vitals. The -more Tom stared at the splendid marvel, the higher he turned up his -nose at his finery and the shabbier and shabbier his own outfit seemed -to him to grow. Neither boy spoke. If one moved, the other moved--but -only sidewise, in a circle; they kept face to face and eye to eye all -the time. Finally Tom said: - -"I can lick you!" - -"I'd like to see you try it." - -"Well, I can do it." - -"No you can't, either." - -"Yes I can." - -"No you can't." - -"I can." - -"You can't." - -"Can!" - -"Can't!" - -An uncomfortable pause. Then Tom said: - -"What's your name?" - -"'Tisn't any of your business, maybe." - -"Well I 'low I'll MAKE it my business." - -"Well why don't you?" - -"If you say much, I will." - -"Much--much--MUCH. There now." - -"Oh, you think you're mighty smart, DON'T you? I could lick you with -one hand tied behind me, if I wanted to." - -"Well why don't you DO it? You SAY you can do it." - -"Well I WILL, if you fool with me." - -"Oh yes--I've seen whole families in the same fix." - -"Smarty! You think you're SOME, now, DON'T you? Oh, what a hat!" - -"You can lump that hat if you don't like it. I dare you to knock it -off--and anybody that'll take a dare will suck eggs." - -"You're a liar!" - -"You're another." - -"You're a fighting liar and dasn't take it up." - -"Aw--take a walk!" - -"Say--if you give me much more of your sass I'll take and bounce a -rock off'n your head." - -"Oh, of COURSE you will." - -"Well I WILL." - -"Well why don't you DO it then? What do you keep SAYING you will for? -Why don't you DO it? It's because you're afraid." - -"I AIN'T afraid." - -"You are." - -"I ain't." - -"You are." - -Another pause, and more eying and sidling around each other. Presently -they were shoulder to shoulder. Tom said: - -"Get away from here!" - -"Go away yourself!" - -"I won't." - -"I won't either." - -So they stood, each with a foot placed at an angle as a brace, and -both shoving with might and main, and glowering at each other with -hate. But neither could get an advantage. After struggling till both -were hot and flushed, each relaxed his strain with watchful caution, -and Tom said: - -"You're a coward and a pup. I'll tell my big brother on you, and he -can thrash you with his little finger, and I'll make him do it, too." - -"What do I care for your big brother? I've got a brother that's bigger -than he is--and what's more, he can throw him over that fence, too." -[Both brothers were imaginary.] - -"That's a lie." - -"YOUR saying so don't make it so." - -Tom drew a line in the dust with his big toe, and said: - -"I dare you to step over that, and I'll lick you till you can't stand -up. Anybody that'll take a dare will steal sheep." - -The new boy stepped over promptly, and said: - -"Now you said you'd do it, now let's see you do it." - -"Don't you crowd me now; you better look out." - -"Well, you SAID you'd do it--why don't you do it?" - -"By jingo! for two cents I WILL do it." - -The new boy took two broad coppers out of his pocket and held them out -with derision. Tom struck them to the ground. In an instant both boys -were rolling and tumbling in the dirt, gripped together like cats; and -for the space of a minute they tugged and tore at each other's hair and -clothes, punched and scratched each other's nose, and covered -themselves with dust and glory. Presently the confusion took form, and -through the fog of battle Tom appeared, seated astride the new boy, and -pounding him with his fists. "Holler 'nuff!" said he. - -The boy only struggled to free himself. He was crying--mainly from rage. - -"Holler 'nuff!"--and the pounding went on. - -At last the stranger got out a smothered "'Nuff!" and Tom let him up -and said: - -"Now that'll learn you. Better look out who you're fooling with next -time." - -The new boy went off brushing the dust from his clothes, sobbing, -snuffling, and occasionally looking back and shaking his head and -threatening what he would do to Tom the "next time he caught him out." -To which Tom responded with jeers, and started off in high feather, and -as soon as his back was turned the new boy snatched up a stone, threw -it and hit him between the shoulders and then turned tail and ran like -an antelope. Tom chased the traitor home, and thus found out where he -lived. He then held a position at the gate for some time, daring the -enemy to come outside, but the enemy only made faces at him through the -window and declined. At last the enemy's mother appeared, and called -Tom a bad, vicious, vulgar child, and ordered him away. So he went -away; but he said he "'lowed" to "lay" for that boy. - -He got home pretty late that night, and when he climbed cautiously in -at the window, he uncovered an ambuscade, in the person of his aunt; -and when she saw the state his clothes were in her resolution to turn -his Saturday holiday into captivity at hard labor became adamantine in -its firmness. diff --git a/vendor/github.com/golang/snappy/testdata/Mark.Twain-Tom.Sawyer.txt.rawsnappy b/vendor/github.com/golang/snappy/testdata/Mark.Twain-Tom.Sawyer.txt.rawsnappy deleted file mode 100644 index 9c56d985888e48a9967e187523f0e3326330aeca..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 9871 zcmW++d3+ni*`6u8u)A7~rM0|~WyP6QqBu?@lY=8Ar;;dGG#9aRhm?^tk~Wrh#qP>N zQ1~GlQvw7CE&Q640B$iMX|9tMwyz{)z z`#jI+Pu3f296Mjj@jT5o=rT5F=II7AU*t{??Jsd!b@-rZ*Idf;rf1p~tuvR_s(I#d zarWGEY?mu5xy7wKzoT}^s4@KYtwyn^>W(3dL`{kZP=7vyJ{wy zjghcqlQS7jS(#e zmZs@)nxac-T2WT6?(3&^fqJk`mLKGnS97>a9iFqDJZ#9c;8&(gv$j0|KV{`|gW9=V z&2e~s9<|~5wxf7;o7TC*DZjAF9g|x*cmzDO0)Jb#690QRJk^6QV6)f`XYGbd>m~Wj zDT<_YJpY-0H!a>nduzD?y4AHM2i^#*q+E4y}r zW^0~e*`7RQF@yiG@+MTj#>&ZLiQ{Ec7|q$0tZ6aN_|*UD)?A4>Ea$-*UC+#ouTk+z z4(Dw@o>`$8I$-g7f-);_bS-vx!G;}WdD*s#F7<5OhPQ;PwrhHv8gr3@d!x+BmEhZ) zu~zC)l+;Dj?0lZOHK>8>r7yyZ617|jYFMj|pSiJ2g6gGzyoQyb%9K^$xfK4toOigT z;2mMN%m$%u@x7GO4dc~fo;YM&BG`HPwbP&$u4g-qWYR0aX3z@HrY>tFVEv}!L7g31 zu(7mcktmzOUc|f?epI z^F!FaQZ${in}CXx>?)_GN9$&}OrgwB59*d`mc_X#0j%&$iPmkbNU%fkuEi{uvul=@ zP@s0S${Y`GoewXsmf>tl0QxM4e|cW=oCZE+W3|kg%PbEN+Ga;}X>V#xe|jy=C`t_a(ugU_DjSgFLL2mK z4x#7<*g=;|i|iw{ZhEEe{k?$s1cj2;ZD@375#40hhVjC+D1G9xa-No19_mypmjN#& z0JmG^IgjPau;RR3&q!0s(17qedyuEIp$cVLyOw?}q&T)+nP(Iwi@L*4$rIEHpe=q+ z@GQ0F(Hg!EPoG)op?%QYP`$ieY3>&+jyL&dg!_Bfu3mR`uh^Y_Dc}N@D(n)(n#aQc zN;oKn-x2Gmk(qxQEJkrXpei^|)7kmw0Ms& zo`^9CI-5I#W{xM5_&Mmp8f>z}2Jzjg*92&}9f*k>Z=MOD9^}cgnRn^-8v92SXn~i@ zJqjN4AED4l0Fn!NUbG|zs9op3TJTE7T$P#m1io|%EtuTN@p2EhWS)LbhRK1t>}){`}ZwY8IH<7+mLvV9nGBvc;u>S2GT(0A)oySKzL=B-0TES{&!n+}OV)XfJFT8tzeP zj(IE~WN79L{2R&TTWWiOi)}Y6RoAv+P-LEO@(}ajWUSEzsD>XfZeLbp!1-1W z1tFRB_$ZJEN@c^b;X{&J zvhlZ#4?~-hcCwbNy(;vVMt^MFGm}xbY!*u%@QCRednG8H`6_I{AqwP@hMbNLYRo!2so{4J6pevxa zjML`=1_0B*e{K&=S^4<_FLm|a;U^s~zr)8k#R+k3@!zN242`Ep@ zU9b#Kka}olc$>PNRgy>VLS3X?7MPcs5isG2%a7h}EL2@US#xq201OaR*uXf_@HVN; zJ+Xn)5)8HBJENH{7$XCwg&%_l_8Mv9GQ26|0*~ZM_{!m%Wg^hhaP9DY>GcxyB)%5$ z=6b;_$B)m?b07`}x;2@E9jADog<@HA1}vb#o4@WN`!_A>K2;1SbmO z0DzM$jL&_sNELtrt5!L~f5NW|)J)BldQDe6Akj58c}_8ZB)GNrev9&}z9~X;*aOhW zfKdkB2(tj+EWyM|U7_}ZfDv~b4G1}AT)+Et6^4bwENtgp1)DEbjJZs?IW}5(G>V~luD6joYPJOOrWTDF?bA3?D;v(A1 zX_s*q(R{LjwvWE2!c0?0$%if2GWlFaj%F#S(sZ>eea~DbR2r$XLNGYS`TZ+A3zv{P zdbN`@wFkKOVg@pkNe~@NuVMS&d zFG+ct!Mg(h=cyKwi60X597F0bK*w? z*Y5r{d0%c@FzY99H0ck?Z>5k&E4J%t2}vD*Ee`LNU}1keAnsS0E*z+@iO_? zPgckQ(98oo5|O8HknDh_1f8Bvs9UDfzR{vk=f+lK8NZlGW2^v0uxz0G^E6jBXGKB3 z+1WDZtwaUT2q7C?^*=YB~K`+zEH`ALC*9o3L8Qt?T;;Zi3%lgQokTi`w{nt5xYC8 zQeJkA`GS>y=BRY`>VAAVFmt-aXkL;@bxDft$e*KZC5mcGUTu*;93MU^*p;zqL;I7w z`&w~m_@C)g^VM|4xG{Yj%987)X(f<$N|rew1IW{o>Vl{oy*6rtI4M`@IjNr+qW-+3 z=IpXPS~={S@3J~H^@Uy%TC~em>TDl=Lv^cVGxJplC^ug=NjiYCJD;5bLLRs@nPlGZ zEy#^8CKdVaM+8fO(=fuS2U{m3f)@)<(C|~ zB?Ow+wQD3DBnyM^er>uOs)B4AWcW#(3xPc|`Jk5aBL&9-Ak6G~r{M>&=;@0!{3A8b zoPl^2h`}YtH1cIAnm4Ou5JkC)8g+p@%b-5_zrrAbJYK1iPF1{cWO~4JK*^6@9R-_~ z2kV|ABk=g2Cq=4ksXTQR0>@PEz=brY4aIZ7zWT3K0nNo zF!L*wkyYbQ#@o~s&6-+8Q9(%M#?BMm(&6heN22gMz$o>|p4Mg~50ccb0Ev1Gx*U!a zh&Tt4Lyyu+?r9-MTLqp!FEchDF><*gfp<0@1F*`z)COcla;iZ4nhOvmzxEju1FP(= zO7gh|iM6IT2$kB@VIN+sWqc|?Gl$FMWQBawM{ZE#uV@cyLch&kKrZ=9Yvm+?nIa<( z$;JfzCP@NOY77V!s_j*g<_hvEy&d+1oc1_G3e&tB_@b||# zYcRc6>a#22`C*7{sx^SdcQnDG+DHBnG^U$OlB;xRX7+?sr45oHLLb*=l6`aWGiXq& z2iXY~)a^jqOU4xPh>xwu?`H;AC^^%~)hh5Z%QIP-0%C*75Yo<-wUn%#XxXD=7|Nukhl}PQrwyLoMjrJ=hUIaM z+U1b_fw9~1ivgy%aDMA#kV5SI(J1?sw#64Rq3&RVz6mO1s&=m*me${!>C@Lr5w#V+ zHMR~Y4CYrO^@lJ-BSn@oHBXNJCy)cDV2_PNNwkC1QxUqAK`*()_a z5bCVyAzM4xMs~FZz^$ZQjY{~)P-^EkEz=smVCTy!tRRo8WPy(i`o)Lk3}RX~C}gbR zze=pY$qA?u7Y8MNA!EM>sd+0203wQ9n;jyT>jBXExzc;bBnY}lgBV-2oF9a9^|?1T z9Xh~_YsmRBESYA>J=)ZRdH}e@wu0M`l6)C+iQkug34by^zk29W+bW1VB-1mrIX*I75nN>$_=>|5$c}cf@bV7% z5b2l2sYCb9o{Id}B_I|7uMJF~6gk<{jv9D6*{G@Tf`R78_^9zX>Jpz8lF8227HNQU z{f9xpsc8%RQWu1qZb-?1|Muc{cRwPq3O*$MI`tNNK`58We}G)AvC~qgp` zE>BMnMl$P->1>&}hBS#V*hrrwWX;2XW(g0lW^ zA3jA~gsPnrWtgDXbdV2%D68lF0iN_h;4HaV}dus{Sp|S9<5{aVF|h(>uooK0AMg} z%Qa4p`BWHyp0>_E4l36p2YqBSVLKyx{rLKj!g8QM$PJSA)kOTIRCMgjlc<#GnEFD1 zyiCaR6UhyF0Ei#n{)!&0InZ%EN=BbIf!q?%u9`5PLEHmrg2@~hb6_XQ!O%-^6H!7{GO?3%NPwquNCM|hZVaP@_<0m=0boF}VrKdOlFN~Q zYLB&y-eP>OeP86sNs?Qu;GZ_5+QTi{(GKzhi7XAK-^BBX<3|a7wh~waffm>Hb&fru zLiU{_0mYb-IiW(XpEAzV{-uklh(92dOzldwd3apOl*rrC;Rlhkvnvc%1X9WZUKjun zCUfx=3?$a&*?81nfom1lg&_i3%rVK&D8C%Puie$EmQ85F zcho?MWwVW+ZN(c->jPe0!@&u;*_u-{|AueYE+!-AqdeYB9{Ha{(T}LXRp(5syGQ6A`gkVr-fRLzZj=$FSetkeuDx+nP#C+ za0s|o(asO+b0T=FVDST!RY+O9&Z7W@?t5@Asv2L0AkVejp0WDy-$00huT*U7naRlP zejZ|r<5XDgke4*E+xRzmJwT!g9BgjDGsjj-H0x+PB9S+JA;-*SZA+YqHvw%x1Dget z{K+SnPNYjBtz`655OTLhi6y0I6ewlT^jLXhkpwE6m@zU7#Db?C4o8SAf&C%LiR9l3 zxuhlYJVYf$@_>K%IaEu%O$LJ_mkYM5ebIqGlL|1D``X67VH}UYBbYq$lc@0t_Tz5$ zSqO5n0tk=KqHrycZ#v?A)TY1zVBlA+JTG-C1lr(O!T1*rze~3ID_L!VBnMH_cmOH! zRUsEL<+@2eZY9_2u2P~|O5`kA8M*JNI-at0~z1Eb-f*4>Yp+rJHc`lgEppI5)3;p`h2||L( zA8YR`XcM`r1BXS@(}l!fdk4`BEf^8Q(o$xLyR=K&u_C0bv$h7HX%1MkM#SsYg8 z460yWC|(=x0~_Tv4Dma&zk$`h)hpAH9sY^`HIgvL;3T6HQpUL{m6mH2vc?By0CGlZ zcHf|H>P){fE^HqmC{ON?@gd}COFAfA6`Gy`G0o23cU~QAdW6H|g~K1tLUH0pC^#Va z6J9K42-?(%TS!a}?lKO+n|`AqUQvQ^RA_!p{4(z0_X~NJzC|pMt7TM;x2manGAXt~ zl2(VSC2?b*kHPJm|79pI6fC`6J2gCdv36rXb(!mCo{$$KORxFKAE^YFGTJw(ROqtg z1Q`v8w$V3!8@hD$y0v|292Jj=9Scsh2hN%Ijp1&=TBN5a%J0;)zo{W8bY|7rZT+Ov zCoNJo^ejB-+>23$J=*z*uXAHSEjY}SZ&x!km0>4MY)#!QFeh^n%ILpr6>pyyMyc*A zcFxfskPjyz;@TZBIt2h=sSUBCVQ8xPp##^Wo~hrLhR=z-I)!{{pfYik;mN4Pu7qr} zX9ff+D05RAxnzQxwY?I401c*U;{2fvII8WR z*4-K?f}!Txdr>%Ux%kWtcovASJrV6Dh`Fr;e~nxjCofEqfbwIVN*LzIc%(&cmmw!E zlKm>8+N?PKE>g7r%4DH#xKg29BVw!nMKG#mMy`#ItqH-n1;U1b46#-3L~=xzK#`H_ zefqu8p6$4{A&|4dt?P+E08~h=8lf%P*X_fH1hW|XWxIY`t9&Bzac2k=G~|WG9=U;c zM&6K7fjp~4X0&TpwZz`X5wuiS{0zecfm10>ElE8Wh~q&}wf*7lE90C0@O=98;ajyRoiKC+x_gUQD0RJ&M~;D5DW=*DSbkWrPYSMK~si3DV6 zm0cI>7!115l{b!%`2?=H>SW_2a&;%-dYc^odZ>(+#S`K^Al&Nt;Tu)>{X9|AKL`P2 zU(?A{MVvXj=DiQqzEoxz?h2Mb)1I+G6!Y2^i7si-?J$~cBfXKF9t-5)2EqOCTDcpa z*Yi&VSbUx0889$yFgQ88N>(ND_lA$9-V&-d`LWJ-LwsJ=?oXl$rZ9zG0A@Ls1U92V?p;7Lcu1xJMa*DRDf576QoN+MdKZ{Xl0wQO55_b@O7-ebv<#n2c4HNt*rdMVrxA^Z57Ycxg znj5=ci@z`a$H=ccTXpShmQ-2<#Wjj}Tl`0GubP8vr!O_z&%VGHYB2-<5kfvD2JyGZ zr7~_y>Ej8i_DpQ#`0z1s{+lKCXBbHTAn~mzh9B0B_;C#7$Y5Mw8yD;v`!D2UWO`7@ zmE!Y&mfR|-_C%zFx*a~I5ZOAJOm0b?hu-`#W&p`4RGkMpjNDuTCbP zXrwz3Ikm-jXV&c4ZPUoM98(TnH9m35bGj7MaakS-vcqYIE=-E%xa!Rw$9Q&olHKSOfH?H zdhkIS4H@EOaC5J#^2OShQvlzd2G`Q$x*q+oEYw^L6sHmy4tD#r*Ceu#>N6%8pM)!* z^>Qpv9w&N98hwnood`ayL>A9L^M*oVQG2`{G5y6bNN(fl)I*iK^xtWzV*=dNP7v&} zwo1WcsX<8WW^~>k2g1KGu9vhBcn!}SB( -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above copyright -notice, this list of conditions and the following disclaimer in the -documentation and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/syndtr/goleveldb/README.md b/vendor/github.com/syndtr/goleveldb/README.md deleted file mode 100644 index 259286f..0000000 --- a/vendor/github.com/syndtr/goleveldb/README.md +++ /dev/null @@ -1,105 +0,0 @@ -This is an implementation of the [LevelDB key/value database](http:code.google.com/p/leveldb) in the [Go programming language](http:golang.org). - -[![Build Status](https://travis-ci.org/syndtr/goleveldb.png?branch=master)](https://travis-ci.org/syndtr/goleveldb) - -Installation ------------ - - go get github.com/syndtr/goleveldb/leveldb - -Requirements ------------ - -* Need at least `go1.4` or newer. - -Usage ------------ - -Create or open a database: -```go -db, err := leveldb.OpenFile("path/to/db", nil) -... -defer db.Close() -... -``` -Read or modify the database content: -```go -// Remember that the contents of the returned slice should not be modified. -data, err := db.Get([]byte("key"), nil) -... -err = db.Put([]byte("key"), []byte("value"), nil) -... -err = db.Delete([]byte("key"), nil) -... -``` - -Iterate over database content: -```go -iter := db.NewIterator(nil, nil) -for iter.Next() { - // Remember that the contents of the returned slice should not be modified, and - // only valid until the next call to Next. - key := iter.Key() - value := iter.Value() - ... -} -iter.Release() -err = iter.Error() -... -``` -Seek-then-Iterate: -```go -iter := db.NewIterator(nil, nil) -for ok := iter.Seek(key); ok; ok = iter.Next() { - // Use key/value. - ... -} -iter.Release() -err = iter.Error() -... -``` -Iterate over subset of database content: -```go -iter := db.NewIterator(&util.Range{Start: []byte("foo"), Limit: []byte("xoo")}, nil) -for iter.Next() { - // Use key/value. - ... -} -iter.Release() -err = iter.Error() -... -``` -Iterate over subset of database content with a particular prefix: -```go -iter := db.NewIterator(util.BytesPrefix([]byte("foo-")), nil) -for iter.Next() { - // Use key/value. - ... -} -iter.Release() -err = iter.Error() -... -``` -Batch writes: -```go -batch := new(leveldb.Batch) -batch.Put([]byte("foo"), []byte("value")) -batch.Put([]byte("bar"), []byte("another value")) -batch.Delete([]byte("baz")) -err = db.Write(batch, nil) -... -``` -Use bloom filter: -```go -o := &opt.Options{ - Filter: filter.NewBloomFilter(10), -} -db, err := leveldb.OpenFile("path/to/db", o) -... -defer db.Close() -... -``` -Documentation ------------ - -You can read package documentation [here](http:godoc.org/github.com/syndtr/goleveldb). diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/batch.go b/vendor/github.com/syndtr/goleveldb/leveldb/batch.go deleted file mode 100644 index 2259200..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/batch.go +++ /dev/null @@ -1,349 +0,0 @@ -// Copyright (c) 2012, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package leveldb - -import ( - "encoding/binary" - "fmt" - "io" - - "github.com/syndtr/goleveldb/leveldb/errors" - "github.com/syndtr/goleveldb/leveldb/memdb" - "github.com/syndtr/goleveldb/leveldb/storage" -) - -// ErrBatchCorrupted records reason of batch corruption. This error will be -// wrapped with errors.ErrCorrupted. -type ErrBatchCorrupted struct { - Reason string -} - -func (e *ErrBatchCorrupted) Error() string { - return fmt.Sprintf("leveldb: batch corrupted: %s", e.Reason) -} - -func newErrBatchCorrupted(reason string) error { - return errors.NewErrCorrupted(storage.FileDesc{}, &ErrBatchCorrupted{reason}) -} - -const ( - batchHeaderLen = 8 + 4 - batchGrowRec = 3000 - batchBufioSize = 16 -) - -// BatchReplay wraps basic batch operations. -type BatchReplay interface { - Put(key, value []byte) - Delete(key []byte) -} - -type batchIndex struct { - keyType keyType - keyPos, keyLen int - valuePos, valueLen int -} - -func (index batchIndex) k(data []byte) []byte { - return data[index.keyPos : index.keyPos+index.keyLen] -} - -func (index batchIndex) v(data []byte) []byte { - if index.valueLen != 0 { - return data[index.valuePos : index.valuePos+index.valueLen] - } - return nil -} - -func (index batchIndex) kv(data []byte) (key, value []byte) { - return index.k(data), index.v(data) -} - -// Batch is a write batch. -type Batch struct { - data []byte - index []batchIndex - - // internalLen is sums of key/value pair length plus 8-bytes internal key. - internalLen int -} - -func (b *Batch) grow(n int) { - o := len(b.data) - if cap(b.data)-o < n { - div := 1 - if len(b.index) > batchGrowRec { - div = len(b.index) / batchGrowRec - } - ndata := make([]byte, o, o+n+o/div) - copy(ndata, b.data) - b.data = ndata - } -} - -func (b *Batch) appendRec(kt keyType, key, value []byte) { - n := 1 + binary.MaxVarintLen32 + len(key) - if kt == keyTypeVal { - n += binary.MaxVarintLen32 + len(value) - } - b.grow(n) - index := batchIndex{keyType: kt} - o := len(b.data) - data := b.data[:o+n] - data[o] = byte(kt) - o++ - o += binary.PutUvarint(data[o:], uint64(len(key))) - index.keyPos = o - index.keyLen = len(key) - o += copy(data[o:], key) - if kt == keyTypeVal { - o += binary.PutUvarint(data[o:], uint64(len(value))) - index.valuePos = o - index.valueLen = len(value) - o += copy(data[o:], value) - } - b.data = data[:o] - b.index = append(b.index, index) - b.internalLen += index.keyLen + index.valueLen + 8 -} - -// Put appends 'put operation' of the given key/value pair to the batch. -// It is safe to modify the contents of the argument after Put returns but not -// before. -func (b *Batch) Put(key, value []byte) { - b.appendRec(keyTypeVal, key, value) -} - -// Delete appends 'delete operation' of the given key to the batch. -// It is safe to modify the contents of the argument after Delete returns but -// not before. -func (b *Batch) Delete(key []byte) { - b.appendRec(keyTypeDel, key, nil) -} - -// Dump dumps batch contents. The returned slice can be loaded into the -// batch using Load method. -// The returned slice is not its own copy, so the contents should not be -// modified. -func (b *Batch) Dump() []byte { - return b.data -} - -// Load loads given slice into the batch. Previous contents of the batch -// will be discarded. -// The given slice will not be copied and will be used as batch buffer, so -// it is not safe to modify the contents of the slice. -func (b *Batch) Load(data []byte) error { - return b.decode(data, -1) -} - -// Replay replays batch contents. -func (b *Batch) Replay(r BatchReplay) error { - for _, index := range b.index { - switch index.keyType { - case keyTypeVal: - r.Put(index.k(b.data), index.v(b.data)) - case keyTypeDel: - r.Delete(index.k(b.data)) - } - } - return nil -} - -// Len returns number of records in the batch. -func (b *Batch) Len() int { - return len(b.index) -} - -// Reset resets the batch. -func (b *Batch) Reset() { - b.data = b.data[:0] - b.index = b.index[:0] - b.internalLen = 0 -} - -func (b *Batch) replayInternal(fn func(i int, kt keyType, k, v []byte) error) error { - for i, index := range b.index { - if err := fn(i, index.keyType, index.k(b.data), index.v(b.data)); err != nil { - return err - } - } - return nil -} - -func (b *Batch) append(p *Batch) { - ob := len(b.data) - oi := len(b.index) - b.data = append(b.data, p.data...) - b.index = append(b.index, p.index...) - b.internalLen += p.internalLen - - // Updating index offset. - if ob != 0 { - for ; oi < len(b.index); oi++ { - index := &b.index[oi] - index.keyPos += ob - if index.valueLen != 0 { - index.valuePos += ob - } - } - } -} - -func (b *Batch) decode(data []byte, expectedLen int) error { - b.data = data - b.index = b.index[:0] - b.internalLen = 0 - err := decodeBatch(data, func(i int, index batchIndex) error { - b.index = append(b.index, index) - b.internalLen += index.keyLen + index.valueLen + 8 - return nil - }) - if err != nil { - return err - } - if expectedLen >= 0 && len(b.index) != expectedLen { - return newErrBatchCorrupted(fmt.Sprintf("invalid records length: %d vs %d", expectedLen, len(b.index))) - } - return nil -} - -func (b *Batch) putMem(seq uint64, mdb *memdb.DB) error { - var ik []byte - for i, index := range b.index { - ik = makeInternalKey(ik, index.k(b.data), seq+uint64(i), index.keyType) - if err := mdb.Put(ik, index.v(b.data)); err != nil { - return err - } - } - return nil -} - -func (b *Batch) revertMem(seq uint64, mdb *memdb.DB) error { - var ik []byte - for i, index := range b.index { - ik = makeInternalKey(ik, index.k(b.data), seq+uint64(i), index.keyType) - if err := mdb.Delete(ik); err != nil { - return err - } - } - return nil -} - -func newBatch() interface{} { - return &Batch{} -} - -func decodeBatch(data []byte, fn func(i int, index batchIndex) error) error { - var index batchIndex - for i, o := 0, 0; o < len(data); i++ { - // Key type. - index.keyType = keyType(data[o]) - if index.keyType > keyTypeVal { - return newErrBatchCorrupted(fmt.Sprintf("bad record: invalid type %#x", uint(index.keyType))) - } - o++ - - // Key. - x, n := binary.Uvarint(data[o:]) - o += n - if n <= 0 || o+int(x) > len(data) { - return newErrBatchCorrupted("bad record: invalid key length") - } - index.keyPos = o - index.keyLen = int(x) - o += index.keyLen - - // Value. - if index.keyType == keyTypeVal { - x, n = binary.Uvarint(data[o:]) - o += n - if n <= 0 || o+int(x) > len(data) { - return newErrBatchCorrupted("bad record: invalid value length") - } - index.valuePos = o - index.valueLen = int(x) - o += index.valueLen - } else { - index.valuePos = 0 - index.valueLen = 0 - } - - if err := fn(i, index); err != nil { - return err - } - } - return nil -} - -func decodeBatchToMem(data []byte, expectSeq uint64, mdb *memdb.DB) (seq uint64, batchLen int, err error) { - seq, batchLen, err = decodeBatchHeader(data) - if err != nil { - return 0, 0, err - } - if seq < expectSeq { - return 0, 0, newErrBatchCorrupted("invalid sequence number") - } - data = data[batchHeaderLen:] - var ik []byte - var decodedLen int - err = decodeBatch(data, func(i int, index batchIndex) error { - if i >= batchLen { - return newErrBatchCorrupted("invalid records length") - } - ik = makeInternalKey(ik, index.k(data), seq+uint64(i), index.keyType) - if err := mdb.Put(ik, index.v(data)); err != nil { - return err - } - decodedLen++ - return nil - }) - if err == nil && decodedLen != batchLen { - err = newErrBatchCorrupted(fmt.Sprintf("invalid records length: %d vs %d", batchLen, decodedLen)) - } - return -} - -func encodeBatchHeader(dst []byte, seq uint64, batchLen int) []byte { - dst = ensureBuffer(dst, batchHeaderLen) - binary.LittleEndian.PutUint64(dst, seq) - binary.LittleEndian.PutUint32(dst[8:], uint32(batchLen)) - return dst -} - -func decodeBatchHeader(data []byte) (seq uint64, batchLen int, err error) { - if len(data) < batchHeaderLen { - return 0, 0, newErrBatchCorrupted("too short") - } - - seq = binary.LittleEndian.Uint64(data) - batchLen = int(binary.LittleEndian.Uint32(data[8:])) - if batchLen < 0 { - return 0, 0, newErrBatchCorrupted("invalid records length") - } - return -} - -func batchesLen(batches []*Batch) int { - batchLen := 0 - for _, batch := range batches { - batchLen += batch.Len() - } - return batchLen -} - -func writeBatchesWithHeader(wr io.Writer, batches []*Batch, seq uint64) error { - if _, err := wr.Write(encodeBatchHeader(nil, seq, batchesLen(batches))); err != nil { - return err - } - for _, batch := range batches { - if _, err := wr.Write(batch.data); err != nil { - return err - } - } - return nil -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/batch_test.go b/vendor/github.com/syndtr/goleveldb/leveldb/batch_test.go deleted file mode 100644 index 62775f7..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/batch_test.go +++ /dev/null @@ -1,147 +0,0 @@ -// Copyright (c) 2012, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package leveldb - -import ( - "bytes" - "fmt" - "testing" - "testing/quick" - - "github.com/syndtr/goleveldb/leveldb/testutil" -) - -func TestBatchHeader(t *testing.T) { - f := func(seq uint64, length uint32) bool { - encoded := encodeBatchHeader(nil, seq, int(length)) - decSeq, decLength, err := decodeBatchHeader(encoded) - return err == nil && decSeq == seq && decLength == int(length) - } - config := &quick.Config{ - Rand: testutil.NewRand(), - } - if err := quick.Check(f, config); err != nil { - t.Error(err) - } -} - -type batchKV struct { - kt keyType - k, v []byte -} - -func TestBatch(t *testing.T) { - var ( - kvs []batchKV - internalLen int - ) - batch := new(Batch) - rbatch := new(Batch) - abatch := new(Batch) - testBatch := func(i int, kt keyType, k, v []byte) error { - kv := kvs[i] - if kv.kt != kt { - return fmt.Errorf("invalid key type, index=%d: %d vs %d", i, kv.kt, kt) - } - if !bytes.Equal(kv.k, k) { - return fmt.Errorf("invalid key, index=%d", i) - } - if !bytes.Equal(kv.v, v) { - return fmt.Errorf("invalid value, index=%d", i) - } - return nil - } - f := func(ktr uint8, k, v []byte) bool { - kt := keyType(ktr % 2) - if kt == keyTypeVal { - batch.Put(k, v) - rbatch.Put(k, v) - kvs = append(kvs, batchKV{kt: kt, k: k, v: v}) - internalLen += len(k) + len(v) + 8 - } else { - batch.Delete(k) - rbatch.Delete(k) - kvs = append(kvs, batchKV{kt: kt, k: k}) - internalLen += len(k) + 8 - } - if batch.Len() != len(kvs) { - t.Logf("batch.Len: %d vs %d", len(kvs), batch.Len()) - return false - } - if batch.internalLen != internalLen { - t.Logf("abatch.internalLen: %d vs %d", internalLen, batch.internalLen) - return false - } - if len(kvs)%1000 == 0 { - if err := batch.replayInternal(testBatch); err != nil { - t.Logf("batch.replayInternal: %v", err) - return false - } - - abatch.append(rbatch) - rbatch.Reset() - if abatch.Len() != len(kvs) { - t.Logf("abatch.Len: %d vs %d", len(kvs), abatch.Len()) - return false - } - if abatch.internalLen != internalLen { - t.Logf("abatch.internalLen: %d vs %d", internalLen, abatch.internalLen) - return false - } - if err := abatch.replayInternal(testBatch); err != nil { - t.Logf("abatch.replayInternal: %v", err) - return false - } - - nbatch := new(Batch) - if err := nbatch.Load(batch.Dump()); err != nil { - t.Logf("nbatch.Load: %v", err) - return false - } - if nbatch.Len() != len(kvs) { - t.Logf("nbatch.Len: %d vs %d", len(kvs), nbatch.Len()) - return false - } - if nbatch.internalLen != internalLen { - t.Logf("nbatch.internalLen: %d vs %d", internalLen, nbatch.internalLen) - return false - } - if err := nbatch.replayInternal(testBatch); err != nil { - t.Logf("nbatch.replayInternal: %v", err) - return false - } - } - if len(kvs)%10000 == 0 { - nbatch := new(Batch) - if err := batch.Replay(nbatch); err != nil { - t.Logf("batch.Replay: %v", err) - return false - } - if nbatch.Len() != len(kvs) { - t.Logf("nbatch.Len: %d vs %d", len(kvs), nbatch.Len()) - return false - } - if nbatch.internalLen != internalLen { - t.Logf("nbatch.internalLen: %d vs %d", internalLen, nbatch.internalLen) - return false - } - if err := nbatch.replayInternal(testBatch); err != nil { - t.Logf("nbatch.replayInternal: %v", err) - return false - } - } - return true - } - config := &quick.Config{ - MaxCount: 40000, - Rand: testutil.NewRand(), - } - if err := quick.Check(f, config); err != nil { - t.Error(err) - } - t.Logf("length=%d internalLen=%d", len(kvs), internalLen) -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/bench_test.go b/vendor/github.com/syndtr/goleveldb/leveldb/bench_test.go deleted file mode 100644 index 435250c..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/bench_test.go +++ /dev/null @@ -1,507 +0,0 @@ -// Copyright (c) 2012, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package leveldb - -import ( - "bytes" - "fmt" - "math/rand" - "os" - "path/filepath" - "runtime" - "sync/atomic" - "testing" - - "github.com/syndtr/goleveldb/leveldb/iterator" - "github.com/syndtr/goleveldb/leveldb/opt" - "github.com/syndtr/goleveldb/leveldb/storage" -) - -func randomString(r *rand.Rand, n int) []byte { - b := new(bytes.Buffer) - for i := 0; i < n; i++ { - b.WriteByte(' ' + byte(r.Intn(95))) - } - return b.Bytes() -} - -func compressibleStr(r *rand.Rand, frac float32, n int) []byte { - nn := int(float32(n) * frac) - rb := randomString(r, nn) - b := make([]byte, 0, n+nn) - for len(b) < n { - b = append(b, rb...) - } - return b[:n] -} - -type valueGen struct { - src []byte - pos int -} - -func newValueGen(frac float32) *valueGen { - v := new(valueGen) - r := rand.New(rand.NewSource(301)) - v.src = make([]byte, 0, 1048576+100) - for len(v.src) < 1048576 { - v.src = append(v.src, compressibleStr(r, frac, 100)...) - } - return v -} - -func (v *valueGen) get(n int) []byte { - if v.pos+n > len(v.src) { - v.pos = 0 - } - v.pos += n - return v.src[v.pos-n : v.pos] -} - -var benchDB = filepath.Join(os.TempDir(), fmt.Sprintf("goleveldbbench-%d", os.Getuid())) - -type dbBench struct { - b *testing.B - stor storage.Storage - db *DB - - o *opt.Options - ro *opt.ReadOptions - wo *opt.WriteOptions - - keys, values [][]byte -} - -func openDBBench(b *testing.B, noCompress bool) *dbBench { - _, err := os.Stat(benchDB) - if err == nil { - err = os.RemoveAll(benchDB) - if err != nil { - b.Fatal("cannot remove old db: ", err) - } - } - - p := &dbBench{ - b: b, - o: &opt.Options{}, - ro: &opt.ReadOptions{}, - wo: &opt.WriteOptions{}, - } - p.stor, err = storage.OpenFile(benchDB, false) - if err != nil { - b.Fatal("cannot open stor: ", err) - } - if noCompress { - p.o.Compression = opt.NoCompression - } - - p.db, err = Open(p.stor, p.o) - if err != nil { - b.Fatal("cannot open db: ", err) - } - - return p -} - -func (p *dbBench) reopen() { - p.db.Close() - var err error - p.db, err = Open(p.stor, p.o) - if err != nil { - p.b.Fatal("Reopen: got error: ", err) - } -} - -func (p *dbBench) populate(n int) { - p.keys, p.values = make([][]byte, n), make([][]byte, n) - v := newValueGen(0.5) - for i := range p.keys { - p.keys[i], p.values[i] = []byte(fmt.Sprintf("%016d", i)), v.get(100) - } -} - -func (p *dbBench) randomize() { - m := len(p.keys) - times := m * 2 - r1, r2 := rand.New(rand.NewSource(0xdeadbeef)), rand.New(rand.NewSource(0xbeefface)) - for n := 0; n < times; n++ { - i, j := r1.Int()%m, r2.Int()%m - if i == j { - continue - } - p.keys[i], p.keys[j] = p.keys[j], p.keys[i] - p.values[i], p.values[j] = p.values[j], p.values[i] - } -} - -func (p *dbBench) writes(perBatch int) { - b := p.b - db := p.db - - n := len(p.keys) - m := n / perBatch - if n%perBatch > 0 { - m++ - } - batches := make([]Batch, m) - j := 0 - for i := range batches { - first := true - for ; j < n && ((j+1)%perBatch != 0 || first); j++ { - first = false - batches[i].Put(p.keys[j], p.values[j]) - } - } - runtime.GC() - - b.ResetTimer() - b.StartTimer() - for i := range batches { - err := db.Write(&(batches[i]), p.wo) - if err != nil { - b.Fatal("write failed: ", err) - } - } - b.StopTimer() - b.SetBytes(116) -} - -func (p *dbBench) gc() { - p.keys, p.values = nil, nil - runtime.GC() -} - -func (p *dbBench) puts() { - b := p.b - db := p.db - - b.ResetTimer() - b.StartTimer() - for i := range p.keys { - err := db.Put(p.keys[i], p.values[i], p.wo) - if err != nil { - b.Fatal("put failed: ", err) - } - } - b.StopTimer() - b.SetBytes(116) -} - -func (p *dbBench) fill() { - b := p.b - db := p.db - - perBatch := 10000 - batch := new(Batch) - for i, n := 0, len(p.keys); i < n; { - first := true - for ; i < n && ((i+1)%perBatch != 0 || first); i++ { - first = false - batch.Put(p.keys[i], p.values[i]) - } - err := db.Write(batch, p.wo) - if err != nil { - b.Fatal("write failed: ", err) - } - batch.Reset() - } -} - -func (p *dbBench) gets() { - b := p.b - db := p.db - - b.ResetTimer() - for i := range p.keys { - _, err := db.Get(p.keys[i], p.ro) - if err != nil { - b.Error("got error: ", err) - } - } - b.StopTimer() -} - -func (p *dbBench) seeks() { - b := p.b - - iter := p.newIter() - defer iter.Release() - b.ResetTimer() - for i := range p.keys { - if !iter.Seek(p.keys[i]) { - b.Error("value not found for: ", string(p.keys[i])) - } - } - b.StopTimer() -} - -func (p *dbBench) newIter() iterator.Iterator { - iter := p.db.NewIterator(nil, p.ro) - err := iter.Error() - if err != nil { - p.b.Fatal("cannot create iterator: ", err) - } - return iter -} - -func (p *dbBench) close() { - if bp, err := p.db.GetProperty("leveldb.blockpool"); err == nil { - p.b.Log("Block pool stats: ", bp) - } - p.db.Close() - p.stor.Close() - os.RemoveAll(benchDB) - p.db = nil - p.keys = nil - p.values = nil - runtime.GC() -} - -func BenchmarkDBWrite(b *testing.B) { - p := openDBBench(b, false) - p.populate(b.N) - p.writes(1) - p.close() -} - -func BenchmarkDBWriteBatch(b *testing.B) { - p := openDBBench(b, false) - p.populate(b.N) - p.writes(1000) - p.close() -} - -func BenchmarkDBWriteUncompressed(b *testing.B) { - p := openDBBench(b, true) - p.populate(b.N) - p.writes(1) - p.close() -} - -func BenchmarkDBWriteBatchUncompressed(b *testing.B) { - p := openDBBench(b, true) - p.populate(b.N) - p.writes(1000) - p.close() -} - -func BenchmarkDBWriteRandom(b *testing.B) { - p := openDBBench(b, false) - p.populate(b.N) - p.randomize() - p.writes(1) - p.close() -} - -func BenchmarkDBWriteRandomSync(b *testing.B) { - p := openDBBench(b, false) - p.wo.Sync = true - p.populate(b.N) - p.writes(1) - p.close() -} - -func BenchmarkDBOverwrite(b *testing.B) { - p := openDBBench(b, false) - p.populate(b.N) - p.writes(1) - p.writes(1) - p.close() -} - -func BenchmarkDBOverwriteRandom(b *testing.B) { - p := openDBBench(b, false) - p.populate(b.N) - p.writes(1) - p.randomize() - p.writes(1) - p.close() -} - -func BenchmarkDBPut(b *testing.B) { - p := openDBBench(b, false) - p.populate(b.N) - p.puts() - p.close() -} - -func BenchmarkDBRead(b *testing.B) { - p := openDBBench(b, false) - p.populate(b.N) - p.fill() - p.gc() - - iter := p.newIter() - b.ResetTimer() - for iter.Next() { - } - iter.Release() - b.StopTimer() - b.SetBytes(116) - p.close() -} - -func BenchmarkDBReadGC(b *testing.B) { - p := openDBBench(b, false) - p.populate(b.N) - p.fill() - - iter := p.newIter() - b.ResetTimer() - for iter.Next() { - } - iter.Release() - b.StopTimer() - b.SetBytes(116) - p.close() -} - -func BenchmarkDBReadUncompressed(b *testing.B) { - p := openDBBench(b, true) - p.populate(b.N) - p.fill() - p.gc() - - iter := p.newIter() - b.ResetTimer() - for iter.Next() { - } - iter.Release() - b.StopTimer() - b.SetBytes(116) - p.close() -} - -func BenchmarkDBReadTable(b *testing.B) { - p := openDBBench(b, false) - p.populate(b.N) - p.fill() - p.reopen() - p.gc() - - iter := p.newIter() - b.ResetTimer() - for iter.Next() { - } - iter.Release() - b.StopTimer() - b.SetBytes(116) - p.close() -} - -func BenchmarkDBReadReverse(b *testing.B) { - p := openDBBench(b, false) - p.populate(b.N) - p.fill() - p.gc() - - iter := p.newIter() - b.ResetTimer() - iter.Last() - for iter.Prev() { - } - iter.Release() - b.StopTimer() - b.SetBytes(116) - p.close() -} - -func BenchmarkDBReadReverseTable(b *testing.B) { - p := openDBBench(b, false) - p.populate(b.N) - p.fill() - p.reopen() - p.gc() - - iter := p.newIter() - b.ResetTimer() - iter.Last() - for iter.Prev() { - } - iter.Release() - b.StopTimer() - b.SetBytes(116) - p.close() -} - -func BenchmarkDBSeek(b *testing.B) { - p := openDBBench(b, false) - p.populate(b.N) - p.fill() - p.seeks() - p.close() -} - -func BenchmarkDBSeekRandom(b *testing.B) { - p := openDBBench(b, false) - p.populate(b.N) - p.fill() - p.randomize() - p.seeks() - p.close() -} - -func BenchmarkDBGet(b *testing.B) { - p := openDBBench(b, false) - p.populate(b.N) - p.fill() - p.gets() - p.close() -} - -func BenchmarkDBGetRandom(b *testing.B) { - p := openDBBench(b, false) - p.populate(b.N) - p.fill() - p.randomize() - p.gets() - p.close() -} - -func BenchmarkDBReadConcurrent(b *testing.B) { - p := openDBBench(b, false) - p.populate(b.N) - p.fill() - p.gc() - defer p.close() - - b.ResetTimer() - b.SetBytes(116) - - b.RunParallel(func(pb *testing.PB) { - iter := p.newIter() - defer iter.Release() - for pb.Next() && iter.Next() { - } - }) -} - -func BenchmarkDBReadConcurrent2(b *testing.B) { - p := openDBBench(b, false) - p.populate(b.N) - p.fill() - p.gc() - defer p.close() - - b.ResetTimer() - b.SetBytes(116) - - var dir uint32 - b.RunParallel(func(pb *testing.PB) { - iter := p.newIter() - defer iter.Release() - if atomic.AddUint32(&dir, 1)%2 == 0 { - for pb.Next() && iter.Next() { - } - } else { - if pb.Next() && iter.Last() { - for pb.Next() && iter.Prev() { - } - } - } - }) -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/cache/bench_test.go b/vendor/github.com/syndtr/goleveldb/leveldb/cache/bench_test.go deleted file mode 100644 index 89aef69..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/cache/bench_test.go +++ /dev/null @@ -1,29 +0,0 @@ -// Copyright (c) 2012, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package cache - -import ( - "math/rand" - "testing" - "time" -) - -func BenchmarkLRUCache(b *testing.B) { - c := NewCache(NewLRU(10000)) - - b.SetParallelism(10) - b.RunParallel(func(pb *testing.PB) { - r := rand.New(rand.NewSource(time.Now().UnixNano())) - - for pb.Next() { - key := uint64(r.Intn(1000000)) - c.Get(0, key, func() (int, Value) { - return 1, key - }).Release() - } - }) -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/cache/cache.go b/vendor/github.com/syndtr/goleveldb/leveldb/cache/cache.go deleted file mode 100644 index c5940b2..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/cache/cache.go +++ /dev/null @@ -1,705 +0,0 @@ -// Copyright (c) 2012, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -// Package cache provides interface and implementation of a cache algorithms. -package cache - -import ( - "sync" - "sync/atomic" - "unsafe" - - "github.com/syndtr/goleveldb/leveldb/util" -) - -// Cacher provides interface to implements a caching functionality. -// An implementation must be safe for concurrent use. -type Cacher interface { - // Capacity returns cache capacity. - Capacity() int - - // SetCapacity sets cache capacity. - SetCapacity(capacity int) - - // Promote promotes the 'cache node'. - Promote(n *Node) - - // Ban evicts the 'cache node' and prevent subsequent 'promote'. - Ban(n *Node) - - // Evict evicts the 'cache node'. - Evict(n *Node) - - // EvictNS evicts 'cache node' with the given namespace. - EvictNS(ns uint64) - - // EvictAll evicts all 'cache node'. - EvictAll() - - // Close closes the 'cache tree' - Close() error -} - -// Value is a 'cacheable object'. It may implements util.Releaser, if -// so the the Release method will be called once object is released. -type Value interface{} - -// NamespaceGetter provides convenient wrapper for namespace. -type NamespaceGetter struct { - Cache *Cache - NS uint64 -} - -// Get simply calls Cache.Get() method. -func (g *NamespaceGetter) Get(key uint64, setFunc func() (size int, value Value)) *Handle { - return g.Cache.Get(g.NS, key, setFunc) -} - -// The hash tables implementation is based on: -// "Dynamic-Sized Nonblocking Hash Tables", by Yujie Liu, -// Kunlong Zhang, and Michael Spear. -// ACM Symposium on Principles of Distributed Computing, Jul 2014. - -const ( - mInitialSize = 1 << 4 - mOverflowThreshold = 1 << 5 - mOverflowGrowThreshold = 1 << 7 -) - -type mBucket struct { - mu sync.Mutex - node []*Node - frozen bool -} - -func (b *mBucket) freeze() []*Node { - b.mu.Lock() - defer b.mu.Unlock() - if !b.frozen { - b.frozen = true - } - return b.node -} - -func (b *mBucket) get(r *Cache, h *mNode, hash uint32, ns, key uint64, noset bool) (done, added bool, n *Node) { - b.mu.Lock() - - if b.frozen { - b.mu.Unlock() - return - } - - // Scan the node. - for _, n := range b.node { - if n.hash == hash && n.ns == ns && n.key == key { - atomic.AddInt32(&n.ref, 1) - b.mu.Unlock() - return true, false, n - } - } - - // Get only. - if noset { - b.mu.Unlock() - return true, false, nil - } - - // Create node. - n = &Node{ - r: r, - hash: hash, - ns: ns, - key: key, - ref: 1, - } - // Add node to bucket. - b.node = append(b.node, n) - bLen := len(b.node) - b.mu.Unlock() - - // Update counter. - grow := atomic.AddInt32(&r.nodes, 1) >= h.growThreshold - if bLen > mOverflowThreshold { - grow = grow || atomic.AddInt32(&h.overflow, 1) >= mOverflowGrowThreshold - } - - // Grow. - if grow && atomic.CompareAndSwapInt32(&h.resizeInProgess, 0, 1) { - nhLen := len(h.buckets) << 1 - nh := &mNode{ - buckets: make([]unsafe.Pointer, nhLen), - mask: uint32(nhLen) - 1, - pred: unsafe.Pointer(h), - growThreshold: int32(nhLen * mOverflowThreshold), - shrinkThreshold: int32(nhLen >> 1), - } - ok := atomic.CompareAndSwapPointer(&r.mHead, unsafe.Pointer(h), unsafe.Pointer(nh)) - if !ok { - panic("BUG: failed swapping head") - } - go nh.initBuckets() - } - - return true, true, n -} - -func (b *mBucket) delete(r *Cache, h *mNode, hash uint32, ns, key uint64) (done, deleted bool) { - b.mu.Lock() - - if b.frozen { - b.mu.Unlock() - return - } - - // Scan the node. - var ( - n *Node - bLen int - ) - for i := range b.node { - n = b.node[i] - if n.ns == ns && n.key == key { - if atomic.LoadInt32(&n.ref) == 0 { - deleted = true - - // Call releaser. - if n.value != nil { - if r, ok := n.value.(util.Releaser); ok { - r.Release() - } - n.value = nil - } - - // Remove node from bucket. - b.node = append(b.node[:i], b.node[i+1:]...) - bLen = len(b.node) - } - break - } - } - b.mu.Unlock() - - if deleted { - // Call OnDel. - for _, f := range n.onDel { - f() - } - - // Update counter. - atomic.AddInt32(&r.size, int32(n.size)*-1) - shrink := atomic.AddInt32(&r.nodes, -1) < h.shrinkThreshold - if bLen >= mOverflowThreshold { - atomic.AddInt32(&h.overflow, -1) - } - - // Shrink. - if shrink && len(h.buckets) > mInitialSize && atomic.CompareAndSwapInt32(&h.resizeInProgess, 0, 1) { - nhLen := len(h.buckets) >> 1 - nh := &mNode{ - buckets: make([]unsafe.Pointer, nhLen), - mask: uint32(nhLen) - 1, - pred: unsafe.Pointer(h), - growThreshold: int32(nhLen * mOverflowThreshold), - shrinkThreshold: int32(nhLen >> 1), - } - ok := atomic.CompareAndSwapPointer(&r.mHead, unsafe.Pointer(h), unsafe.Pointer(nh)) - if !ok { - panic("BUG: failed swapping head") - } - go nh.initBuckets() - } - } - - return true, deleted -} - -type mNode struct { - buckets []unsafe.Pointer // []*mBucket - mask uint32 - pred unsafe.Pointer // *mNode - resizeInProgess int32 - - overflow int32 - growThreshold int32 - shrinkThreshold int32 -} - -func (n *mNode) initBucket(i uint32) *mBucket { - if b := (*mBucket)(atomic.LoadPointer(&n.buckets[i])); b != nil { - return b - } - - p := (*mNode)(atomic.LoadPointer(&n.pred)) - if p != nil { - var node []*Node - if n.mask > p.mask { - // Grow. - pb := (*mBucket)(atomic.LoadPointer(&p.buckets[i&p.mask])) - if pb == nil { - pb = p.initBucket(i & p.mask) - } - m := pb.freeze() - // Split nodes. - for _, x := range m { - if x.hash&n.mask == i { - node = append(node, x) - } - } - } else { - // Shrink. - pb0 := (*mBucket)(atomic.LoadPointer(&p.buckets[i])) - if pb0 == nil { - pb0 = p.initBucket(i) - } - pb1 := (*mBucket)(atomic.LoadPointer(&p.buckets[i+uint32(len(n.buckets))])) - if pb1 == nil { - pb1 = p.initBucket(i + uint32(len(n.buckets))) - } - m0 := pb0.freeze() - m1 := pb1.freeze() - // Merge nodes. - node = make([]*Node, 0, len(m0)+len(m1)) - node = append(node, m0...) - node = append(node, m1...) - } - b := &mBucket{node: node} - if atomic.CompareAndSwapPointer(&n.buckets[i], nil, unsafe.Pointer(b)) { - if len(node) > mOverflowThreshold { - atomic.AddInt32(&n.overflow, int32(len(node)-mOverflowThreshold)) - } - return b - } - } - - return (*mBucket)(atomic.LoadPointer(&n.buckets[i])) -} - -func (n *mNode) initBuckets() { - for i := range n.buckets { - n.initBucket(uint32(i)) - } - atomic.StorePointer(&n.pred, nil) -} - -// Cache is a 'cache map'. -type Cache struct { - mu sync.RWMutex - mHead unsafe.Pointer // *mNode - nodes int32 - size int32 - cacher Cacher - closed bool -} - -// NewCache creates a new 'cache map'. The cacher is optional and -// may be nil. -func NewCache(cacher Cacher) *Cache { - h := &mNode{ - buckets: make([]unsafe.Pointer, mInitialSize), - mask: mInitialSize - 1, - growThreshold: int32(mInitialSize * mOverflowThreshold), - shrinkThreshold: 0, - } - for i := range h.buckets { - h.buckets[i] = unsafe.Pointer(&mBucket{}) - } - r := &Cache{ - mHead: unsafe.Pointer(h), - cacher: cacher, - } - return r -} - -func (r *Cache) getBucket(hash uint32) (*mNode, *mBucket) { - h := (*mNode)(atomic.LoadPointer(&r.mHead)) - i := hash & h.mask - b := (*mBucket)(atomic.LoadPointer(&h.buckets[i])) - if b == nil { - b = h.initBucket(i) - } - return h, b -} - -func (r *Cache) delete(n *Node) bool { - for { - h, b := r.getBucket(n.hash) - done, deleted := b.delete(r, h, n.hash, n.ns, n.key) - if done { - return deleted - } - } - return false -} - -// Nodes returns number of 'cache node' in the map. -func (r *Cache) Nodes() int { - return int(atomic.LoadInt32(&r.nodes)) -} - -// Size returns sums of 'cache node' size in the map. -func (r *Cache) Size() int { - return int(atomic.LoadInt32(&r.size)) -} - -// Capacity returns cache capacity. -func (r *Cache) Capacity() int { - if r.cacher == nil { - return 0 - } - return r.cacher.Capacity() -} - -// SetCapacity sets cache capacity. -func (r *Cache) SetCapacity(capacity int) { - if r.cacher != nil { - r.cacher.SetCapacity(capacity) - } -} - -// Get gets 'cache node' with the given namespace and key. -// If cache node is not found and setFunc is not nil, Get will atomically creates -// the 'cache node' by calling setFunc. Otherwise Get will returns nil. -// -// The returned 'cache handle' should be released after use by calling Release -// method. -func (r *Cache) Get(ns, key uint64, setFunc func() (size int, value Value)) *Handle { - r.mu.RLock() - defer r.mu.RUnlock() - if r.closed { - return nil - } - - hash := murmur32(ns, key, 0xf00) - for { - h, b := r.getBucket(hash) - done, _, n := b.get(r, h, hash, ns, key, setFunc == nil) - if done { - if n != nil { - n.mu.Lock() - if n.value == nil { - if setFunc == nil { - n.mu.Unlock() - n.unref() - return nil - } - - n.size, n.value = setFunc() - if n.value == nil { - n.size = 0 - n.mu.Unlock() - n.unref() - return nil - } - atomic.AddInt32(&r.size, int32(n.size)) - } - n.mu.Unlock() - if r.cacher != nil { - r.cacher.Promote(n) - } - return &Handle{unsafe.Pointer(n)} - } - - break - } - } - return nil -} - -// Delete removes and ban 'cache node' with the given namespace and key. -// A banned 'cache node' will never inserted into the 'cache tree'. Ban -// only attributed to the particular 'cache node', so when a 'cache node' -// is recreated it will not be banned. -// -// If onDel is not nil, then it will be executed if such 'cache node' -// doesn't exist or once the 'cache node' is released. -// -// Delete return true is such 'cache node' exist. -func (r *Cache) Delete(ns, key uint64, onDel func()) bool { - r.mu.RLock() - defer r.mu.RUnlock() - if r.closed { - return false - } - - hash := murmur32(ns, key, 0xf00) - for { - h, b := r.getBucket(hash) - done, _, n := b.get(r, h, hash, ns, key, true) - if done { - if n != nil { - if onDel != nil { - n.mu.Lock() - n.onDel = append(n.onDel, onDel) - n.mu.Unlock() - } - if r.cacher != nil { - r.cacher.Ban(n) - } - n.unref() - return true - } - - break - } - } - - if onDel != nil { - onDel() - } - - return false -} - -// Evict evicts 'cache node' with the given namespace and key. This will -// simply call Cacher.Evict. -// -// Evict return true is such 'cache node' exist. -func (r *Cache) Evict(ns, key uint64) bool { - r.mu.RLock() - defer r.mu.RUnlock() - if r.closed { - return false - } - - hash := murmur32(ns, key, 0xf00) - for { - h, b := r.getBucket(hash) - done, _, n := b.get(r, h, hash, ns, key, true) - if done { - if n != nil { - if r.cacher != nil { - r.cacher.Evict(n) - } - n.unref() - return true - } - - break - } - } - - return false -} - -// EvictNS evicts 'cache node' with the given namespace. This will -// simply call Cacher.EvictNS. -func (r *Cache) EvictNS(ns uint64) { - r.mu.RLock() - defer r.mu.RUnlock() - if r.closed { - return - } - - if r.cacher != nil { - r.cacher.EvictNS(ns) - } -} - -// EvictAll evicts all 'cache node'. This will simply call Cacher.EvictAll. -func (r *Cache) EvictAll() { - r.mu.RLock() - defer r.mu.RUnlock() - if r.closed { - return - } - - if r.cacher != nil { - r.cacher.EvictAll() - } -} - -// Close closes the 'cache map' and forcefully releases all 'cache node'. -func (r *Cache) Close() error { - r.mu.Lock() - if !r.closed { - r.closed = true - - h := (*mNode)(r.mHead) - h.initBuckets() - - for i := range h.buckets { - b := (*mBucket)(h.buckets[i]) - for _, n := range b.node { - // Call releaser. - if n.value != nil { - if r, ok := n.value.(util.Releaser); ok { - r.Release() - } - n.value = nil - } - - // Call OnDel. - for _, f := range n.onDel { - f() - } - n.onDel = nil - } - } - } - r.mu.Unlock() - - // Avoid deadlock. - if r.cacher != nil { - if err := r.cacher.Close(); err != nil { - return err - } - } - return nil -} - -// CloseWeak closes the 'cache map' and evict all 'cache node' from cacher, but -// unlike Close it doesn't forcefully releases 'cache node'. -func (r *Cache) CloseWeak() error { - r.mu.Lock() - if !r.closed { - r.closed = true - } - r.mu.Unlock() - - // Avoid deadlock. - if r.cacher != nil { - r.cacher.EvictAll() - if err := r.cacher.Close(); err != nil { - return err - } - } - return nil -} - -// Node is a 'cache node'. -type Node struct { - r *Cache - - hash uint32 - ns, key uint64 - - mu sync.Mutex - size int - value Value - - ref int32 - onDel []func() - - CacheData unsafe.Pointer -} - -// NS returns this 'cache node' namespace. -func (n *Node) NS() uint64 { - return n.ns -} - -// Key returns this 'cache node' key. -func (n *Node) Key() uint64 { - return n.key -} - -// Size returns this 'cache node' size. -func (n *Node) Size() int { - return n.size -} - -// Value returns this 'cache node' value. -func (n *Node) Value() Value { - return n.value -} - -// Ref returns this 'cache node' ref counter. -func (n *Node) Ref() int32 { - return atomic.LoadInt32(&n.ref) -} - -// GetHandle returns an handle for this 'cache node'. -func (n *Node) GetHandle() *Handle { - if atomic.AddInt32(&n.ref, 1) <= 1 { - panic("BUG: Node.GetHandle on zero ref") - } - return &Handle{unsafe.Pointer(n)} -} - -func (n *Node) unref() { - if atomic.AddInt32(&n.ref, -1) == 0 { - n.r.delete(n) - } -} - -func (n *Node) unrefLocked() { - if atomic.AddInt32(&n.ref, -1) == 0 { - n.r.mu.RLock() - if !n.r.closed { - n.r.delete(n) - } - n.r.mu.RUnlock() - } -} - -// Handle is a 'cache handle' of a 'cache node'. -type Handle struct { - n unsafe.Pointer // *Node -} - -// Value returns the value of the 'cache node'. -func (h *Handle) Value() Value { - n := (*Node)(atomic.LoadPointer(&h.n)) - if n != nil { - return n.value - } - return nil -} - -// Release releases this 'cache handle'. -// It is safe to call release multiple times. -func (h *Handle) Release() { - nPtr := atomic.LoadPointer(&h.n) - if nPtr != nil && atomic.CompareAndSwapPointer(&h.n, nPtr, nil) { - n := (*Node)(nPtr) - n.unrefLocked() - } -} - -func murmur32(ns, key uint64, seed uint32) uint32 { - const ( - m = uint32(0x5bd1e995) - r = 24 - ) - - k1 := uint32(ns >> 32) - k2 := uint32(ns) - k3 := uint32(key >> 32) - k4 := uint32(key) - - k1 *= m - k1 ^= k1 >> r - k1 *= m - - k2 *= m - k2 ^= k2 >> r - k2 *= m - - k3 *= m - k3 ^= k3 >> r - k3 *= m - - k4 *= m - k4 ^= k4 >> r - k4 *= m - - h := seed - - h *= m - h ^= k1 - h *= m - h ^= k2 - h *= m - h ^= k3 - h *= m - h ^= k4 - - h ^= h >> 13 - h *= m - h ^= h >> 15 - - return h -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/cache/cache_test.go b/vendor/github.com/syndtr/goleveldb/leveldb/cache/cache_test.go deleted file mode 100644 index 6b017bd..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/cache/cache_test.go +++ /dev/null @@ -1,563 +0,0 @@ -// Copyright (c) 2012, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package cache - -import ( - "math/rand" - "runtime" - "sync" - "sync/atomic" - "testing" - "time" - "unsafe" -) - -type int32o int32 - -func (o *int32o) acquire() { - if atomic.AddInt32((*int32)(o), 1) != 1 { - panic("BUG: invalid ref") - } -} - -func (o *int32o) Release() { - if atomic.AddInt32((*int32)(o), -1) != 0 { - panic("BUG: invalid ref") - } -} - -type releaserFunc struct { - fn func() - value Value -} - -func (r releaserFunc) Release() { - if r.fn != nil { - r.fn() - } -} - -func set(c *Cache, ns, key uint64, value Value, charge int, relf func()) *Handle { - return c.Get(ns, key, func() (int, Value) { - if relf != nil { - return charge, releaserFunc{relf, value} - } - return charge, value - }) -} - -type cacheMapTestParams struct { - nobjects, nhandles, concurrent, repeat int -} - -func TestCacheMap(t *testing.T) { - runtime.GOMAXPROCS(runtime.NumCPU()) - - var params []cacheMapTestParams - if testing.Short() { - params = []cacheMapTestParams{ - {1000, 100, 20, 3}, - {10000, 300, 50, 10}, - } - } else { - params = []cacheMapTestParams{ - {10000, 400, 50, 3}, - {100000, 1000, 100, 10}, - } - } - - var ( - objects [][]int32o - handles [][]unsafe.Pointer - ) - - for _, x := range params { - objects = append(objects, make([]int32o, x.nobjects)) - handles = append(handles, make([]unsafe.Pointer, x.nhandles)) - } - - c := NewCache(nil) - - wg := new(sync.WaitGroup) - var done int32 - - for ns, x := range params { - for i := 0; i < x.concurrent; i++ { - wg.Add(1) - go func(ns, i, repeat int, objects []int32o, handles []unsafe.Pointer) { - defer wg.Done() - r := rand.New(rand.NewSource(time.Now().UnixNano())) - - for j := len(objects) * repeat; j >= 0; j-- { - key := uint64(r.Intn(len(objects))) - h := c.Get(uint64(ns), key, func() (int, Value) { - o := &objects[key] - o.acquire() - return 1, o - }) - if v := h.Value().(*int32o); v != &objects[key] { - t.Fatalf("#%d invalid value: want=%p got=%p", ns, &objects[key], v) - } - if objects[key] != 1 { - t.Fatalf("#%d invalid object %d: %d", ns, key, objects[key]) - } - if !atomic.CompareAndSwapPointer(&handles[r.Intn(len(handles))], nil, unsafe.Pointer(h)) { - h.Release() - } - } - }(ns, i, x.repeat, objects[ns], handles[ns]) - } - - go func(handles []unsafe.Pointer) { - r := rand.New(rand.NewSource(time.Now().UnixNano())) - - for atomic.LoadInt32(&done) == 0 { - i := r.Intn(len(handles)) - h := (*Handle)(atomic.LoadPointer(&handles[i])) - if h != nil && atomic.CompareAndSwapPointer(&handles[i], unsafe.Pointer(h), nil) { - h.Release() - } - time.Sleep(time.Millisecond) - } - }(handles[ns]) - } - - go func() { - handles := make([]*Handle, 100000) - for atomic.LoadInt32(&done) == 0 { - for i := range handles { - handles[i] = c.Get(999999999, uint64(i), func() (int, Value) { - return 1, 1 - }) - } - for _, h := range handles { - h.Release() - } - } - }() - - wg.Wait() - - atomic.StoreInt32(&done, 1) - - for _, handles0 := range handles { - for i := range handles0 { - h := (*Handle)(atomic.LoadPointer(&handles0[i])) - if h != nil && atomic.CompareAndSwapPointer(&handles0[i], unsafe.Pointer(h), nil) { - h.Release() - } - } - } - - for ns, objects0 := range objects { - for i, o := range objects0 { - if o != 0 { - t.Fatalf("invalid object #%d.%d: ref=%d", ns, i, o) - } - } - } -} - -func TestCacheMap_NodesAndSize(t *testing.T) { - c := NewCache(nil) - if c.Nodes() != 0 { - t.Errorf("invalid nodes counter: want=%d got=%d", 0, c.Nodes()) - } - if c.Size() != 0 { - t.Errorf("invalid size counter: want=%d got=%d", 0, c.Size()) - } - set(c, 0, 1, 1, 1, nil) - set(c, 0, 2, 2, 2, nil) - set(c, 1, 1, 3, 3, nil) - set(c, 2, 1, 4, 1, nil) - if c.Nodes() != 4 { - t.Errorf("invalid nodes counter: want=%d got=%d", 4, c.Nodes()) - } - if c.Size() != 7 { - t.Errorf("invalid size counter: want=%d got=%d", 4, c.Size()) - } -} - -func TestLRUCache_Capacity(t *testing.T) { - c := NewCache(NewLRU(10)) - if c.Capacity() != 10 { - t.Errorf("invalid capacity: want=%d got=%d", 10, c.Capacity()) - } - set(c, 0, 1, 1, 1, nil).Release() - set(c, 0, 2, 2, 2, nil).Release() - set(c, 1, 1, 3, 3, nil).Release() - set(c, 2, 1, 4, 1, nil).Release() - set(c, 2, 2, 5, 1, nil).Release() - set(c, 2, 3, 6, 1, nil).Release() - set(c, 2, 4, 7, 1, nil).Release() - set(c, 2, 5, 8, 1, nil).Release() - if c.Nodes() != 7 { - t.Errorf("invalid nodes counter: want=%d got=%d", 7, c.Nodes()) - } - if c.Size() != 10 { - t.Errorf("invalid size counter: want=%d got=%d", 10, c.Size()) - } - c.SetCapacity(9) - if c.Capacity() != 9 { - t.Errorf("invalid capacity: want=%d got=%d", 9, c.Capacity()) - } - if c.Nodes() != 6 { - t.Errorf("invalid nodes counter: want=%d got=%d", 6, c.Nodes()) - } - if c.Size() != 8 { - t.Errorf("invalid size counter: want=%d got=%d", 8, c.Size()) - } -} - -func TestCacheMap_NilValue(t *testing.T) { - c := NewCache(NewLRU(10)) - h := c.Get(0, 0, func() (size int, value Value) { - return 1, nil - }) - if h != nil { - t.Error("cache handle is non-nil") - } - if c.Nodes() != 0 { - t.Errorf("invalid nodes counter: want=%d got=%d", 0, c.Nodes()) - } - if c.Size() != 0 { - t.Errorf("invalid size counter: want=%d got=%d", 0, c.Size()) - } -} - -func TestLRUCache_GetLatency(t *testing.T) { - runtime.GOMAXPROCS(runtime.NumCPU()) - - const ( - concurrentSet = 30 - concurrentGet = 3 - duration = 3 * time.Second - delay = 3 * time.Millisecond - maxkey = 100000 - ) - - var ( - set, getHit, getAll int32 - getMaxLatency, getDuration int64 - ) - - c := NewCache(NewLRU(5000)) - wg := &sync.WaitGroup{} - until := time.Now().Add(duration) - for i := 0; i < concurrentSet; i++ { - wg.Add(1) - go func(i int) { - defer wg.Done() - r := rand.New(rand.NewSource(time.Now().UnixNano())) - for time.Now().Before(until) { - c.Get(0, uint64(r.Intn(maxkey)), func() (int, Value) { - time.Sleep(delay) - atomic.AddInt32(&set, 1) - return 1, 1 - }).Release() - } - }(i) - } - for i := 0; i < concurrentGet; i++ { - wg.Add(1) - go func(i int) { - defer wg.Done() - r := rand.New(rand.NewSource(time.Now().UnixNano())) - for { - mark := time.Now() - if mark.Before(until) { - h := c.Get(0, uint64(r.Intn(maxkey)), nil) - latency := int64(time.Now().Sub(mark)) - m := atomic.LoadInt64(&getMaxLatency) - if latency > m { - atomic.CompareAndSwapInt64(&getMaxLatency, m, latency) - } - atomic.AddInt64(&getDuration, latency) - if h != nil { - atomic.AddInt32(&getHit, 1) - h.Release() - } - atomic.AddInt32(&getAll, 1) - } else { - break - } - } - }(i) - } - - wg.Wait() - getAvglatency := time.Duration(getDuration) / time.Duration(getAll) - t.Logf("set=%d getHit=%d getAll=%d getMaxLatency=%v getAvgLatency=%v", - set, getHit, getAll, time.Duration(getMaxLatency), getAvglatency) - - if getAvglatency > delay/3 { - t.Errorf("get avg latency > %v: got=%v", delay/3, getAvglatency) - } -} - -func TestLRUCache_HitMiss(t *testing.T) { - cases := []struct { - key uint64 - value string - }{ - {1, "vvvvvvvvv"}, - {100, "v1"}, - {0, "v2"}, - {12346, "v3"}, - {777, "v4"}, - {999, "v5"}, - {7654, "v6"}, - {2, "v7"}, - {3, "v8"}, - {9, "v9"}, - } - - setfin := 0 - c := NewCache(NewLRU(1000)) - for i, x := range cases { - set(c, 0, x.key, x.value, len(x.value), func() { - setfin++ - }).Release() - for j, y := range cases { - h := c.Get(0, y.key, nil) - if j <= i { - // should hit - if h == nil { - t.Errorf("case '%d' iteration '%d' is miss", i, j) - } else { - if x := h.Value().(releaserFunc).value.(string); x != y.value { - t.Errorf("case '%d' iteration '%d' has invalid value got '%s', want '%s'", i, j, x, y.value) - } - } - } else { - // should miss - if h != nil { - t.Errorf("case '%d' iteration '%d' is hit , value '%s'", i, j, h.Value().(releaserFunc).value.(string)) - } - } - if h != nil { - h.Release() - } - } - } - - for i, x := range cases { - finalizerOk := false - c.Delete(0, x.key, func() { - finalizerOk = true - }) - - if !finalizerOk { - t.Errorf("case %d delete finalizer not executed", i) - } - - for j, y := range cases { - h := c.Get(0, y.key, nil) - if j > i { - // should hit - if h == nil { - t.Errorf("case '%d' iteration '%d' is miss", i, j) - } else { - if x := h.Value().(releaserFunc).value.(string); x != y.value { - t.Errorf("case '%d' iteration '%d' has invalid value got '%s', want '%s'", i, j, x, y.value) - } - } - } else { - // should miss - if h != nil { - t.Errorf("case '%d' iteration '%d' is hit, value '%s'", i, j, h.Value().(releaserFunc).value.(string)) - } - } - if h != nil { - h.Release() - } - } - } - - if setfin != len(cases) { - t.Errorf("some set finalizer may not be executed, want=%d got=%d", len(cases), setfin) - } -} - -func TestLRUCache_Eviction(t *testing.T) { - c := NewCache(NewLRU(12)) - o1 := set(c, 0, 1, 1, 1, nil) - set(c, 0, 2, 2, 1, nil).Release() - set(c, 0, 3, 3, 1, nil).Release() - set(c, 0, 4, 4, 1, nil).Release() - set(c, 0, 5, 5, 1, nil).Release() - if h := c.Get(0, 2, nil); h != nil { // 1,3,4,5,2 - h.Release() - } - set(c, 0, 9, 9, 10, nil).Release() // 5,2,9 - - for _, key := range []uint64{9, 2, 5, 1} { - h := c.Get(0, key, nil) - if h == nil { - t.Errorf("miss for key '%d'", key) - } else { - if x := h.Value().(int); x != int(key) { - t.Errorf("invalid value for key '%d' want '%d', got '%d'", key, key, x) - } - h.Release() - } - } - o1.Release() - for _, key := range []uint64{1, 2, 5} { - h := c.Get(0, key, nil) - if h == nil { - t.Errorf("miss for key '%d'", key) - } else { - if x := h.Value().(int); x != int(key) { - t.Errorf("invalid value for key '%d' want '%d', got '%d'", key, key, x) - } - h.Release() - } - } - for _, key := range []uint64{3, 4, 9} { - h := c.Get(0, key, nil) - if h != nil { - t.Errorf("hit for key '%d'", key) - if x := h.Value().(int); x != int(key) { - t.Errorf("invalid value for key '%d' want '%d', got '%d'", key, key, x) - } - h.Release() - } - } -} - -func TestLRUCache_Evict(t *testing.T) { - c := NewCache(NewLRU(6)) - set(c, 0, 1, 1, 1, nil).Release() - set(c, 0, 2, 2, 1, nil).Release() - set(c, 1, 1, 4, 1, nil).Release() - set(c, 1, 2, 5, 1, nil).Release() - set(c, 2, 1, 6, 1, nil).Release() - set(c, 2, 2, 7, 1, nil).Release() - - for ns := 0; ns < 3; ns++ { - for key := 1; key < 3; key++ { - if h := c.Get(uint64(ns), uint64(key), nil); h != nil { - h.Release() - } else { - t.Errorf("Cache.Get on #%d.%d return nil", ns, key) - } - } - } - - if ok := c.Evict(0, 1); !ok { - t.Error("first Cache.Evict on #0.1 return false") - } - if ok := c.Evict(0, 1); ok { - t.Error("second Cache.Evict on #0.1 return true") - } - if h := c.Get(0, 1, nil); h != nil { - t.Errorf("Cache.Get on #0.1 return non-nil: %v", h.Value()) - } - - c.EvictNS(1) - if h := c.Get(1, 1, nil); h != nil { - t.Errorf("Cache.Get on #1.1 return non-nil: %v", h.Value()) - } - if h := c.Get(1, 2, nil); h != nil { - t.Errorf("Cache.Get on #1.2 return non-nil: %v", h.Value()) - } - - c.EvictAll() - for ns := 0; ns < 3; ns++ { - for key := 1; key < 3; key++ { - if h := c.Get(uint64(ns), uint64(key), nil); h != nil { - t.Errorf("Cache.Get on #%d.%d return non-nil: %v", ns, key, h.Value()) - } - } - } -} - -func TestLRUCache_Delete(t *testing.T) { - delFuncCalled := 0 - delFunc := func() { - delFuncCalled++ - } - - c := NewCache(NewLRU(2)) - set(c, 0, 1, 1, 1, nil).Release() - set(c, 0, 2, 2, 1, nil).Release() - - if ok := c.Delete(0, 1, delFunc); !ok { - t.Error("Cache.Delete on #1 return false") - } - if h := c.Get(0, 1, nil); h != nil { - t.Errorf("Cache.Get on #1 return non-nil: %v", h.Value()) - } - if ok := c.Delete(0, 1, delFunc); ok { - t.Error("Cache.Delete on #1 return true") - } - - h2 := c.Get(0, 2, nil) - if h2 == nil { - t.Error("Cache.Get on #2 return nil") - } - if ok := c.Delete(0, 2, delFunc); !ok { - t.Error("(1) Cache.Delete on #2 return false") - } - if ok := c.Delete(0, 2, delFunc); !ok { - t.Error("(2) Cache.Delete on #2 return false") - } - - set(c, 0, 3, 3, 1, nil).Release() - set(c, 0, 4, 4, 1, nil).Release() - c.Get(0, 2, nil).Release() - - for key := 2; key <= 4; key++ { - if h := c.Get(0, uint64(key), nil); h != nil { - h.Release() - } else { - t.Errorf("Cache.Get on #%d return nil", key) - } - } - - h2.Release() - if h := c.Get(0, 2, nil); h != nil { - t.Errorf("Cache.Get on #2 return non-nil: %v", h.Value()) - } - - if delFuncCalled != 4 { - t.Errorf("delFunc isn't called 4 times: got=%d", delFuncCalled) - } -} - -func TestLRUCache_Close(t *testing.T) { - relFuncCalled := 0 - relFunc := func() { - relFuncCalled++ - } - delFuncCalled := 0 - delFunc := func() { - delFuncCalled++ - } - - c := NewCache(NewLRU(2)) - set(c, 0, 1, 1, 1, relFunc).Release() - set(c, 0, 2, 2, 1, relFunc).Release() - - h3 := set(c, 0, 3, 3, 1, relFunc) - if h3 == nil { - t.Error("Cache.Get on #3 return nil") - } - if ok := c.Delete(0, 3, delFunc); !ok { - t.Error("Cache.Delete on #3 return false") - } - - c.Close() - - if relFuncCalled != 3 { - t.Errorf("relFunc isn't called 3 times: got=%d", relFuncCalled) - } - if delFuncCalled != 1 { - t.Errorf("delFunc isn't called 1 times: got=%d", delFuncCalled) - } -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/cache/lru.go b/vendor/github.com/syndtr/goleveldb/leveldb/cache/lru.go deleted file mode 100644 index d9a84cd..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/cache/lru.go +++ /dev/null @@ -1,195 +0,0 @@ -// Copyright (c) 2012, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package cache - -import ( - "sync" - "unsafe" -) - -type lruNode struct { - n *Node - h *Handle - ban bool - - next, prev *lruNode -} - -func (n *lruNode) insert(at *lruNode) { - x := at.next - at.next = n - n.prev = at - n.next = x - x.prev = n -} - -func (n *lruNode) remove() { - if n.prev != nil { - n.prev.next = n.next - n.next.prev = n.prev - n.prev = nil - n.next = nil - } else { - panic("BUG: removing removed node") - } -} - -type lru struct { - mu sync.Mutex - capacity int - used int - recent lruNode -} - -func (r *lru) reset() { - r.recent.next = &r.recent - r.recent.prev = &r.recent - r.used = 0 -} - -func (r *lru) Capacity() int { - r.mu.Lock() - defer r.mu.Unlock() - return r.capacity -} - -func (r *lru) SetCapacity(capacity int) { - var evicted []*lruNode - - r.mu.Lock() - r.capacity = capacity - for r.used > r.capacity { - rn := r.recent.prev - if rn == nil { - panic("BUG: invalid LRU used or capacity counter") - } - rn.remove() - rn.n.CacheData = nil - r.used -= rn.n.Size() - evicted = append(evicted, rn) - } - r.mu.Unlock() - - for _, rn := range evicted { - rn.h.Release() - } -} - -func (r *lru) Promote(n *Node) { - var evicted []*lruNode - - r.mu.Lock() - if n.CacheData == nil { - if n.Size() <= r.capacity { - rn := &lruNode{n: n, h: n.GetHandle()} - rn.insert(&r.recent) - n.CacheData = unsafe.Pointer(rn) - r.used += n.Size() - - for r.used > r.capacity { - rn := r.recent.prev - if rn == nil { - panic("BUG: invalid LRU used or capacity counter") - } - rn.remove() - rn.n.CacheData = nil - r.used -= rn.n.Size() - evicted = append(evicted, rn) - } - } - } else { - rn := (*lruNode)(n.CacheData) - if !rn.ban { - rn.remove() - rn.insert(&r.recent) - } - } - r.mu.Unlock() - - for _, rn := range evicted { - rn.h.Release() - } -} - -func (r *lru) Ban(n *Node) { - r.mu.Lock() - if n.CacheData == nil { - n.CacheData = unsafe.Pointer(&lruNode{n: n, ban: true}) - } else { - rn := (*lruNode)(n.CacheData) - if !rn.ban { - rn.remove() - rn.ban = true - r.used -= rn.n.Size() - r.mu.Unlock() - - rn.h.Release() - rn.h = nil - return - } - } - r.mu.Unlock() -} - -func (r *lru) Evict(n *Node) { - r.mu.Lock() - rn := (*lruNode)(n.CacheData) - if rn == nil || rn.ban { - r.mu.Unlock() - return - } - n.CacheData = nil - r.mu.Unlock() - - rn.h.Release() -} - -func (r *lru) EvictNS(ns uint64) { - var evicted []*lruNode - - r.mu.Lock() - for e := r.recent.prev; e != &r.recent; { - rn := e - e = e.prev - if rn.n.NS() == ns { - rn.remove() - rn.n.CacheData = nil - r.used -= rn.n.Size() - evicted = append(evicted, rn) - } - } - r.mu.Unlock() - - for _, rn := range evicted { - rn.h.Release() - } -} - -func (r *lru) EvictAll() { - r.mu.Lock() - back := r.recent.prev - for rn := back; rn != &r.recent; rn = rn.prev { - rn.n.CacheData = nil - } - r.reset() - r.mu.Unlock() - - for rn := back; rn != &r.recent; rn = rn.prev { - rn.h.Release() - } -} - -func (r *lru) Close() error { - return nil -} - -// NewLRU create a new LRU-cache. -func NewLRU(capacity int) Cacher { - r := &lru{capacity: capacity} - r.reset() - return r -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/comparer.go b/vendor/github.com/syndtr/goleveldb/leveldb/comparer.go deleted file mode 100644 index 448402b..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/comparer.go +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright (c) 2012, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package leveldb - -import ( - "github.com/syndtr/goleveldb/leveldb/comparer" -) - -type iComparer struct { - ucmp comparer.Comparer -} - -func (icmp *iComparer) uName() string { - return icmp.ucmp.Name() -} - -func (icmp *iComparer) uCompare(a, b []byte) int { - return icmp.ucmp.Compare(a, b) -} - -func (icmp *iComparer) uSeparator(dst, a, b []byte) []byte { - return icmp.ucmp.Separator(dst, a, b) -} - -func (icmp *iComparer) uSuccessor(dst, b []byte) []byte { - return icmp.ucmp.Successor(dst, b) -} - -func (icmp *iComparer) Name() string { - return icmp.uName() -} - -func (icmp *iComparer) Compare(a, b []byte) int { - x := icmp.uCompare(internalKey(a).ukey(), internalKey(b).ukey()) - if x == 0 { - if m, n := internalKey(a).num(), internalKey(b).num(); m > n { - return -1 - } else if m < n { - return 1 - } - } - return x -} - -func (icmp *iComparer) Separator(dst, a, b []byte) []byte { - ua, ub := internalKey(a).ukey(), internalKey(b).ukey() - dst = icmp.uSeparator(dst, ua, ub) - if dst != nil && len(dst) < len(ua) && icmp.uCompare(ua, dst) < 0 { - // Append earliest possible number. - return append(dst, keyMaxNumBytes...) - } - return nil -} - -func (icmp *iComparer) Successor(dst, b []byte) []byte { - ub := internalKey(b).ukey() - dst = icmp.uSuccessor(dst, ub) - if dst != nil && len(dst) < len(ub) && icmp.uCompare(ub, dst) < 0 { - // Append earliest possible number. - return append(dst, keyMaxNumBytes...) - } - return nil -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/comparer/bytes_comparer.go b/vendor/github.com/syndtr/goleveldb/leveldb/comparer/bytes_comparer.go deleted file mode 100644 index 14dddf8..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/comparer/bytes_comparer.go +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright (c) 2012, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package comparer - -import "bytes" - -type bytesComparer struct{} - -func (bytesComparer) Compare(a, b []byte) int { - return bytes.Compare(a, b) -} - -func (bytesComparer) Name() string { - return "leveldb.BytewiseComparator" -} - -func (bytesComparer) Separator(dst, a, b []byte) []byte { - i, n := 0, len(a) - if n > len(b) { - n = len(b) - } - for ; i < n && a[i] == b[i]; i++ { - } - if i >= n { - // Do not shorten if one string is a prefix of the other - } else if c := a[i]; c < 0xff && c+1 < b[i] { - dst = append(dst, a[:i+1]...) - dst[i]++ - return dst - } - return nil -} - -func (bytesComparer) Successor(dst, b []byte) []byte { - for i, c := range b { - if c != 0xff { - dst = append(dst, b[:i+1]...) - dst[i]++ - return dst - } - } - return nil -} - -// DefaultComparer are default implementation of the Comparer interface. -// It uses the natural ordering, consistent with bytes.Compare. -var DefaultComparer = bytesComparer{} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/comparer/comparer.go b/vendor/github.com/syndtr/goleveldb/leveldb/comparer/comparer.go deleted file mode 100644 index 14a28f1..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/comparer/comparer.go +++ /dev/null @@ -1,57 +0,0 @@ -// Copyright (c) 2012, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -// Package comparer provides interface and implementation for ordering -// sets of data. -package comparer - -// BasicComparer is the interface that wraps the basic Compare method. -type BasicComparer interface { - // Compare returns -1, 0, or +1 depending on whether a is 'less than', - // 'equal to' or 'greater than' b. The two arguments can only be 'equal' - // if their contents are exactly equal. Furthermore, the empty slice - // must be 'less than' any non-empty slice. - Compare(a, b []byte) int -} - -// Comparer defines a total ordering over the space of []byte keys: a 'less -// than' relationship. -type Comparer interface { - BasicComparer - - // Name returns name of the comparer. - // - // The Level-DB on-disk format stores the comparer name, and opening a - // database with a different comparer from the one it was created with - // will result in an error. - // - // An implementation to a new name whenever the comparer implementation - // changes in a way that will cause the relative ordering of any two keys - // to change. - // - // Names starting with "leveldb." are reserved and should not be used - // by any users of this package. - Name() string - - // Bellow are advanced functions used used to reduce the space requirements - // for internal data structures such as index blocks. - - // Separator appends a sequence of bytes x to dst such that a <= x && x < b, - // where 'less than' is consistent with Compare. An implementation should - // return nil if x equal to a. - // - // Either contents of a or b should not by any means modified. Doing so - // may cause corruption on the internal state. - Separator(dst, a, b []byte) []byte - - // Successor appends a sequence of bytes x to dst such that x >= b, where - // 'less than' is consistent with Compare. An implementation should return - // nil if x equal to b. - // - // Contents of b should not by any means modified. Doing so may cause - // corruption on the internal state. - Successor(dst, b []byte) []byte -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/corrupt_test.go b/vendor/github.com/syndtr/goleveldb/leveldb/corrupt_test.go deleted file mode 100644 index fef2026..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/corrupt_test.go +++ /dev/null @@ -1,496 +0,0 @@ -// Copyright (c) 2013, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package leveldb - -import ( - "bytes" - "fmt" - "io" - "math/rand" - "testing" - - "github.com/syndtr/goleveldb/leveldb/filter" - "github.com/syndtr/goleveldb/leveldb/opt" - "github.com/syndtr/goleveldb/leveldb/storage" -) - -const ctValSize = 1000 - -type dbCorruptHarness struct { - dbHarness -} - -func newDbCorruptHarnessWopt(t *testing.T, o *opt.Options) *dbCorruptHarness { - h := new(dbCorruptHarness) - h.init(t, o) - return h -} - -func newDbCorruptHarness(t *testing.T) *dbCorruptHarness { - return newDbCorruptHarnessWopt(t, &opt.Options{ - BlockCacheCapacity: 100, - Strict: opt.StrictJournalChecksum, - }) -} - -func (h *dbCorruptHarness) recover() { - p := &h.dbHarness - t := p.t - - var err error - p.db, err = Recover(h.stor, h.o) - if err != nil { - t.Fatal("Repair: got error: ", err) - } -} - -func (h *dbCorruptHarness) build(n int) { - p := &h.dbHarness - t := p.t - db := p.db - - batch := new(Batch) - for i := 0; i < n; i++ { - batch.Reset() - batch.Put(tkey(i), tval(i, ctValSize)) - err := db.Write(batch, p.wo) - if err != nil { - t.Fatal("write error: ", err) - } - } -} - -func (h *dbCorruptHarness) buildShuffled(n int, rnd *rand.Rand) { - p := &h.dbHarness - t := p.t - db := p.db - - batch := new(Batch) - for i := range rnd.Perm(n) { - batch.Reset() - batch.Put(tkey(i), tval(i, ctValSize)) - err := db.Write(batch, p.wo) - if err != nil { - t.Fatal("write error: ", err) - } - } -} - -func (h *dbCorruptHarness) deleteRand(n, max int, rnd *rand.Rand) { - p := &h.dbHarness - t := p.t - db := p.db - - batch := new(Batch) - for i := 0; i < n; i++ { - batch.Reset() - batch.Delete(tkey(rnd.Intn(max))) - err := db.Write(batch, p.wo) - if err != nil { - t.Fatal("write error: ", err) - } - } -} - -func (h *dbCorruptHarness) corrupt(ft storage.FileType, fi, offset, n int) { - p := &h.dbHarness - t := p.t - - fds, _ := p.stor.List(ft) - sortFds(fds) - if fi < 0 { - fi = len(fds) - 1 - } - if fi >= len(fds) { - t.Fatalf("no such file with type %q with index %d", ft, fi) - } - - fd := fds[fi] - r, err := h.stor.Open(fd) - if err != nil { - t.Fatal("cannot open file: ", err) - } - x, err := r.Seek(0, 2) - if err != nil { - t.Fatal("cannot query file size: ", err) - } - m := int(x) - if _, err := r.Seek(0, 0); err != nil { - t.Fatal(err) - } - - if offset < 0 { - if -offset > m { - offset = 0 - } else { - offset = m + offset - } - } - if offset > m { - offset = m - } - if offset+n > m { - n = m - offset - } - - buf := make([]byte, m) - _, err = io.ReadFull(r, buf) - if err != nil { - t.Fatal("cannot read file: ", err) - } - r.Close() - - for i := 0; i < n; i++ { - buf[offset+i] ^= 0x80 - } - - err = h.stor.Remove(fd) - if err != nil { - t.Fatal("cannot remove old file: ", err) - } - w, err := h.stor.Create(fd) - if err != nil { - t.Fatal("cannot create new file: ", err) - } - _, err = w.Write(buf) - if err != nil { - t.Fatal("cannot write new file: ", err) - } - w.Close() -} - -func (h *dbCorruptHarness) removeAll(ft storage.FileType) { - fds, err := h.stor.List(ft) - if err != nil { - h.t.Fatal("get files: ", err) - } - for _, fd := range fds { - if err := h.stor.Remove(fd); err != nil { - h.t.Error("remove file: ", err) - } - } -} - -func (h *dbCorruptHarness) forceRemoveAll(ft storage.FileType) { - fds, err := h.stor.List(ft) - if err != nil { - h.t.Fatal("get files: ", err) - } - for _, fd := range fds { - if err := h.stor.ForceRemove(fd); err != nil { - h.t.Error("remove file: ", err) - } - } -} - -func (h *dbCorruptHarness) removeOne(ft storage.FileType) { - fds, err := h.stor.List(ft) - if err != nil { - h.t.Fatal("get files: ", err) - } - fd := fds[rand.Intn(len(fds))] - h.t.Logf("removing file @%d", fd.Num) - if err := h.stor.Remove(fd); err != nil { - h.t.Error("remove file: ", err) - } -} - -func (h *dbCorruptHarness) check(min, max int) { - p := &h.dbHarness - t := p.t - db := p.db - - var n, badk, badv, missed, good int - iter := db.NewIterator(nil, p.ro) - for iter.Next() { - k := 0 - fmt.Sscanf(string(iter.Key()), "%d", &k) - if k < n { - badk++ - continue - } - missed += k - n - n = k + 1 - if !bytes.Equal(iter.Value(), tval(k, ctValSize)) { - badv++ - } else { - good++ - } - } - err := iter.Error() - iter.Release() - t.Logf("want=%d..%d got=%d badkeys=%d badvalues=%d missed=%d, err=%v", - min, max, good, badk, badv, missed, err) - if good < min || good > max { - t.Errorf("good entries number not in range") - } -} - -func TestCorruptDB_Journal(t *testing.T) { - h := newDbCorruptHarness(t) - defer h.close() - - h.build(100) - h.check(100, 100) - h.closeDB() - h.corrupt(storage.TypeJournal, -1, 19, 1) - h.corrupt(storage.TypeJournal, -1, 32*1024+1000, 1) - - h.openDB() - h.check(36, 36) -} - -func TestCorruptDB_Table(t *testing.T) { - h := newDbCorruptHarness(t) - defer h.close() - - h.build(100) - h.compactMem() - h.compactRangeAt(0, "", "") - h.compactRangeAt(1, "", "") - h.closeDB() - h.corrupt(storage.TypeTable, -1, 100, 1) - - h.openDB() - h.check(99, 99) -} - -func TestCorruptDB_TableIndex(t *testing.T) { - h := newDbCorruptHarness(t) - defer h.close() - - h.build(10000) - h.compactMem() - h.closeDB() - h.corrupt(storage.TypeTable, -1, -2000, 500) - - h.openDB() - h.check(5000, 9999) -} - -func TestCorruptDB_MissingManifest(t *testing.T) { - rnd := rand.New(rand.NewSource(0x0badda7a)) - h := newDbCorruptHarnessWopt(t, &opt.Options{ - BlockCacheCapacity: 100, - Strict: opt.StrictJournalChecksum, - WriteBuffer: 1000 * 60, - }) - defer h.close() - - h.build(1000) - h.compactMem() - h.buildShuffled(1000, rnd) - h.compactMem() - h.deleteRand(500, 1000, rnd) - h.compactMem() - h.buildShuffled(1000, rnd) - h.compactMem() - h.deleteRand(500, 1000, rnd) - h.compactMem() - h.buildShuffled(1000, rnd) - h.compactMem() - h.closeDB() - - h.forceRemoveAll(storage.TypeManifest) - h.openAssert(false) - - h.recover() - h.check(1000, 1000) - h.build(1000) - h.compactMem() - h.compactRange("", "") - h.closeDB() - - h.recover() - h.check(1000, 1000) -} - -func TestCorruptDB_SequenceNumberRecovery(t *testing.T) { - h := newDbCorruptHarness(t) - defer h.close() - - h.put("foo", "v1") - h.put("foo", "v2") - h.put("foo", "v3") - h.put("foo", "v4") - h.put("foo", "v5") - h.closeDB() - - h.recover() - h.getVal("foo", "v5") - h.put("foo", "v6") - h.getVal("foo", "v6") - - h.reopenDB() - h.getVal("foo", "v6") -} - -func TestCorruptDB_SequenceNumberRecoveryTable(t *testing.T) { - h := newDbCorruptHarness(t) - defer h.close() - - h.put("foo", "v1") - h.put("foo", "v2") - h.put("foo", "v3") - h.compactMem() - h.put("foo", "v4") - h.put("foo", "v5") - h.compactMem() - h.closeDB() - - h.recover() - h.getVal("foo", "v5") - h.put("foo", "v6") - h.getVal("foo", "v6") - - h.reopenDB() - h.getVal("foo", "v6") -} - -func TestCorruptDB_CorruptedManifest(t *testing.T) { - h := newDbCorruptHarness(t) - defer h.close() - - h.put("foo", "hello") - h.compactMem() - h.compactRange("", "") - h.closeDB() - h.corrupt(storage.TypeManifest, -1, 0, 1000) - h.openAssert(false) - - h.recover() - h.getVal("foo", "hello") -} - -func TestCorruptDB_CompactionInputError(t *testing.T) { - h := newDbCorruptHarness(t) - defer h.close() - - h.build(10) - h.compactMem() - h.closeDB() - h.corrupt(storage.TypeTable, -1, 100, 1) - - h.openDB() - h.check(9, 9) - - h.build(10000) - h.check(10000, 10000) -} - -func TestCorruptDB_UnrelatedKeys(t *testing.T) { - h := newDbCorruptHarness(t) - defer h.close() - - h.build(10) - h.compactMem() - h.closeDB() - h.corrupt(storage.TypeTable, -1, 100, 1) - - h.openDB() - h.put(string(tkey(1000)), string(tval(1000, ctValSize))) - h.getVal(string(tkey(1000)), string(tval(1000, ctValSize))) - h.compactMem() - h.getVal(string(tkey(1000)), string(tval(1000, ctValSize))) -} - -func TestCorruptDB_Level0NewerFileHasOlderSeqnum(t *testing.T) { - h := newDbCorruptHarness(t) - defer h.close() - - h.put("a", "v1") - h.put("b", "v1") - h.compactMem() - h.put("a", "v2") - h.put("b", "v2") - h.compactMem() - h.put("a", "v3") - h.put("b", "v3") - h.compactMem() - h.put("c", "v0") - h.put("d", "v0") - h.compactMem() - h.compactRangeAt(1, "", "") - h.closeDB() - - h.recover() - h.getVal("a", "v3") - h.getVal("b", "v3") - h.getVal("c", "v0") - h.getVal("d", "v0") -} - -func TestCorruptDB_RecoverInvalidSeq_Issue53(t *testing.T) { - h := newDbCorruptHarness(t) - defer h.close() - - h.put("a", "v1") - h.put("b", "v1") - h.compactMem() - h.put("a", "v2") - h.put("b", "v2") - h.compactMem() - h.put("a", "v3") - h.put("b", "v3") - h.compactMem() - h.put("c", "v0") - h.put("d", "v0") - h.compactMem() - h.compactRangeAt(0, "", "") - h.closeDB() - - h.recover() - h.getVal("a", "v3") - h.getVal("b", "v3") - h.getVal("c", "v0") - h.getVal("d", "v0") -} - -func TestCorruptDB_MissingTableFiles(t *testing.T) { - h := newDbCorruptHarness(t) - defer h.close() - - h.put("a", "v1") - h.put("b", "v1") - h.compactMem() - h.put("c", "v2") - h.put("d", "v2") - h.compactMem() - h.put("e", "v3") - h.put("f", "v3") - h.closeDB() - - h.removeOne(storage.TypeTable) - h.openAssert(false) -} - -func TestCorruptDB_RecoverTable(t *testing.T) { - h := newDbCorruptHarnessWopt(t, &opt.Options{ - WriteBuffer: 112 * opt.KiB, - CompactionTableSize: 90 * opt.KiB, - Filter: filter.NewBloomFilter(10), - }) - defer h.close() - - h.build(1000) - h.compactMem() - h.compactRangeAt(0, "", "") - h.compactRangeAt(1, "", "") - seq := h.db.seq - h.closeDB() - h.corrupt(storage.TypeTable, 0, 1000, 1) - h.corrupt(storage.TypeTable, 3, 10000, 1) - // Corrupted filter shouldn't affect recovery. - h.corrupt(storage.TypeTable, 3, 113888, 10) - h.corrupt(storage.TypeTable, -1, 20000, 1) - - h.recover() - if h.db.seq != seq { - t.Errorf("invalid seq, want=%d got=%d", seq, h.db.seq) - } - h.check(985, 985) -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/db.go b/vendor/github.com/syndtr/goleveldb/leveldb/db.go deleted file mode 100644 index a02cb2c..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/db.go +++ /dev/null @@ -1,1091 +0,0 @@ -// Copyright (c) 2012, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package leveldb - -import ( - "container/list" - "fmt" - "io" - "os" - "runtime" - "strings" - "sync" - "sync/atomic" - "time" - - "github.com/syndtr/goleveldb/leveldb/errors" - "github.com/syndtr/goleveldb/leveldb/iterator" - "github.com/syndtr/goleveldb/leveldb/journal" - "github.com/syndtr/goleveldb/leveldb/memdb" - "github.com/syndtr/goleveldb/leveldb/opt" - "github.com/syndtr/goleveldb/leveldb/storage" - "github.com/syndtr/goleveldb/leveldb/table" - "github.com/syndtr/goleveldb/leveldb/util" -) - -// DB is a LevelDB database. -type DB struct { - // Need 64-bit alignment. - seq uint64 - - // Session. - s *session - - // MemDB. - memMu sync.RWMutex - memPool chan *memdb.DB - mem, frozenMem *memDB - journal *journal.Writer - journalWriter storage.Writer - journalFd storage.FileDesc - frozenJournalFd storage.FileDesc - frozenSeq uint64 - - // Snapshot. - snapsMu sync.Mutex - snapsList *list.List - - // Stats. - aliveSnaps, aliveIters int32 - - // Write. - batchPool sync.Pool - writeMergeC chan writeMerge - writeMergedC chan bool - writeLockC chan struct{} - writeAckC chan error - writeDelay time.Duration - writeDelayN int - tr *Transaction - - // Compaction. - compCommitLk sync.Mutex - tcompCmdC chan cCmd - tcompPauseC chan chan<- struct{} - mcompCmdC chan cCmd - compErrC chan error - compPerErrC chan error - compErrSetC chan error - compWriteLocking bool - compStats cStats - memdbMaxLevel int // For testing. - - // Close. - closeW sync.WaitGroup - closeC chan struct{} - closed uint32 - closer io.Closer -} - -func openDB(s *session) (*DB, error) { - s.log("db@open opening") - start := time.Now() - db := &DB{ - s: s, - // Initial sequence - seq: s.stSeqNum, - // MemDB - memPool: make(chan *memdb.DB, 1), - // Snapshot - snapsList: list.New(), - // Write - batchPool: sync.Pool{New: newBatch}, - writeMergeC: make(chan writeMerge), - writeMergedC: make(chan bool), - writeLockC: make(chan struct{}, 1), - writeAckC: make(chan error), - // Compaction - tcompCmdC: make(chan cCmd), - tcompPauseC: make(chan chan<- struct{}), - mcompCmdC: make(chan cCmd), - compErrC: make(chan error), - compPerErrC: make(chan error), - compErrSetC: make(chan error), - // Close - closeC: make(chan struct{}), - } - - // Read-only mode. - readOnly := s.o.GetReadOnly() - - if readOnly { - // Recover journals (read-only mode). - if err := db.recoverJournalRO(); err != nil { - return nil, err - } - } else { - // Recover journals. - if err := db.recoverJournal(); err != nil { - return nil, err - } - - // Remove any obsolete files. - if err := db.checkAndCleanFiles(); err != nil { - // Close journal. - if db.journal != nil { - db.journal.Close() - db.journalWriter.Close() - } - return nil, err - } - - } - - // Doesn't need to be included in the wait group. - go db.compactionError() - go db.mpoolDrain() - - if readOnly { - db.SetReadOnly() - } else { - db.closeW.Add(2) - go db.tCompaction() - go db.mCompaction() - // go db.jWriter() - } - - s.logf("db@open done T·%v", time.Since(start)) - - runtime.SetFinalizer(db, (*DB).Close) - return db, nil -} - -// Open opens or creates a DB for the given storage. -// The DB will be created if not exist, unless ErrorIfMissing is true. -// Also, if ErrorIfExist is true and the DB exist Open will returns -// os.ErrExist error. -// -// Open will return an error with type of ErrCorrupted if corruption -// detected in the DB. Use errors.IsCorrupted to test whether an error is -// due to corruption. Corrupted DB can be recovered with Recover function. -// -// The returned DB instance is safe for concurrent use. -// The DB must be closed after use, by calling Close method. -func Open(stor storage.Storage, o *opt.Options) (db *DB, err error) { - s, err := newSession(stor, o) - if err != nil { - return - } - defer func() { - if err != nil { - s.close() - s.release() - } - }() - - err = s.recover() - if err != nil { - if !os.IsNotExist(err) || s.o.GetErrorIfMissing() { - return - } - err = s.create() - if err != nil { - return - } - } else if s.o.GetErrorIfExist() { - err = os.ErrExist - return - } - - return openDB(s) -} - -// OpenFile opens or creates a DB for the given path. -// The DB will be created if not exist, unless ErrorIfMissing is true. -// Also, if ErrorIfExist is true and the DB exist OpenFile will returns -// os.ErrExist error. -// -// OpenFile uses standard file-system backed storage implementation as -// described in the leveldb/storage package. -// -// OpenFile will return an error with type of ErrCorrupted if corruption -// detected in the DB. Use errors.IsCorrupted to test whether an error is -// due to corruption. Corrupted DB can be recovered with Recover function. -// -// The returned DB instance is safe for concurrent use. -// The DB must be closed after use, by calling Close method. -func OpenFile(path string, o *opt.Options) (db *DB, err error) { - stor, err := storage.OpenFile(path, o.GetReadOnly()) - if err != nil { - return - } - db, err = Open(stor, o) - if err != nil { - stor.Close() - } else { - db.closer = stor - } - return -} - -// Recover recovers and opens a DB with missing or corrupted manifest files -// for the given storage. It will ignore any manifest files, valid or not. -// The DB must already exist or it will returns an error. -// Also, Recover will ignore ErrorIfMissing and ErrorIfExist options. -// -// The returned DB instance is safe for concurrent use. -// The DB must be closed after use, by calling Close method. -func Recover(stor storage.Storage, o *opt.Options) (db *DB, err error) { - s, err := newSession(stor, o) - if err != nil { - return - } - defer func() { - if err != nil { - s.close() - s.release() - } - }() - - err = recoverTable(s, o) - if err != nil { - return - } - return openDB(s) -} - -// RecoverFile recovers and opens a DB with missing or corrupted manifest files -// for the given path. It will ignore any manifest files, valid or not. -// The DB must already exist or it will returns an error. -// Also, Recover will ignore ErrorIfMissing and ErrorIfExist options. -// -// RecoverFile uses standard file-system backed storage implementation as described -// in the leveldb/storage package. -// -// The returned DB instance is safe for concurrent use. -// The DB must be closed after use, by calling Close method. -func RecoverFile(path string, o *opt.Options) (db *DB, err error) { - stor, err := storage.OpenFile(path, false) - if err != nil { - return - } - db, err = Recover(stor, o) - if err != nil { - stor.Close() - } else { - db.closer = stor - } - return -} - -func recoverTable(s *session, o *opt.Options) error { - o = dupOptions(o) - // Mask StrictReader, lets StrictRecovery doing its job. - o.Strict &= ^opt.StrictReader - - // Get all tables and sort it by file number. - fds, err := s.stor.List(storage.TypeTable) - if err != nil { - return err - } - sortFds(fds) - - var ( - maxSeq uint64 - recoveredKey, goodKey, corruptedKey, corruptedBlock, droppedTable int - - // We will drop corrupted table. - strict = o.GetStrict(opt.StrictRecovery) - noSync = o.GetNoSync() - - rec = &sessionRecord{} - bpool = util.NewBufferPool(o.GetBlockSize() + 5) - ) - buildTable := func(iter iterator.Iterator) (tmpFd storage.FileDesc, size int64, err error) { - tmpFd = s.newTemp() - writer, err := s.stor.Create(tmpFd) - if err != nil { - return - } - defer func() { - writer.Close() - if err != nil { - s.stor.Remove(tmpFd) - tmpFd = storage.FileDesc{} - } - }() - - // Copy entries. - tw := table.NewWriter(writer, o) - for iter.Next() { - key := iter.Key() - if validInternalKey(key) { - err = tw.Append(key, iter.Value()) - if err != nil { - return - } - } - } - err = iter.Error() - if err != nil { - return - } - err = tw.Close() - if err != nil { - return - } - if !noSync { - err = writer.Sync() - if err != nil { - return - } - } - size = int64(tw.BytesLen()) - return - } - recoverTable := func(fd storage.FileDesc) error { - s.logf("table@recovery recovering @%d", fd.Num) - reader, err := s.stor.Open(fd) - if err != nil { - return err - } - var closed bool - defer func() { - if !closed { - reader.Close() - } - }() - - // Get file size. - size, err := reader.Seek(0, 2) - if err != nil { - return err - } - - var ( - tSeq uint64 - tgoodKey, tcorruptedKey, tcorruptedBlock int - imin, imax []byte - ) - tr, err := table.NewReader(reader, size, fd, nil, bpool, o) - if err != nil { - return err - } - iter := tr.NewIterator(nil, nil) - if itererr, ok := iter.(iterator.ErrorCallbackSetter); ok { - itererr.SetErrorCallback(func(err error) { - if errors.IsCorrupted(err) { - s.logf("table@recovery block corruption @%d %q", fd.Num, err) - tcorruptedBlock++ - } - }) - } - - // Scan the table. - for iter.Next() { - key := iter.Key() - _, seq, _, kerr := parseInternalKey(key) - if kerr != nil { - tcorruptedKey++ - continue - } - tgoodKey++ - if seq > tSeq { - tSeq = seq - } - if imin == nil { - imin = append([]byte{}, key...) - } - imax = append(imax[:0], key...) - } - if err := iter.Error(); err != nil { - iter.Release() - return err - } - iter.Release() - - goodKey += tgoodKey - corruptedKey += tcorruptedKey - corruptedBlock += tcorruptedBlock - - if strict && (tcorruptedKey > 0 || tcorruptedBlock > 0) { - droppedTable++ - s.logf("table@recovery dropped @%d Gk·%d Ck·%d Cb·%d S·%d Q·%d", fd.Num, tgoodKey, tcorruptedKey, tcorruptedBlock, size, tSeq) - return nil - } - - if tgoodKey > 0 { - if tcorruptedKey > 0 || tcorruptedBlock > 0 { - // Rebuild the table. - s.logf("table@recovery rebuilding @%d", fd.Num) - iter := tr.NewIterator(nil, nil) - tmpFd, newSize, err := buildTable(iter) - iter.Release() - if err != nil { - return err - } - closed = true - reader.Close() - if err := s.stor.Rename(tmpFd, fd); err != nil { - return err - } - size = newSize - } - if tSeq > maxSeq { - maxSeq = tSeq - } - recoveredKey += tgoodKey - // Add table to level 0. - rec.addTable(0, fd.Num, size, imin, imax) - s.logf("table@recovery recovered @%d Gk·%d Ck·%d Cb·%d S·%d Q·%d", fd.Num, tgoodKey, tcorruptedKey, tcorruptedBlock, size, tSeq) - } else { - droppedTable++ - s.logf("table@recovery unrecoverable @%d Ck·%d Cb·%d S·%d", fd.Num, tcorruptedKey, tcorruptedBlock, size) - } - - return nil - } - - // Recover all tables. - if len(fds) > 0 { - s.logf("table@recovery F·%d", len(fds)) - - // Mark file number as used. - s.markFileNum(fds[len(fds)-1].Num) - - for _, fd := range fds { - if err := recoverTable(fd); err != nil { - return err - } - } - - s.logf("table@recovery recovered F·%d N·%d Gk·%d Ck·%d Q·%d", len(fds), recoveredKey, goodKey, corruptedKey, maxSeq) - } - - // Set sequence number. - rec.setSeqNum(maxSeq) - - // Create new manifest. - if err := s.create(); err != nil { - return err - } - - // Commit. - return s.commit(rec) -} - -func (db *DB) recoverJournal() error { - // Get all journals and sort it by file number. - rawFds, err := db.s.stor.List(storage.TypeJournal) - if err != nil { - return err - } - sortFds(rawFds) - - // Journals that will be recovered. - var fds []storage.FileDesc - for _, fd := range rawFds { - if fd.Num >= db.s.stJournalNum || fd.Num == db.s.stPrevJournalNum { - fds = append(fds, fd) - } - } - - var ( - ofd storage.FileDesc // Obsolete file. - rec = &sessionRecord{} - ) - - // Recover journals. - if len(fds) > 0 { - db.logf("journal@recovery F·%d", len(fds)) - - // Mark file number as used. - db.s.markFileNum(fds[len(fds)-1].Num) - - var ( - // Options. - strict = db.s.o.GetStrict(opt.StrictJournal) - checksum = db.s.o.GetStrict(opt.StrictJournalChecksum) - writeBuffer = db.s.o.GetWriteBuffer() - - jr *journal.Reader - mdb = memdb.New(db.s.icmp, writeBuffer) - buf = &util.Buffer{} - batchSeq uint64 - batchLen int - ) - - for _, fd := range fds { - db.logf("journal@recovery recovering @%d", fd.Num) - - fr, err := db.s.stor.Open(fd) - if err != nil { - return err - } - - // Create or reset journal reader instance. - if jr == nil { - jr = journal.NewReader(fr, dropper{db.s, fd}, strict, checksum) - } else { - jr.Reset(fr, dropper{db.s, fd}, strict, checksum) - } - - // Flush memdb and remove obsolete journal file. - if !ofd.Zero() { - if mdb.Len() > 0 { - if _, err := db.s.flushMemdb(rec, mdb, 0); err != nil { - fr.Close() - return err - } - } - - rec.setJournalNum(fd.Num) - rec.setSeqNum(db.seq) - if err := db.s.commit(rec); err != nil { - fr.Close() - return err - } - rec.resetAddedTables() - - db.s.stor.Remove(ofd) - ofd = storage.FileDesc{} - } - - // Replay journal to memdb. - mdb.Reset() - for { - r, err := jr.Next() - if err != nil { - if err == io.EOF { - break - } - - fr.Close() - return errors.SetFd(err, fd) - } - - buf.Reset() - if _, err := buf.ReadFrom(r); err != nil { - if err == io.ErrUnexpectedEOF { - // This is error returned due to corruption, with strict == false. - continue - } - - fr.Close() - return errors.SetFd(err, fd) - } - batchSeq, batchLen, err = decodeBatchToMem(buf.Bytes(), db.seq, mdb) - if err != nil { - if !strict && errors.IsCorrupted(err) { - db.s.logf("journal error: %v (skipped)", err) - // We won't apply sequence number as it might be corrupted. - continue - } - - fr.Close() - return errors.SetFd(err, fd) - } - - // Save sequence number. - db.seq = batchSeq + uint64(batchLen) - - // Flush it if large enough. - if mdb.Size() >= writeBuffer { - if _, err := db.s.flushMemdb(rec, mdb, 0); err != nil { - fr.Close() - return err - } - - mdb.Reset() - } - } - - fr.Close() - ofd = fd - } - - // Flush the last memdb. - if mdb.Len() > 0 { - if _, err := db.s.flushMemdb(rec, mdb, 0); err != nil { - return err - } - } - } - - // Create a new journal. - if _, err := db.newMem(0); err != nil { - return err - } - - // Commit. - rec.setJournalNum(db.journalFd.Num) - rec.setSeqNum(db.seq) - if err := db.s.commit(rec); err != nil { - // Close journal on error. - if db.journal != nil { - db.journal.Close() - db.journalWriter.Close() - } - return err - } - - // Remove the last obsolete journal file. - if !ofd.Zero() { - db.s.stor.Remove(ofd) - } - - return nil -} - -func (db *DB) recoverJournalRO() error { - // Get all journals and sort it by file number. - rawFds, err := db.s.stor.List(storage.TypeJournal) - if err != nil { - return err - } - sortFds(rawFds) - - // Journals that will be recovered. - var fds []storage.FileDesc - for _, fd := range rawFds { - if fd.Num >= db.s.stJournalNum || fd.Num == db.s.stPrevJournalNum { - fds = append(fds, fd) - } - } - - var ( - // Options. - strict = db.s.o.GetStrict(opt.StrictJournal) - checksum = db.s.o.GetStrict(opt.StrictJournalChecksum) - writeBuffer = db.s.o.GetWriteBuffer() - - mdb = memdb.New(db.s.icmp, writeBuffer) - ) - - // Recover journals. - if len(fds) > 0 { - db.logf("journal@recovery RO·Mode F·%d", len(fds)) - - var ( - jr *journal.Reader - buf = &util.Buffer{} - batchSeq uint64 - batchLen int - ) - - for _, fd := range fds { - db.logf("journal@recovery recovering @%d", fd.Num) - - fr, err := db.s.stor.Open(fd) - if err != nil { - return err - } - - // Create or reset journal reader instance. - if jr == nil { - jr = journal.NewReader(fr, dropper{db.s, fd}, strict, checksum) - } else { - jr.Reset(fr, dropper{db.s, fd}, strict, checksum) - } - - // Replay journal to memdb. - for { - r, err := jr.Next() - if err != nil { - if err == io.EOF { - break - } - - fr.Close() - return errors.SetFd(err, fd) - } - - buf.Reset() - if _, err := buf.ReadFrom(r); err != nil { - if err == io.ErrUnexpectedEOF { - // This is error returned due to corruption, with strict == false. - continue - } - - fr.Close() - return errors.SetFd(err, fd) - } - batchSeq, batchLen, err = decodeBatchToMem(buf.Bytes(), db.seq, mdb) - if err != nil { - if !strict && errors.IsCorrupted(err) { - db.s.logf("journal error: %v (skipped)", err) - // We won't apply sequence number as it might be corrupted. - continue - } - - fr.Close() - return errors.SetFd(err, fd) - } - - // Save sequence number. - db.seq = batchSeq + uint64(batchLen) - } - - fr.Close() - } - } - - // Set memDB. - db.mem = &memDB{db: db, DB: mdb, ref: 1} - - return nil -} - -func memGet(mdb *memdb.DB, ikey internalKey, icmp *iComparer) (ok bool, mv []byte, err error) { - mk, mv, err := mdb.Find(ikey) - if err == nil { - ukey, _, kt, kerr := parseInternalKey(mk) - if kerr != nil { - // Shouldn't have had happen. - panic(kerr) - } - if icmp.uCompare(ukey, ikey.ukey()) == 0 { - if kt == keyTypeDel { - return true, nil, ErrNotFound - } - return true, mv, nil - - } - } else if err != ErrNotFound { - return true, nil, err - } - return -} - -func (db *DB) get(auxm *memdb.DB, auxt tFiles, key []byte, seq uint64, ro *opt.ReadOptions) (value []byte, err error) { - ikey := makeInternalKey(nil, key, seq, keyTypeSeek) - - if auxm != nil { - if ok, mv, me := memGet(auxm, ikey, db.s.icmp); ok { - return append([]byte{}, mv...), me - } - } - - em, fm := db.getMems() - for _, m := range [...]*memDB{em, fm} { - if m == nil { - continue - } - defer m.decref() - - if ok, mv, me := memGet(m.DB, ikey, db.s.icmp); ok { - return append([]byte{}, mv...), me - } - } - - v := db.s.version() - value, cSched, err := v.get(auxt, ikey, ro, false) - v.release() - if cSched { - // Trigger table compaction. - db.compTrigger(db.tcompCmdC) - } - return -} - -func nilIfNotFound(err error) error { - if err == ErrNotFound { - return nil - } - return err -} - -func (db *DB) has(auxm *memdb.DB, auxt tFiles, key []byte, seq uint64, ro *opt.ReadOptions) (ret bool, err error) { - ikey := makeInternalKey(nil, key, seq, keyTypeSeek) - - if auxm != nil { - if ok, _, me := memGet(auxm, ikey, db.s.icmp); ok { - return me == nil, nilIfNotFound(me) - } - } - - em, fm := db.getMems() - for _, m := range [...]*memDB{em, fm} { - if m == nil { - continue - } - defer m.decref() - - if ok, _, me := memGet(m.DB, ikey, db.s.icmp); ok { - return me == nil, nilIfNotFound(me) - } - } - - v := db.s.version() - _, cSched, err := v.get(auxt, ikey, ro, true) - v.release() - if cSched { - // Trigger table compaction. - db.compTrigger(db.tcompCmdC) - } - if err == nil { - ret = true - } else if err == ErrNotFound { - err = nil - } - return -} - -// Get gets the value for the given key. It returns ErrNotFound if the -// DB does not contains the key. -// -// The returned slice is its own copy, it is safe to modify the contents -// of the returned slice. -// It is safe to modify the contents of the argument after Get returns. -func (db *DB) Get(key []byte, ro *opt.ReadOptions) (value []byte, err error) { - err = db.ok() - if err != nil { - return - } - - se := db.acquireSnapshot() - defer db.releaseSnapshot(se) - return db.get(nil, nil, key, se.seq, ro) -} - -// Has returns true if the DB does contains the given key. -// -// It is safe to modify the contents of the argument after Get returns. -func (db *DB) Has(key []byte, ro *opt.ReadOptions) (ret bool, err error) { - err = db.ok() - if err != nil { - return - } - - se := db.acquireSnapshot() - defer db.releaseSnapshot(se) - return db.has(nil, nil, key, se.seq, ro) -} - -// NewIterator returns an iterator for the latest snapshot of the -// underlying DB. -// The returned iterator is not safe for concurrent use, but it is safe to use -// multiple iterators concurrently, with each in a dedicated goroutine. -// It is also safe to use an iterator concurrently with modifying its -// underlying DB. The resultant key/value pairs are guaranteed to be -// consistent. -// -// Slice allows slicing the iterator to only contains keys in the given -// range. A nil Range.Start is treated as a key before all keys in the -// DB. And a nil Range.Limit is treated as a key after all keys in -// the DB. -// -// The iterator must be released after use, by calling Release method. -// -// Also read Iterator documentation of the leveldb/iterator package. -func (db *DB) NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator { - if err := db.ok(); err != nil { - return iterator.NewEmptyIterator(err) - } - - se := db.acquireSnapshot() - defer db.releaseSnapshot(se) - // Iterator holds 'version' lock, 'version' is immutable so snapshot - // can be released after iterator created. - return db.newIterator(nil, nil, se.seq, slice, ro) -} - -// GetSnapshot returns a latest snapshot of the underlying DB. A snapshot -// is a frozen snapshot of a DB state at a particular point in time. The -// content of snapshot are guaranteed to be consistent. -// -// The snapshot must be released after use, by calling Release method. -func (db *DB) GetSnapshot() (*Snapshot, error) { - if err := db.ok(); err != nil { - return nil, err - } - - return db.newSnapshot(), nil -} - -// GetProperty returns value of the given property name. -// -// Property names: -// leveldb.num-files-at-level{n} -// Returns the number of files at level 'n'. -// leveldb.stats -// Returns statistics of the underlying DB. -// leveldb.sstables -// Returns sstables list for each level. -// leveldb.blockpool -// Returns block pool stats. -// leveldb.cachedblock -// Returns size of cached block. -// leveldb.openedtables -// Returns number of opened tables. -// leveldb.alivesnaps -// Returns number of alive snapshots. -// leveldb.aliveiters -// Returns number of alive iterators. -func (db *DB) GetProperty(name string) (value string, err error) { - err = db.ok() - if err != nil { - return - } - - const prefix = "leveldb." - if !strings.HasPrefix(name, prefix) { - return "", ErrNotFound - } - p := name[len(prefix):] - - v := db.s.version() - defer v.release() - - numFilesPrefix := "num-files-at-level" - switch { - case strings.HasPrefix(p, numFilesPrefix): - var level uint - var rest string - n, _ := fmt.Sscanf(p[len(numFilesPrefix):], "%d%s", &level, &rest) - if n != 1 { - err = ErrNotFound - } else { - value = fmt.Sprint(v.tLen(int(level))) - } - case p == "stats": - value = "Compactions\n" + - " Level | Tables | Size(MB) | Time(sec) | Read(MB) | Write(MB)\n" + - "-------+------------+---------------+---------------+---------------+---------------\n" - for level, tables := range v.levels { - duration, read, write := db.compStats.getStat(level) - if len(tables) == 0 && duration == 0 { - continue - } - value += fmt.Sprintf(" %3d | %10d | %13.5f | %13.5f | %13.5f | %13.5f\n", - level, len(tables), float64(tables.size())/1048576.0, duration.Seconds(), - float64(read)/1048576.0, float64(write)/1048576.0) - } - case p == "sstables": - for level, tables := range v.levels { - value += fmt.Sprintf("--- level %d ---\n", level) - for _, t := range tables { - value += fmt.Sprintf("%d:%d[%q .. %q]\n", t.fd.Num, t.size, t.imin, t.imax) - } - } - case p == "blockpool": - value = fmt.Sprintf("%v", db.s.tops.bpool) - case p == "cachedblock": - if db.s.tops.bcache != nil { - value = fmt.Sprintf("%d", db.s.tops.bcache.Size()) - } else { - value = "" - } - case p == "openedtables": - value = fmt.Sprintf("%d", db.s.tops.cache.Size()) - case p == "alivesnaps": - value = fmt.Sprintf("%d", atomic.LoadInt32(&db.aliveSnaps)) - case p == "aliveiters": - value = fmt.Sprintf("%d", atomic.LoadInt32(&db.aliveIters)) - default: - err = ErrNotFound - } - - return -} - -// SizeOf calculates approximate sizes of the given key ranges. -// The length of the returned sizes are equal with the length of the given -// ranges. The returned sizes measure storage space usage, so if the user -// data compresses by a factor of ten, the returned sizes will be one-tenth -// the size of the corresponding user data size. -// The results may not include the sizes of recently written data. -func (db *DB) SizeOf(ranges []util.Range) (Sizes, error) { - if err := db.ok(); err != nil { - return nil, err - } - - v := db.s.version() - defer v.release() - - sizes := make(Sizes, 0, len(ranges)) - for _, r := range ranges { - imin := makeInternalKey(nil, r.Start, keyMaxSeq, keyTypeSeek) - imax := makeInternalKey(nil, r.Limit, keyMaxSeq, keyTypeSeek) - start, err := v.offsetOf(imin) - if err != nil { - return nil, err - } - limit, err := v.offsetOf(imax) - if err != nil { - return nil, err - } - var size int64 - if limit >= start { - size = limit - start - } - sizes = append(sizes, size) - } - - return sizes, nil -} - -// Close closes the DB. This will also releases any outstanding snapshot, -// abort any in-flight compaction and discard open transaction. -// -// It is not safe to close a DB until all outstanding iterators are released. -// It is valid to call Close multiple times. Other methods should not be -// called after the DB has been closed. -func (db *DB) Close() error { - if !db.setClosed() { - return ErrClosed - } - - start := time.Now() - db.log("db@close closing") - - // Clear the finalizer. - runtime.SetFinalizer(db, nil) - - // Get compaction error. - var err error - select { - case err = <-db.compErrC: - if err == ErrReadOnly { - err = nil - } - default: - } - - // Signal all goroutines. - close(db.closeC) - - // Discard open transaction. - if db.tr != nil { - db.tr.Discard() - } - - // Acquire writer lock. - db.writeLockC <- struct{}{} - - // Wait for all gorotines to exit. - db.closeW.Wait() - - // Closes journal. - if db.journal != nil { - db.journal.Close() - db.journalWriter.Close() - db.journal = nil - db.journalWriter = nil - } - - if db.writeDelayN > 0 { - db.logf("db@write was delayed N·%d T·%v", db.writeDelayN, db.writeDelay) - } - - // Close session. - db.s.close() - db.logf("db@close done T·%v", time.Since(start)) - db.s.release() - - if db.closer != nil { - if err1 := db.closer.Close(); err == nil { - err = err1 - } - db.closer = nil - } - - // Clear memdbs. - db.clearMems() - - return err -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go b/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go deleted file mode 100644 index b6563e8..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go +++ /dev/null @@ -1,826 +0,0 @@ -// Copyright (c) 2012, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package leveldb - -import ( - "sync" - "time" - - "github.com/syndtr/goleveldb/leveldb/errors" - "github.com/syndtr/goleveldb/leveldb/opt" - "github.com/syndtr/goleveldb/leveldb/storage" -) - -var ( - errCompactionTransactExiting = errors.New("leveldb: compaction transact exiting") -) - -type cStat struct { - duration time.Duration - read int64 - write int64 -} - -func (p *cStat) add(n *cStatStaging) { - p.duration += n.duration - p.read += n.read - p.write += n.write -} - -func (p *cStat) get() (duration time.Duration, read, write int64) { - return p.duration, p.read, p.write -} - -type cStatStaging struct { - start time.Time - duration time.Duration - on bool - read int64 - write int64 -} - -func (p *cStatStaging) startTimer() { - if !p.on { - p.start = time.Now() - p.on = true - } -} - -func (p *cStatStaging) stopTimer() { - if p.on { - p.duration += time.Since(p.start) - p.on = false - } -} - -type cStats struct { - lk sync.Mutex - stats []cStat -} - -func (p *cStats) addStat(level int, n *cStatStaging) { - p.lk.Lock() - if level >= len(p.stats) { - newStats := make([]cStat, level+1) - copy(newStats, p.stats) - p.stats = newStats - } - p.stats[level].add(n) - p.lk.Unlock() -} - -func (p *cStats) getStat(level int) (duration time.Duration, read, write int64) { - p.lk.Lock() - defer p.lk.Unlock() - if level < len(p.stats) { - return p.stats[level].get() - } - return -} - -func (db *DB) compactionError() { - var err error -noerr: - // No error. - for { - select { - case err = <-db.compErrSetC: - switch { - case err == nil: - case err == ErrReadOnly, errors.IsCorrupted(err): - goto hasperr - default: - goto haserr - } - case <-db.closeC: - return - } - } -haserr: - // Transient error. - for { - select { - case db.compErrC <- err: - case err = <-db.compErrSetC: - switch { - case err == nil: - goto noerr - case err == ErrReadOnly, errors.IsCorrupted(err): - goto hasperr - default: - } - case <-db.closeC: - return - } - } -hasperr: - // Persistent error. - for { - select { - case db.compErrC <- err: - case db.compPerErrC <- err: - case db.writeLockC <- struct{}{}: - // Hold write lock, so that write won't pass-through. - db.compWriteLocking = true - case <-db.closeC: - if db.compWriteLocking { - // We should release the lock or Close will hang. - <-db.writeLockC - } - return - } - } -} - -type compactionTransactCounter int - -func (cnt *compactionTransactCounter) incr() { - *cnt++ -} - -type compactionTransactInterface interface { - run(cnt *compactionTransactCounter) error - revert() error -} - -func (db *DB) compactionTransact(name string, t compactionTransactInterface) { - defer func() { - if x := recover(); x != nil { - if x == errCompactionTransactExiting { - if err := t.revert(); err != nil { - db.logf("%s revert error %q", name, err) - } - } - panic(x) - } - }() - - const ( - backoffMin = 1 * time.Second - backoffMax = 8 * time.Second - backoffMul = 2 * time.Second - ) - var ( - backoff = backoffMin - backoffT = time.NewTimer(backoff) - lastCnt = compactionTransactCounter(0) - - disableBackoff = db.s.o.GetDisableCompactionBackoff() - ) - for n := 0; ; n++ { - // Check whether the DB is closed. - if db.isClosed() { - db.logf("%s exiting", name) - db.compactionExitTransact() - } else if n > 0 { - db.logf("%s retrying N·%d", name, n) - } - - // Execute. - cnt := compactionTransactCounter(0) - err := t.run(&cnt) - if err != nil { - db.logf("%s error I·%d %q", name, cnt, err) - } - - // Set compaction error status. - select { - case db.compErrSetC <- err: - case perr := <-db.compPerErrC: - if err != nil { - db.logf("%s exiting (persistent error %q)", name, perr) - db.compactionExitTransact() - } - case <-db.closeC: - db.logf("%s exiting", name) - db.compactionExitTransact() - } - if err == nil { - return - } - if errors.IsCorrupted(err) { - db.logf("%s exiting (corruption detected)", name) - db.compactionExitTransact() - } - - if !disableBackoff { - // Reset backoff duration if counter is advancing. - if cnt > lastCnt { - backoff = backoffMin - lastCnt = cnt - } - - // Backoff. - backoffT.Reset(backoff) - if backoff < backoffMax { - backoff *= backoffMul - if backoff > backoffMax { - backoff = backoffMax - } - } - select { - case <-backoffT.C: - case <-db.closeC: - db.logf("%s exiting", name) - db.compactionExitTransact() - } - } - } -} - -type compactionTransactFunc struct { - runFunc func(cnt *compactionTransactCounter) error - revertFunc func() error -} - -func (t *compactionTransactFunc) run(cnt *compactionTransactCounter) error { - return t.runFunc(cnt) -} - -func (t *compactionTransactFunc) revert() error { - if t.revertFunc != nil { - return t.revertFunc() - } - return nil -} - -func (db *DB) compactionTransactFunc(name string, run func(cnt *compactionTransactCounter) error, revert func() error) { - db.compactionTransact(name, &compactionTransactFunc{run, revert}) -} - -func (db *DB) compactionExitTransact() { - panic(errCompactionTransactExiting) -} - -func (db *DB) compactionCommit(name string, rec *sessionRecord) { - db.compCommitLk.Lock() - defer db.compCommitLk.Unlock() // Defer is necessary. - db.compactionTransactFunc(name+"@commit", func(cnt *compactionTransactCounter) error { - return db.s.commit(rec) - }, nil) -} - -func (db *DB) memCompaction() { - mdb := db.getFrozenMem() - if mdb == nil { - return - } - defer mdb.decref() - - db.logf("memdb@flush N·%d S·%s", mdb.Len(), shortenb(mdb.Size())) - - // Don't compact empty memdb. - if mdb.Len() == 0 { - db.logf("memdb@flush skipping") - // drop frozen memdb - db.dropFrozenMem() - return - } - - // Pause table compaction. - resumeC := make(chan struct{}) - select { - case db.tcompPauseC <- (chan<- struct{})(resumeC): - case <-db.compPerErrC: - close(resumeC) - resumeC = nil - case <-db.closeC: - db.compactionExitTransact() - } - - var ( - rec = &sessionRecord{} - stats = &cStatStaging{} - flushLevel int - ) - - // Generate tables. - db.compactionTransactFunc("memdb@flush", func(cnt *compactionTransactCounter) (err error) { - stats.startTimer() - flushLevel, err = db.s.flushMemdb(rec, mdb.DB, db.memdbMaxLevel) - stats.stopTimer() - return - }, func() error { - for _, r := range rec.addedTables { - db.logf("memdb@flush revert @%d", r.num) - if err := db.s.stor.Remove(storage.FileDesc{Type: storage.TypeTable, Num: r.num}); err != nil { - return err - } - } - return nil - }) - - rec.setJournalNum(db.journalFd.Num) - rec.setSeqNum(db.frozenSeq) - - // Commit. - stats.startTimer() - db.compactionCommit("memdb", rec) - stats.stopTimer() - - db.logf("memdb@flush committed F·%d T·%v", len(rec.addedTables), stats.duration) - - for _, r := range rec.addedTables { - stats.write += r.size - } - db.compStats.addStat(flushLevel, stats) - - // Drop frozen memdb. - db.dropFrozenMem() - - // Resume table compaction. - if resumeC != nil { - select { - case <-resumeC: - close(resumeC) - case <-db.closeC: - db.compactionExitTransact() - } - } - - // Trigger table compaction. - db.compTrigger(db.tcompCmdC) -} - -type tableCompactionBuilder struct { - db *DB - s *session - c *compaction - rec *sessionRecord - stat0, stat1 *cStatStaging - - snapHasLastUkey bool - snapLastUkey []byte - snapLastSeq uint64 - snapIter int - snapKerrCnt int - snapDropCnt int - - kerrCnt int - dropCnt int - - minSeq uint64 - strict bool - tableSize int - - tw *tWriter -} - -func (b *tableCompactionBuilder) appendKV(key, value []byte) error { - // Create new table if not already. - if b.tw == nil { - // Check for pause event. - if b.db != nil { - select { - case ch := <-b.db.tcompPauseC: - b.db.pauseCompaction(ch) - case <-b.db.closeC: - b.db.compactionExitTransact() - default: - } - } - - // Create new table. - var err error - b.tw, err = b.s.tops.create() - if err != nil { - return err - } - } - - // Write key/value into table. - return b.tw.append(key, value) -} - -func (b *tableCompactionBuilder) needFlush() bool { - return b.tw.tw.BytesLen() >= b.tableSize -} - -func (b *tableCompactionBuilder) flush() error { - t, err := b.tw.finish() - if err != nil { - return err - } - b.rec.addTableFile(b.c.sourceLevel+1, t) - b.stat1.write += t.size - b.s.logf("table@build created L%d@%d N·%d S·%s %q:%q", b.c.sourceLevel+1, t.fd.Num, b.tw.tw.EntriesLen(), shortenb(int(t.size)), t.imin, t.imax) - b.tw = nil - return nil -} - -func (b *tableCompactionBuilder) cleanup() { - if b.tw != nil { - b.tw.drop() - b.tw = nil - } -} - -func (b *tableCompactionBuilder) run(cnt *compactionTransactCounter) error { - snapResumed := b.snapIter > 0 - hasLastUkey := b.snapHasLastUkey // The key might has zero length, so this is necessary. - lastUkey := append([]byte{}, b.snapLastUkey...) - lastSeq := b.snapLastSeq - b.kerrCnt = b.snapKerrCnt - b.dropCnt = b.snapDropCnt - // Restore compaction state. - b.c.restore() - - defer b.cleanup() - - b.stat1.startTimer() - defer b.stat1.stopTimer() - - iter := b.c.newIterator() - defer iter.Release() - for i := 0; iter.Next(); i++ { - // Incr transact counter. - cnt.incr() - - // Skip until last state. - if i < b.snapIter { - continue - } - - resumed := false - if snapResumed { - resumed = true - snapResumed = false - } - - ikey := iter.Key() - ukey, seq, kt, kerr := parseInternalKey(ikey) - - if kerr == nil { - shouldStop := !resumed && b.c.shouldStopBefore(ikey) - - if !hasLastUkey || b.s.icmp.uCompare(lastUkey, ukey) != 0 { - // First occurrence of this user key. - - // Only rotate tables if ukey doesn't hop across. - if b.tw != nil && (shouldStop || b.needFlush()) { - if err := b.flush(); err != nil { - return err - } - - // Creates snapshot of the state. - b.c.save() - b.snapHasLastUkey = hasLastUkey - b.snapLastUkey = append(b.snapLastUkey[:0], lastUkey...) - b.snapLastSeq = lastSeq - b.snapIter = i - b.snapKerrCnt = b.kerrCnt - b.snapDropCnt = b.dropCnt - } - - hasLastUkey = true - lastUkey = append(lastUkey[:0], ukey...) - lastSeq = keyMaxSeq - } - - switch { - case lastSeq <= b.minSeq: - // Dropped because newer entry for same user key exist - fallthrough // (A) - case kt == keyTypeDel && seq <= b.minSeq && b.c.baseLevelForKey(lastUkey): - // For this user key: - // (1) there is no data in higher levels - // (2) data in lower levels will have larger seq numbers - // (3) data in layers that are being compacted here and have - // smaller seq numbers will be dropped in the next - // few iterations of this loop (by rule (A) above). - // Therefore this deletion marker is obsolete and can be dropped. - lastSeq = seq - b.dropCnt++ - continue - default: - lastSeq = seq - } - } else { - if b.strict { - return kerr - } - - // Don't drop corrupted keys. - hasLastUkey = false - lastUkey = lastUkey[:0] - lastSeq = keyMaxSeq - b.kerrCnt++ - } - - if err := b.appendKV(ikey, iter.Value()); err != nil { - return err - } - } - - if err := iter.Error(); err != nil { - return err - } - - // Finish last table. - if b.tw != nil && !b.tw.empty() { - return b.flush() - } - return nil -} - -func (b *tableCompactionBuilder) revert() error { - for _, at := range b.rec.addedTables { - b.s.logf("table@build revert @%d", at.num) - if err := b.s.stor.Remove(storage.FileDesc{Type: storage.TypeTable, Num: at.num}); err != nil { - return err - } - } - return nil -} - -func (db *DB) tableCompaction(c *compaction, noTrivial bool) { - defer c.release() - - rec := &sessionRecord{} - rec.addCompPtr(c.sourceLevel, c.imax) - - if !noTrivial && c.trivial() { - t := c.levels[0][0] - db.logf("table@move L%d@%d -> L%d", c.sourceLevel, t.fd.Num, c.sourceLevel+1) - rec.delTable(c.sourceLevel, t.fd.Num) - rec.addTableFile(c.sourceLevel+1, t) - db.compactionCommit("table-move", rec) - return - } - - var stats [2]cStatStaging - for i, tables := range c.levels { - for _, t := range tables { - stats[i].read += t.size - // Insert deleted tables into record - rec.delTable(c.sourceLevel+i, t.fd.Num) - } - } - sourceSize := int(stats[0].read + stats[1].read) - minSeq := db.minSeq() - db.logf("table@compaction L%d·%d -> L%d·%d S·%s Q·%d", c.sourceLevel, len(c.levels[0]), c.sourceLevel+1, len(c.levels[1]), shortenb(sourceSize), minSeq) - - b := &tableCompactionBuilder{ - db: db, - s: db.s, - c: c, - rec: rec, - stat1: &stats[1], - minSeq: minSeq, - strict: db.s.o.GetStrict(opt.StrictCompaction), - tableSize: db.s.o.GetCompactionTableSize(c.sourceLevel + 1), - } - db.compactionTransact("table@build", b) - - // Commit. - stats[1].startTimer() - db.compactionCommit("table", rec) - stats[1].stopTimer() - - resultSize := int(stats[1].write) - db.logf("table@compaction committed F%s S%s Ke·%d D·%d T·%v", sint(len(rec.addedTables)-len(rec.deletedTables)), sshortenb(resultSize-sourceSize), b.kerrCnt, b.dropCnt, stats[1].duration) - - // Save compaction stats - for i := range stats { - db.compStats.addStat(c.sourceLevel+1, &stats[i]) - } -} - -func (db *DB) tableRangeCompaction(level int, umin, umax []byte) error { - db.logf("table@compaction range L%d %q:%q", level, umin, umax) - if level >= 0 { - if c := db.s.getCompactionRange(level, umin, umax, true); c != nil { - db.tableCompaction(c, true) - } - } else { - // Retry until nothing to compact. - for { - compacted := false - - // Scan for maximum level with overlapped tables. - v := db.s.version() - m := 1 - for i := m; i < len(v.levels); i++ { - tables := v.levels[i] - if tables.overlaps(db.s.icmp, umin, umax, false) { - m = i - } - } - v.release() - - for level := 0; level < m; level++ { - if c := db.s.getCompactionRange(level, umin, umax, false); c != nil { - db.tableCompaction(c, true) - compacted = true - } - } - - if !compacted { - break - } - } - } - - return nil -} - -func (db *DB) tableAutoCompaction() { - if c := db.s.pickCompaction(); c != nil { - db.tableCompaction(c, false) - } -} - -func (db *DB) tableNeedCompaction() bool { - v := db.s.version() - defer v.release() - return v.needCompaction() -} - -func (db *DB) pauseCompaction(ch chan<- struct{}) { - select { - case ch <- struct{}{}: - case <-db.closeC: - db.compactionExitTransact() - } -} - -type cCmd interface { - ack(err error) -} - -type cAuto struct { - ackC chan<- error -} - -func (r cAuto) ack(err error) { - if r.ackC != nil { - defer func() { - recover() - }() - r.ackC <- err - } -} - -type cRange struct { - level int - min, max []byte - ackC chan<- error -} - -func (r cRange) ack(err error) { - if r.ackC != nil { - defer func() { - recover() - }() - r.ackC <- err - } -} - -// This will trigger auto compaction but will not wait for it. -func (db *DB) compTrigger(compC chan<- cCmd) { - select { - case compC <- cAuto{}: - default: - } -} - -// This will trigger auto compaction and/or wait for all compaction to be done. -func (db *DB) compTriggerWait(compC chan<- cCmd) (err error) { - ch := make(chan error) - defer close(ch) - // Send cmd. - select { - case compC <- cAuto{ch}: - case err = <-db.compErrC: - return - case <-db.closeC: - return ErrClosed - } - // Wait cmd. - select { - case err = <-ch: - case err = <-db.compErrC: - case <-db.closeC: - return ErrClosed - } - return err -} - -// Send range compaction request. -func (db *DB) compTriggerRange(compC chan<- cCmd, level int, min, max []byte) (err error) { - ch := make(chan error) - defer close(ch) - // Send cmd. - select { - case compC <- cRange{level, min, max, ch}: - case err := <-db.compErrC: - return err - case <-db.closeC: - return ErrClosed - } - // Wait cmd. - select { - case err = <-ch: - case err = <-db.compErrC: - case <-db.closeC: - return ErrClosed - } - return err -} - -func (db *DB) mCompaction() { - var x cCmd - - defer func() { - if x := recover(); x != nil { - if x != errCompactionTransactExiting { - panic(x) - } - } - if x != nil { - x.ack(ErrClosed) - } - db.closeW.Done() - }() - - for { - select { - case x = <-db.mcompCmdC: - switch x.(type) { - case cAuto: - db.memCompaction() - x.ack(nil) - x = nil - default: - panic("leveldb: unknown command") - } - case <-db.closeC: - return - } - } -} - -func (db *DB) tCompaction() { - var x cCmd - var ackQ []cCmd - - defer func() { - if x := recover(); x != nil { - if x != errCompactionTransactExiting { - panic(x) - } - } - for i := range ackQ { - ackQ[i].ack(ErrClosed) - ackQ[i] = nil - } - if x != nil { - x.ack(ErrClosed) - } - db.closeW.Done() - }() - - for { - if db.tableNeedCompaction() { - select { - case x = <-db.tcompCmdC: - case ch := <-db.tcompPauseC: - db.pauseCompaction(ch) - continue - case <-db.closeC: - return - default: - } - } else { - for i := range ackQ { - ackQ[i].ack(nil) - ackQ[i] = nil - } - ackQ = ackQ[:0] - select { - case x = <-db.tcompCmdC: - case ch := <-db.tcompPauseC: - db.pauseCompaction(ch) - continue - case <-db.closeC: - return - } - } - if x != nil { - switch cmd := x.(type) { - case cAuto: - ackQ = append(ackQ, x) - case cRange: - x.ack(db.tableRangeCompaction(cmd.level, cmd.min, cmd.max)) - default: - panic("leveldb: unknown command") - } - x = nil - } - db.tableAutoCompaction() - } -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/db_iter.go b/vendor/github.com/syndtr/goleveldb/leveldb/db_iter.go deleted file mode 100644 index 03c24cd..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/db_iter.go +++ /dev/null @@ -1,360 +0,0 @@ -// Copyright (c) 2012, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package leveldb - -import ( - "errors" - "math/rand" - "runtime" - "sync" - "sync/atomic" - - "github.com/syndtr/goleveldb/leveldb/iterator" - "github.com/syndtr/goleveldb/leveldb/opt" - "github.com/syndtr/goleveldb/leveldb/util" -) - -var ( - errInvalidInternalKey = errors.New("leveldb: Iterator: invalid internal key") -) - -type memdbReleaser struct { - once sync.Once - m *memDB -} - -func (mr *memdbReleaser) Release() { - mr.once.Do(func() { - mr.m.decref() - }) -} - -func (db *DB) newRawIterator(auxm *memDB, auxt tFiles, slice *util.Range, ro *opt.ReadOptions) iterator.Iterator { - strict := opt.GetStrict(db.s.o.Options, ro, opt.StrictReader) - em, fm := db.getMems() - v := db.s.version() - - tableIts := v.getIterators(slice, ro) - n := len(tableIts) + len(auxt) + 3 - its := make([]iterator.Iterator, 0, n) - - if auxm != nil { - ami := auxm.NewIterator(slice) - ami.SetReleaser(&memdbReleaser{m: auxm}) - its = append(its, ami) - } - for _, t := range auxt { - its = append(its, v.s.tops.newIterator(t, slice, ro)) - } - - emi := em.NewIterator(slice) - emi.SetReleaser(&memdbReleaser{m: em}) - its = append(its, emi) - if fm != nil { - fmi := fm.NewIterator(slice) - fmi.SetReleaser(&memdbReleaser{m: fm}) - its = append(its, fmi) - } - its = append(its, tableIts...) - mi := iterator.NewMergedIterator(its, db.s.icmp, strict) - mi.SetReleaser(&versionReleaser{v: v}) - return mi -} - -func (db *DB) newIterator(auxm *memDB, auxt tFiles, seq uint64, slice *util.Range, ro *opt.ReadOptions) *dbIter { - var islice *util.Range - if slice != nil { - islice = &util.Range{} - if slice.Start != nil { - islice.Start = makeInternalKey(nil, slice.Start, keyMaxSeq, keyTypeSeek) - } - if slice.Limit != nil { - islice.Limit = makeInternalKey(nil, slice.Limit, keyMaxSeq, keyTypeSeek) - } - } - rawIter := db.newRawIterator(auxm, auxt, islice, ro) - iter := &dbIter{ - db: db, - icmp: db.s.icmp, - iter: rawIter, - seq: seq, - strict: opt.GetStrict(db.s.o.Options, ro, opt.StrictReader), - key: make([]byte, 0), - value: make([]byte, 0), - } - atomic.AddInt32(&db.aliveIters, 1) - runtime.SetFinalizer(iter, (*dbIter).Release) - return iter -} - -func (db *DB) iterSamplingRate() int { - return rand.Intn(2 * db.s.o.GetIteratorSamplingRate()) -} - -type dir int - -const ( - dirReleased dir = iota - 1 - dirSOI - dirEOI - dirBackward - dirForward -) - -// dbIter represent an interator states over a database session. -type dbIter struct { - db *DB - icmp *iComparer - iter iterator.Iterator - seq uint64 - strict bool - - smaplingGap int - dir dir - key []byte - value []byte - err error - releaser util.Releaser -} - -func (i *dbIter) sampleSeek() { - ikey := i.iter.Key() - i.smaplingGap -= len(ikey) + len(i.iter.Value()) - for i.smaplingGap < 0 { - i.smaplingGap += i.db.iterSamplingRate() - i.db.sampleSeek(ikey) - } -} - -func (i *dbIter) setErr(err error) { - i.err = err - i.key = nil - i.value = nil -} - -func (i *dbIter) iterErr() { - if err := i.iter.Error(); err != nil { - i.setErr(err) - } -} - -func (i *dbIter) Valid() bool { - return i.err == nil && i.dir > dirEOI -} - -func (i *dbIter) First() bool { - if i.err != nil { - return false - } else if i.dir == dirReleased { - i.err = ErrIterReleased - return false - } - - if i.iter.First() { - i.dir = dirSOI - return i.next() - } - i.dir = dirEOI - i.iterErr() - return false -} - -func (i *dbIter) Last() bool { - if i.err != nil { - return false - } else if i.dir == dirReleased { - i.err = ErrIterReleased - return false - } - - if i.iter.Last() { - return i.prev() - } - i.dir = dirSOI - i.iterErr() - return false -} - -func (i *dbIter) Seek(key []byte) bool { - if i.err != nil { - return false - } else if i.dir == dirReleased { - i.err = ErrIterReleased - return false - } - - ikey := makeInternalKey(nil, key, i.seq, keyTypeSeek) - if i.iter.Seek(ikey) { - i.dir = dirSOI - return i.next() - } - i.dir = dirEOI - i.iterErr() - return false -} - -func (i *dbIter) next() bool { - for { - if ukey, seq, kt, kerr := parseInternalKey(i.iter.Key()); kerr == nil { - i.sampleSeek() - if seq <= i.seq { - switch kt { - case keyTypeDel: - // Skip deleted key. - i.key = append(i.key[:0], ukey...) - i.dir = dirForward - case keyTypeVal: - if i.dir == dirSOI || i.icmp.uCompare(ukey, i.key) > 0 { - i.key = append(i.key[:0], ukey...) - i.value = append(i.value[:0], i.iter.Value()...) - i.dir = dirForward - return true - } - } - } - } else if i.strict { - i.setErr(kerr) - break - } - if !i.iter.Next() { - i.dir = dirEOI - i.iterErr() - break - } - } - return false -} - -func (i *dbIter) Next() bool { - if i.dir == dirEOI || i.err != nil { - return false - } else if i.dir == dirReleased { - i.err = ErrIterReleased - return false - } - - if !i.iter.Next() || (i.dir == dirBackward && !i.iter.Next()) { - i.dir = dirEOI - i.iterErr() - return false - } - return i.next() -} - -func (i *dbIter) prev() bool { - i.dir = dirBackward - del := true - if i.iter.Valid() { - for { - if ukey, seq, kt, kerr := parseInternalKey(i.iter.Key()); kerr == nil { - i.sampleSeek() - if seq <= i.seq { - if !del && i.icmp.uCompare(ukey, i.key) < 0 { - return true - } - del = (kt == keyTypeDel) - if !del { - i.key = append(i.key[:0], ukey...) - i.value = append(i.value[:0], i.iter.Value()...) - } - } - } else if i.strict { - i.setErr(kerr) - return false - } - if !i.iter.Prev() { - break - } - } - } - if del { - i.dir = dirSOI - i.iterErr() - return false - } - return true -} - -func (i *dbIter) Prev() bool { - if i.dir == dirSOI || i.err != nil { - return false - } else if i.dir == dirReleased { - i.err = ErrIterReleased - return false - } - - switch i.dir { - case dirEOI: - return i.Last() - case dirForward: - for i.iter.Prev() { - if ukey, _, _, kerr := parseInternalKey(i.iter.Key()); kerr == nil { - i.sampleSeek() - if i.icmp.uCompare(ukey, i.key) < 0 { - goto cont - } - } else if i.strict { - i.setErr(kerr) - return false - } - } - i.dir = dirSOI - i.iterErr() - return false - } - -cont: - return i.prev() -} - -func (i *dbIter) Key() []byte { - if i.err != nil || i.dir <= dirEOI { - return nil - } - return i.key -} - -func (i *dbIter) Value() []byte { - if i.err != nil || i.dir <= dirEOI { - return nil - } - return i.value -} - -func (i *dbIter) Release() { - if i.dir != dirReleased { - // Clear the finalizer. - runtime.SetFinalizer(i, nil) - - if i.releaser != nil { - i.releaser.Release() - i.releaser = nil - } - - i.dir = dirReleased - i.key = nil - i.value = nil - i.iter.Release() - i.iter = nil - atomic.AddInt32(&i.db.aliveIters, -1) - i.db = nil - } -} - -func (i *dbIter) SetReleaser(releaser util.Releaser) { - if i.dir == dirReleased { - panic(util.ErrReleased) - } - if i.releaser != nil && releaser != nil { - panic(util.ErrHasReleaser) - } - i.releaser = releaser -} - -func (i *dbIter) Error() error { - return i.err -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/db_snapshot.go b/vendor/github.com/syndtr/goleveldb/leveldb/db_snapshot.go deleted file mode 100644 index 2c69d2e..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/db_snapshot.go +++ /dev/null @@ -1,183 +0,0 @@ -// Copyright (c) 2012, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package leveldb - -import ( - "container/list" - "fmt" - "runtime" - "sync" - "sync/atomic" - - "github.com/syndtr/goleveldb/leveldb/iterator" - "github.com/syndtr/goleveldb/leveldb/opt" - "github.com/syndtr/goleveldb/leveldb/util" -) - -type snapshotElement struct { - seq uint64 - ref int - e *list.Element -} - -// Acquires a snapshot, based on latest sequence. -func (db *DB) acquireSnapshot() *snapshotElement { - db.snapsMu.Lock() - defer db.snapsMu.Unlock() - - seq := db.getSeq() - - if e := db.snapsList.Back(); e != nil { - se := e.Value.(*snapshotElement) - if se.seq == seq { - se.ref++ - return se - } else if seq < se.seq { - panic("leveldb: sequence number is not increasing") - } - } - se := &snapshotElement{seq: seq, ref: 1} - se.e = db.snapsList.PushBack(se) - return se -} - -// Releases given snapshot element. -func (db *DB) releaseSnapshot(se *snapshotElement) { - db.snapsMu.Lock() - defer db.snapsMu.Unlock() - - se.ref-- - if se.ref == 0 { - db.snapsList.Remove(se.e) - se.e = nil - } else if se.ref < 0 { - panic("leveldb: Snapshot: negative element reference") - } -} - -// Gets minimum sequence that not being snapshotted. -func (db *DB) minSeq() uint64 { - db.snapsMu.Lock() - defer db.snapsMu.Unlock() - - if e := db.snapsList.Front(); e != nil { - return e.Value.(*snapshotElement).seq - } - - return db.getSeq() -} - -// Snapshot is a DB snapshot. -type Snapshot struct { - db *DB - elem *snapshotElement - mu sync.RWMutex - released bool -} - -// Creates new snapshot object. -func (db *DB) newSnapshot() *Snapshot { - snap := &Snapshot{ - db: db, - elem: db.acquireSnapshot(), - } - atomic.AddInt32(&db.aliveSnaps, 1) - runtime.SetFinalizer(snap, (*Snapshot).Release) - return snap -} - -func (snap *Snapshot) String() string { - return fmt.Sprintf("leveldb.Snapshot{%d}", snap.elem.seq) -} - -// Get gets the value for the given key. It returns ErrNotFound if -// the DB does not contains the key. -// -// The caller should not modify the contents of the returned slice, but -// it is safe to modify the contents of the argument after Get returns. -func (snap *Snapshot) Get(key []byte, ro *opt.ReadOptions) (value []byte, err error) { - err = snap.db.ok() - if err != nil { - return - } - snap.mu.RLock() - defer snap.mu.RUnlock() - if snap.released { - err = ErrSnapshotReleased - return - } - return snap.db.get(nil, nil, key, snap.elem.seq, ro) -} - -// Has returns true if the DB does contains the given key. -// -// It is safe to modify the contents of the argument after Get returns. -func (snap *Snapshot) Has(key []byte, ro *opt.ReadOptions) (ret bool, err error) { - err = snap.db.ok() - if err != nil { - return - } - snap.mu.RLock() - defer snap.mu.RUnlock() - if snap.released { - err = ErrSnapshotReleased - return - } - return snap.db.has(nil, nil, key, snap.elem.seq, ro) -} - -// NewIterator returns an iterator for the snapshot of the underlying DB. -// The returned iterator is not safe for concurrent use, but it is safe to use -// multiple iterators concurrently, with each in a dedicated goroutine. -// It is also safe to use an iterator concurrently with modifying its -// underlying DB. The resultant key/value pairs are guaranteed to be -// consistent. -// -// Slice allows slicing the iterator to only contains keys in the given -// range. A nil Range.Start is treated as a key before all keys in the -// DB. And a nil Range.Limit is treated as a key after all keys in -// the DB. -// -// The iterator must be released after use, by calling Release method. -// Releasing the snapshot doesn't mean releasing the iterator too, the -// iterator would be still valid until released. -// -// Also read Iterator documentation of the leveldb/iterator package. -func (snap *Snapshot) NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator { - if err := snap.db.ok(); err != nil { - return iterator.NewEmptyIterator(err) - } - snap.mu.Lock() - defer snap.mu.Unlock() - if snap.released { - return iterator.NewEmptyIterator(ErrSnapshotReleased) - } - // Since iterator already hold version ref, it doesn't need to - // hold snapshot ref. - return snap.db.newIterator(nil, nil, snap.elem.seq, slice, ro) -} - -// Release releases the snapshot. This will not release any returned -// iterators, the iterators would still be valid until released or the -// underlying DB is closed. -// -// Other methods should not be called after the snapshot has been released. -func (snap *Snapshot) Release() { - snap.mu.Lock() - defer snap.mu.Unlock() - - if !snap.released { - // Clear the finalizer. - runtime.SetFinalizer(snap, nil) - - snap.released = true - snap.db.releaseSnapshot(snap.elem) - atomic.AddInt32(&snap.db.aliveSnaps, -1) - snap.db = nil - snap.elem = nil - } -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/db_state.go b/vendor/github.com/syndtr/goleveldb/leveldb/db_state.go deleted file mode 100644 index 65e1c54..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/db_state.go +++ /dev/null @@ -1,239 +0,0 @@ -// Copyright (c) 2013, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package leveldb - -import ( - "errors" - "sync/atomic" - "time" - - "github.com/syndtr/goleveldb/leveldb/journal" - "github.com/syndtr/goleveldb/leveldb/memdb" - "github.com/syndtr/goleveldb/leveldb/storage" -) - -var ( - errHasFrozenMem = errors.New("has frozen mem") -) - -type memDB struct { - db *DB - *memdb.DB - ref int32 -} - -func (m *memDB) getref() int32 { - return atomic.LoadInt32(&m.ref) -} - -func (m *memDB) incref() { - atomic.AddInt32(&m.ref, 1) -} - -func (m *memDB) decref() { - if ref := atomic.AddInt32(&m.ref, -1); ref == 0 { - // Only put back memdb with std capacity. - if m.Capacity() == m.db.s.o.GetWriteBuffer() { - m.Reset() - m.db.mpoolPut(m.DB) - } - m.db = nil - m.DB = nil - } else if ref < 0 { - panic("negative memdb ref") - } -} - -// Get latest sequence number. -func (db *DB) getSeq() uint64 { - return atomic.LoadUint64(&db.seq) -} - -// Atomically adds delta to seq. -func (db *DB) addSeq(delta uint64) { - atomic.AddUint64(&db.seq, delta) -} - -func (db *DB) setSeq(seq uint64) { - atomic.StoreUint64(&db.seq, seq) -} - -func (db *DB) sampleSeek(ikey internalKey) { - v := db.s.version() - if v.sampleSeek(ikey) { - // Trigger table compaction. - db.compTrigger(db.tcompCmdC) - } - v.release() -} - -func (db *DB) mpoolPut(mem *memdb.DB) { - if !db.isClosed() { - select { - case db.memPool <- mem: - default: - } - } -} - -func (db *DB) mpoolGet(n int) *memDB { - var mdb *memdb.DB - select { - case mdb = <-db.memPool: - default: - } - if mdb == nil || mdb.Capacity() < n { - mdb = memdb.New(db.s.icmp, maxInt(db.s.o.GetWriteBuffer(), n)) - } - return &memDB{ - db: db, - DB: mdb, - } -} - -func (db *DB) mpoolDrain() { - ticker := time.NewTicker(30 * time.Second) - for { - select { - case <-ticker.C: - select { - case <-db.memPool: - default: - } - case <-db.closeC: - ticker.Stop() - // Make sure the pool is drained. - select { - case <-db.memPool: - case <-time.After(time.Second): - } - close(db.memPool) - return - } - } -} - -// Create new memdb and froze the old one; need external synchronization. -// newMem only called synchronously by the writer. -func (db *DB) newMem(n int) (mem *memDB, err error) { - fd := storage.FileDesc{Type: storage.TypeJournal, Num: db.s.allocFileNum()} - w, err := db.s.stor.Create(fd) - if err != nil { - db.s.reuseFileNum(fd.Num) - return - } - - db.memMu.Lock() - defer db.memMu.Unlock() - - if db.frozenMem != nil { - return nil, errHasFrozenMem - } - - if db.journal == nil { - db.journal = journal.NewWriter(w) - } else { - db.journal.Reset(w) - db.journalWriter.Close() - db.frozenJournalFd = db.journalFd - } - db.journalWriter = w - db.journalFd = fd - db.frozenMem = db.mem - mem = db.mpoolGet(n) - mem.incref() // for self - mem.incref() // for caller - db.mem = mem - // The seq only incremented by the writer. And whoever called newMem - // should hold write lock, so no need additional synchronization here. - db.frozenSeq = db.seq - return -} - -// Get all memdbs. -func (db *DB) getMems() (e, f *memDB) { - db.memMu.RLock() - defer db.memMu.RUnlock() - if db.mem != nil { - db.mem.incref() - } else if !db.isClosed() { - panic("nil effective mem") - } - if db.frozenMem != nil { - db.frozenMem.incref() - } - return db.mem, db.frozenMem -} - -// Get effective memdb. -func (db *DB) getEffectiveMem() *memDB { - db.memMu.RLock() - defer db.memMu.RUnlock() - if db.mem != nil { - db.mem.incref() - } else if !db.isClosed() { - panic("nil effective mem") - } - return db.mem -} - -// Check whether we has frozen memdb. -func (db *DB) hasFrozenMem() bool { - db.memMu.RLock() - defer db.memMu.RUnlock() - return db.frozenMem != nil -} - -// Get frozen memdb. -func (db *DB) getFrozenMem() *memDB { - db.memMu.RLock() - defer db.memMu.RUnlock() - if db.frozenMem != nil { - db.frozenMem.incref() - } - return db.frozenMem -} - -// Drop frozen memdb; assume that frozen memdb isn't nil. -func (db *DB) dropFrozenMem() { - db.memMu.Lock() - if err := db.s.stor.Remove(db.frozenJournalFd); err != nil { - db.logf("journal@remove removing @%d %q", db.frozenJournalFd.Num, err) - } else { - db.logf("journal@remove removed @%d", db.frozenJournalFd.Num) - } - db.frozenJournalFd = storage.FileDesc{} - db.frozenMem.decref() - db.frozenMem = nil - db.memMu.Unlock() -} - -// Clear mems ptr; used by DB.Close(). -func (db *DB) clearMems() { - db.memMu.Lock() - db.mem = nil - db.frozenMem = nil - db.memMu.Unlock() -} - -// Set closed flag; return true if not already closed. -func (db *DB) setClosed() bool { - return atomic.CompareAndSwapUint32(&db.closed, 0, 1) -} - -// Check whether DB was closed. -func (db *DB) isClosed() bool { - return atomic.LoadUint32(&db.closed) != 0 -} - -// Check read ok status. -func (db *DB) ok() error { - if db.isClosed() { - return ErrClosed - } - return nil -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/db_test.go b/vendor/github.com/syndtr/goleveldb/leveldb/db_test.go deleted file mode 100644 index 602376b..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/db_test.go +++ /dev/null @@ -1,2925 +0,0 @@ -// Copyright (c) 2012, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package leveldb - -import ( - "bytes" - "container/list" - crand "crypto/rand" - "encoding/binary" - "fmt" - "math/rand" - "os" - "path/filepath" - "runtime" - "strings" - "sync" - "sync/atomic" - "testing" - "time" - "unsafe" - - "github.com/onsi/gomega" - - "github.com/syndtr/goleveldb/leveldb/comparer" - "github.com/syndtr/goleveldb/leveldb/errors" - "github.com/syndtr/goleveldb/leveldb/filter" - "github.com/syndtr/goleveldb/leveldb/iterator" - "github.com/syndtr/goleveldb/leveldb/opt" - "github.com/syndtr/goleveldb/leveldb/storage" - "github.com/syndtr/goleveldb/leveldb/testutil" - "github.com/syndtr/goleveldb/leveldb/util" -) - -func tkey(i int) []byte { - return []byte(fmt.Sprintf("%016d", i)) -} - -func tval(seed, n int) []byte { - r := rand.New(rand.NewSource(int64(seed))) - return randomString(r, n) -} - -func testingLogger(t *testing.T) func(log string) { - return func(log string) { - t.Log(log) - } -} - -func testingPreserveOnFailed(t *testing.T) func() (preserve bool, err error) { - return func() (preserve bool, err error) { - preserve = t.Failed() - return - } -} - -type dbHarness struct { - t *testing.T - - stor *testutil.Storage - db *DB - o *opt.Options - ro *opt.ReadOptions - wo *opt.WriteOptions -} - -func newDbHarnessWopt(t *testing.T, o *opt.Options) *dbHarness { - h := new(dbHarness) - h.init(t, o) - return h -} - -func newDbHarness(t *testing.T) *dbHarness { - return newDbHarnessWopt(t, &opt.Options{DisableLargeBatchTransaction: true}) -} - -func (h *dbHarness) init(t *testing.T, o *opt.Options) { - gomega.RegisterTestingT(t) - h.t = t - h.stor = testutil.NewStorage() - h.stor.OnLog(testingLogger(t)) - h.stor.OnClose(testingPreserveOnFailed(t)) - h.o = o - h.ro = nil - h.wo = nil - - if err := h.openDB0(); err != nil { - // So that it will come after fatal message. - defer h.stor.Close() - h.t.Fatal("Open (init): got error: ", err) - } -} - -func (h *dbHarness) openDB0() (err error) { - h.t.Log("opening DB") - h.db, err = Open(h.stor, h.o) - return -} - -func (h *dbHarness) openDB() { - if err := h.openDB0(); err != nil { - h.t.Fatal("Open: got error: ", err) - } -} - -func (h *dbHarness) closeDB0() error { - h.t.Log("closing DB") - return h.db.Close() -} - -func (h *dbHarness) closeDB() { - if h.db != nil { - if err := h.closeDB0(); err != nil { - h.t.Error("Close: got error: ", err) - } - } - h.stor.CloseCheck() - runtime.GC() -} - -func (h *dbHarness) reopenDB() { - if h.db != nil { - h.closeDB() - } - h.openDB() -} - -func (h *dbHarness) close() { - if h.db != nil { - h.closeDB0() - h.db = nil - } - h.stor.Close() - h.stor = nil - runtime.GC() -} - -func (h *dbHarness) openAssert(want bool) { - db, err := Open(h.stor, h.o) - if err != nil { - if want { - h.t.Error("Open: assert: got error: ", err) - } else { - h.t.Log("Open: assert: got error (expected): ", err) - } - } else { - if !want { - h.t.Error("Open: assert: expect error") - } - db.Close() - } -} - -func (h *dbHarness) write(batch *Batch) { - if err := h.db.Write(batch, h.wo); err != nil { - h.t.Error("Write: got error: ", err) - } -} - -func (h *dbHarness) put(key, value string) { - if err := h.db.Put([]byte(key), []byte(value), h.wo); err != nil { - h.t.Error("Put: got error: ", err) - } -} - -func (h *dbHarness) putMulti(n int, low, hi string) { - for i := 0; i < n; i++ { - h.put(low, "begin") - h.put(hi, "end") - h.compactMem() - } -} - -func (h *dbHarness) maxNextLevelOverlappingBytes(want int64) { - t := h.t - db := h.db - - var ( - maxOverlaps int64 - maxLevel int - ) - v := db.s.version() - if len(v.levels) > 2 { - for i, tt := range v.levels[1 : len(v.levels)-1] { - level := i + 1 - next := v.levels[level+1] - for _, t := range tt { - r := next.getOverlaps(nil, db.s.icmp, t.imin.ukey(), t.imax.ukey(), false) - sum := r.size() - if sum > maxOverlaps { - maxOverlaps = sum - maxLevel = level - } - } - } - } - v.release() - - if maxOverlaps > want { - t.Errorf("next level most overlapping bytes is more than %d, got=%d level=%d", want, maxOverlaps, maxLevel) - } else { - t.Logf("next level most overlapping bytes is %d, level=%d want=%d", maxOverlaps, maxLevel, want) - } -} - -func (h *dbHarness) delete(key string) { - t := h.t - db := h.db - - err := db.Delete([]byte(key), h.wo) - if err != nil { - t.Error("Delete: got error: ", err) - } -} - -func (h *dbHarness) assertNumKeys(want int) { - iter := h.db.NewIterator(nil, h.ro) - defer iter.Release() - got := 0 - for iter.Next() { - got++ - } - if err := iter.Error(); err != nil { - h.t.Error("assertNumKeys: ", err) - } - if want != got { - h.t.Errorf("assertNumKeys: want=%d got=%d", want, got) - } -} - -func (h *dbHarness) getr(db Reader, key string, expectFound bool) (found bool, v []byte) { - t := h.t - v, err := db.Get([]byte(key), h.ro) - switch err { - case ErrNotFound: - if expectFound { - t.Errorf("Get: key '%s' not found, want found", key) - } - case nil: - found = true - if !expectFound { - t.Errorf("Get: key '%s' found, want not found", key) - } - default: - t.Error("Get: got error: ", err) - } - return -} - -func (h *dbHarness) get(key string, expectFound bool) (found bool, v []byte) { - return h.getr(h.db, key, expectFound) -} - -func (h *dbHarness) getValr(db Reader, key, value string) { - t := h.t - found, r := h.getr(db, key, true) - if !found { - return - } - rval := string(r) - if rval != value { - t.Errorf("Get: invalid value, got '%s', want '%s'", rval, value) - } -} - -func (h *dbHarness) getVal(key, value string) { - h.getValr(h.db, key, value) -} - -func (h *dbHarness) allEntriesFor(key, want string) { - t := h.t - db := h.db - s := db.s - - ikey := makeInternalKey(nil, []byte(key), keyMaxSeq, keyTypeVal) - iter := db.newRawIterator(nil, nil, nil, nil) - if !iter.Seek(ikey) && iter.Error() != nil { - t.Error("AllEntries: error during seek, err: ", iter.Error()) - return - } - res := "[ " - first := true - for iter.Valid() { - if ukey, _, kt, kerr := parseInternalKey(iter.Key()); kerr == nil { - if s.icmp.uCompare(ikey.ukey(), ukey) != 0 { - break - } - if !first { - res += ", " - } - first = false - switch kt { - case keyTypeVal: - res += string(iter.Value()) - case keyTypeDel: - res += "DEL" - } - } else { - if !first { - res += ", " - } - first = false - res += "CORRUPTED" - } - iter.Next() - } - if !first { - res += " " - } - res += "]" - if res != want { - t.Errorf("AllEntries: assert failed for key %q, got=%q want=%q", key, res, want) - } -} - -// Return a string that contains all key,value pairs in order, -// formatted like "(k1->v1)(k2->v2)". -func (h *dbHarness) getKeyVal(want string) { - t := h.t - db := h.db - - s, err := db.GetSnapshot() - if err != nil { - t.Fatal("GetSnapshot: got error: ", err) - } - res := "" - iter := s.NewIterator(nil, nil) - for iter.Next() { - res += fmt.Sprintf("(%s->%s)", string(iter.Key()), string(iter.Value())) - } - iter.Release() - - if res != want { - t.Errorf("GetKeyVal: invalid key/value pair, got=%q want=%q", res, want) - } - s.Release() -} - -func (h *dbHarness) waitCompaction() { - t := h.t - db := h.db - if err := db.compTriggerWait(db.tcompCmdC); err != nil { - t.Error("compaction error: ", err) - } -} - -func (h *dbHarness) waitMemCompaction() { - t := h.t - db := h.db - - if err := db.compTriggerWait(db.mcompCmdC); err != nil { - t.Error("compaction error: ", err) - } -} - -func (h *dbHarness) compactMem() { - t := h.t - db := h.db - - t.Log("starting memdb compaction") - - db.writeLockC <- struct{}{} - defer func() { - <-db.writeLockC - }() - - if _, err := db.rotateMem(0, true); err != nil { - t.Error("compaction error: ", err) - } - - if h.totalTables() == 0 { - t.Error("zero tables after mem compaction") - } - - t.Log("memdb compaction done") -} - -func (h *dbHarness) compactRangeAtErr(level int, min, max string, wanterr bool) { - t := h.t - db := h.db - - var _min, _max []byte - if min != "" { - _min = []byte(min) - } - if max != "" { - _max = []byte(max) - } - - t.Logf("starting table range compaction: level=%d, min=%q, max=%q", level, min, max) - - if err := db.compTriggerRange(db.tcompCmdC, level, _min, _max); err != nil { - if wanterr { - t.Log("CompactRangeAt: got error (expected): ", err) - } else { - t.Error("CompactRangeAt: got error: ", err) - } - } else if wanterr { - t.Error("CompactRangeAt: expect error") - } - - t.Log("table range compaction done") -} - -func (h *dbHarness) compactRangeAt(level int, min, max string) { - h.compactRangeAtErr(level, min, max, false) -} - -func (h *dbHarness) compactRange(min, max string) { - t := h.t - db := h.db - - t.Logf("starting DB range compaction: min=%q, max=%q", min, max) - - var r util.Range - if min != "" { - r.Start = []byte(min) - } - if max != "" { - r.Limit = []byte(max) - } - if err := db.CompactRange(r); err != nil { - t.Error("CompactRange: got error: ", err) - } - - t.Log("DB range compaction done") -} - -func (h *dbHarness) sizeOf(start, limit string) int64 { - sz, err := h.db.SizeOf([]util.Range{ - {[]byte(start), []byte(limit)}, - }) - if err != nil { - h.t.Error("SizeOf: got error: ", err) - } - return sz.Sum() -} - -func (h *dbHarness) sizeAssert(start, limit string, low, hi int64) { - sz := h.sizeOf(start, limit) - if sz < low || sz > hi { - h.t.Errorf("sizeOf %q to %q not in range, want %d - %d, got %d", - shorten(start), shorten(limit), low, hi, sz) - } -} - -func (h *dbHarness) getSnapshot() (s *Snapshot) { - s, err := h.db.GetSnapshot() - if err != nil { - h.t.Fatal("GetSnapshot: got error: ", err) - } - return -} - -func (h *dbHarness) getTablesPerLevel() string { - res := "" - nz := 0 - v := h.db.s.version() - for level, tables := range v.levels { - if level > 0 { - res += "," - } - res += fmt.Sprint(len(tables)) - if len(tables) > 0 { - nz = len(res) - } - } - v.release() - return res[:nz] -} - -func (h *dbHarness) tablesPerLevel(want string) { - res := h.getTablesPerLevel() - if res != want { - h.t.Errorf("invalid tables len, want=%s, got=%s", want, res) - } -} - -func (h *dbHarness) totalTables() (n int) { - v := h.db.s.version() - for _, tables := range v.levels { - n += len(tables) - } - v.release() - return -} - -type keyValue interface { - Key() []byte - Value() []byte -} - -func testKeyVal(t *testing.T, kv keyValue, want string) { - res := string(kv.Key()) + "->" + string(kv.Value()) - if res != want { - t.Errorf("invalid key/value, want=%q, got=%q", want, res) - } -} - -func numKey(num int) string { - return fmt.Sprintf("key%06d", num) -} - -var testingBloomFilter = filter.NewBloomFilter(10) - -func truno(t *testing.T, o *opt.Options, f func(h *dbHarness)) { - for i := 0; i < 4; i++ { - func() { - switch i { - case 0: - case 1: - if o == nil { - o = &opt.Options{ - DisableLargeBatchTransaction: true, - Filter: testingBloomFilter, - } - } else { - old := o - o = &opt.Options{} - *o = *old - o.Filter = testingBloomFilter - } - case 2: - if o == nil { - o = &opt.Options{ - DisableLargeBatchTransaction: true, - Compression: opt.NoCompression, - } - } else { - old := o - o = &opt.Options{} - *o = *old - o.Compression = opt.NoCompression - } - } - h := newDbHarnessWopt(t, o) - defer h.close() - switch i { - case 3: - h.reopenDB() - } - f(h) - }() - } -} - -func trun(t *testing.T, f func(h *dbHarness)) { - truno(t, nil, f) -} - -func testAligned(t *testing.T, name string, offset uintptr) { - if offset%8 != 0 { - t.Errorf("field %s offset is not 64-bit aligned", name) - } -} - -func Test_FieldsAligned(t *testing.T) { - p1 := new(DB) - testAligned(t, "DB.seq", unsafe.Offsetof(p1.seq)) - p2 := new(session) - testAligned(t, "session.stNextFileNum", unsafe.Offsetof(p2.stNextFileNum)) - testAligned(t, "session.stJournalNum", unsafe.Offsetof(p2.stJournalNum)) - testAligned(t, "session.stPrevJournalNum", unsafe.Offsetof(p2.stPrevJournalNum)) - testAligned(t, "session.stSeqNum", unsafe.Offsetof(p2.stSeqNum)) -} - -func TestDB_Locking(t *testing.T) { - h := newDbHarness(t) - defer h.stor.Close() - h.openAssert(false) - h.closeDB() - h.openAssert(true) -} - -func TestDB_Empty(t *testing.T) { - trun(t, func(h *dbHarness) { - h.get("foo", false) - - h.reopenDB() - h.get("foo", false) - }) -} - -func TestDB_ReadWrite(t *testing.T) { - trun(t, func(h *dbHarness) { - h.put("foo", "v1") - h.getVal("foo", "v1") - h.put("bar", "v2") - h.put("foo", "v3") - h.getVal("foo", "v3") - h.getVal("bar", "v2") - - h.reopenDB() - h.getVal("foo", "v3") - h.getVal("bar", "v2") - }) -} - -func TestDB_PutDeleteGet(t *testing.T) { - trun(t, func(h *dbHarness) { - h.put("foo", "v1") - h.getVal("foo", "v1") - h.put("foo", "v2") - h.getVal("foo", "v2") - h.delete("foo") - h.get("foo", false) - - h.reopenDB() - h.get("foo", false) - }) -} - -func TestDB_EmptyBatch(t *testing.T) { - h := newDbHarness(t) - defer h.close() - - h.get("foo", false) - err := h.db.Write(new(Batch), h.wo) - if err != nil { - t.Error("writing empty batch yield error: ", err) - } - h.get("foo", false) -} - -func TestDB_GetFromFrozen(t *testing.T) { - h := newDbHarnessWopt(t, &opt.Options{ - DisableLargeBatchTransaction: true, - WriteBuffer: 100100, - }) - defer h.close() - - h.put("foo", "v1") - h.getVal("foo", "v1") - - h.stor.Stall(testutil.ModeSync, storage.TypeTable) // Block sync calls - h.put("k1", strings.Repeat("x", 100000)) // Fill memtable - h.put("k2", strings.Repeat("y", 100000)) // Trigger compaction - for i := 0; h.db.getFrozenMem() == nil && i < 100; i++ { - time.Sleep(10 * time.Microsecond) - } - if h.db.getFrozenMem() == nil { - h.stor.Release(testutil.ModeSync, storage.TypeTable) - t.Fatal("No frozen mem") - } - h.getVal("foo", "v1") - h.stor.Release(testutil.ModeSync, storage.TypeTable) // Release sync calls - - h.reopenDB() - h.getVal("foo", "v1") - h.get("k1", true) - h.get("k2", true) -} - -func TestDB_GetFromTable(t *testing.T) { - trun(t, func(h *dbHarness) { - h.put("foo", "v1") - h.compactMem() - h.getVal("foo", "v1") - }) -} - -func TestDB_GetSnapshot(t *testing.T) { - trun(t, func(h *dbHarness) { - bar := strings.Repeat("b", 200) - h.put("foo", "v1") - h.put(bar, "v1") - - snap, err := h.db.GetSnapshot() - if err != nil { - t.Fatal("GetSnapshot: got error: ", err) - } - - h.put("foo", "v2") - h.put(bar, "v2") - - h.getVal("foo", "v2") - h.getVal(bar, "v2") - h.getValr(snap, "foo", "v1") - h.getValr(snap, bar, "v1") - - h.compactMem() - - h.getVal("foo", "v2") - h.getVal(bar, "v2") - h.getValr(snap, "foo", "v1") - h.getValr(snap, bar, "v1") - - snap.Release() - - h.reopenDB() - h.getVal("foo", "v2") - h.getVal(bar, "v2") - }) -} - -func TestDB_GetLevel0Ordering(t *testing.T) { - trun(t, func(h *dbHarness) { - h.db.memdbMaxLevel = 2 - - for i := 0; i < 4; i++ { - h.put("bar", fmt.Sprintf("b%d", i)) - h.put("foo", fmt.Sprintf("v%d", i)) - h.compactMem() - } - h.getVal("foo", "v3") - h.getVal("bar", "b3") - - v := h.db.s.version() - t0len := v.tLen(0) - v.release() - if t0len < 2 { - t.Errorf("level-0 tables is less than 2, got %d", t0len) - } - - h.reopenDB() - h.getVal("foo", "v3") - h.getVal("bar", "b3") - }) -} - -func TestDB_GetOrderedByLevels(t *testing.T) { - trun(t, func(h *dbHarness) { - h.put("foo", "v1") - h.compactMem() - h.compactRange("a", "z") - h.getVal("foo", "v1") - h.put("foo", "v2") - h.compactMem() - h.getVal("foo", "v2") - }) -} - -func TestDB_GetPicksCorrectFile(t *testing.T) { - trun(t, func(h *dbHarness) { - // Arrange to have multiple files in a non-level-0 level. - h.put("a", "va") - h.compactMem() - h.compactRange("a", "b") - h.put("x", "vx") - h.compactMem() - h.compactRange("x", "y") - h.put("f", "vf") - h.compactMem() - h.compactRange("f", "g") - - h.getVal("a", "va") - h.getVal("f", "vf") - h.getVal("x", "vx") - - h.compactRange("", "") - h.getVal("a", "va") - h.getVal("f", "vf") - h.getVal("x", "vx") - }) -} - -func TestDB_GetEncountersEmptyLevel(t *testing.T) { - trun(t, func(h *dbHarness) { - h.db.memdbMaxLevel = 2 - - // Arrange for the following to happen: - // * sstable A in level 0 - // * nothing in level 1 - // * sstable B in level 2 - // Then do enough Get() calls to arrange for an automatic compaction - // of sstable A. A bug would cause the compaction to be marked as - // occuring at level 1 (instead of the correct level 0). - - // Step 1: First place sstables in levels 0 and 2 - for i := 0; ; i++ { - if i >= 100 { - t.Fatal("could not fill levels-0 and level-2") - } - v := h.db.s.version() - if v.tLen(0) > 0 && v.tLen(2) > 0 { - v.release() - break - } - v.release() - h.put("a", "begin") - h.put("z", "end") - h.compactMem() - - h.getVal("a", "begin") - h.getVal("z", "end") - } - - // Step 2: clear level 1 if necessary. - h.compactRangeAt(1, "", "") - h.tablesPerLevel("1,0,1") - - h.getVal("a", "begin") - h.getVal("z", "end") - - // Step 3: read a bunch of times - for i := 0; i < 200; i++ { - h.get("missing", false) - } - - // Step 4: Wait for compaction to finish - h.waitCompaction() - - v := h.db.s.version() - if v.tLen(0) > 0 { - t.Errorf("level-0 tables more than 0, got %d", v.tLen(0)) - } - v.release() - - h.getVal("a", "begin") - h.getVal("z", "end") - }) -} - -func TestDB_IterMultiWithDelete(t *testing.T) { - trun(t, func(h *dbHarness) { - h.put("a", "va") - h.put("b", "vb") - h.put("c", "vc") - h.delete("b") - h.get("b", false) - - iter := h.db.NewIterator(nil, nil) - iter.Seek([]byte("c")) - testKeyVal(t, iter, "c->vc") - iter.Prev() - testKeyVal(t, iter, "a->va") - iter.Release() - - h.compactMem() - - iter = h.db.NewIterator(nil, nil) - iter.Seek([]byte("c")) - testKeyVal(t, iter, "c->vc") - iter.Prev() - testKeyVal(t, iter, "a->va") - iter.Release() - }) -} - -func TestDB_IteratorPinsRef(t *testing.T) { - h := newDbHarness(t) - defer h.close() - - h.put("foo", "hello") - - // Get iterator that will yield the current contents of the DB. - iter := h.db.NewIterator(nil, nil) - - // Write to force compactions - h.put("foo", "newvalue1") - for i := 0; i < 100; i++ { - h.put(numKey(i), strings.Repeat(fmt.Sprintf("v%09d", i), 100000/10)) - } - h.put("foo", "newvalue2") - - iter.First() - testKeyVal(t, iter, "foo->hello") - if iter.Next() { - t.Errorf("expect eof") - } - iter.Release() -} - -func TestDB_Recover(t *testing.T) { - trun(t, func(h *dbHarness) { - h.put("foo", "v1") - h.put("baz", "v5") - - h.reopenDB() - h.getVal("foo", "v1") - - h.getVal("foo", "v1") - h.getVal("baz", "v5") - h.put("bar", "v2") - h.put("foo", "v3") - - h.reopenDB() - h.getVal("foo", "v3") - h.put("foo", "v4") - h.getVal("foo", "v4") - h.getVal("bar", "v2") - h.getVal("baz", "v5") - }) -} - -func TestDB_RecoverWithEmptyJournal(t *testing.T) { - trun(t, func(h *dbHarness) { - h.put("foo", "v1") - h.put("foo", "v2") - - h.reopenDB() - h.reopenDB() - h.put("foo", "v3") - - h.reopenDB() - h.getVal("foo", "v3") - }) -} - -func TestDB_RecoverDuringMemtableCompaction(t *testing.T) { - truno(t, &opt.Options{DisableLargeBatchTransaction: true, WriteBuffer: 1000000}, func(h *dbHarness) { - - h.stor.Stall(testutil.ModeSync, storage.TypeTable) - h.put("big1", strings.Repeat("x", 10000000)) - h.put("big2", strings.Repeat("y", 1000)) - h.put("bar", "v2") - h.stor.Release(testutil.ModeSync, storage.TypeTable) - - h.reopenDB() - h.getVal("bar", "v2") - h.getVal("big1", strings.Repeat("x", 10000000)) - h.getVal("big2", strings.Repeat("y", 1000)) - }) -} - -func TestDB_MinorCompactionsHappen(t *testing.T) { - h := newDbHarnessWopt(t, &opt.Options{DisableLargeBatchTransaction: true, WriteBuffer: 10000}) - defer h.close() - - n := 500 - - key := func(i int) string { - return fmt.Sprintf("key%06d", i) - } - - for i := 0; i < n; i++ { - h.put(key(i), key(i)+strings.Repeat("v", 1000)) - } - - for i := 0; i < n; i++ { - h.getVal(key(i), key(i)+strings.Repeat("v", 1000)) - } - - h.reopenDB() - for i := 0; i < n; i++ { - h.getVal(key(i), key(i)+strings.Repeat("v", 1000)) - } -} - -func TestDB_RecoverWithLargeJournal(t *testing.T) { - h := newDbHarness(t) - defer h.close() - - h.put("big1", strings.Repeat("1", 200000)) - h.put("big2", strings.Repeat("2", 200000)) - h.put("small3", strings.Repeat("3", 10)) - h.put("small4", strings.Repeat("4", 10)) - h.tablesPerLevel("") - - // Make sure that if we re-open with a small write buffer size that - // we flush table files in the middle of a large journal file. - h.o.WriteBuffer = 100000 - h.reopenDB() - h.getVal("big1", strings.Repeat("1", 200000)) - h.getVal("big2", strings.Repeat("2", 200000)) - h.getVal("small3", strings.Repeat("3", 10)) - h.getVal("small4", strings.Repeat("4", 10)) - v := h.db.s.version() - if v.tLen(0) <= 1 { - t.Errorf("tables-0 less than one") - } - v.release() -} - -func TestDB_CompactionsGenerateMultipleFiles(t *testing.T) { - h := newDbHarnessWopt(t, &opt.Options{ - DisableLargeBatchTransaction: true, - WriteBuffer: 10000000, - Compression: opt.NoCompression, - }) - defer h.close() - - v := h.db.s.version() - if v.tLen(0) > 0 { - t.Errorf("level-0 tables more than 0, got %d", v.tLen(0)) - } - v.release() - - n := 80 - - // Write 8MB (80 values, each 100K) - for i := 0; i < n; i++ { - h.put(numKey(i), strings.Repeat(fmt.Sprintf("v%09d", i), 100000/10)) - } - - // Reopening moves updates to level-0 - h.reopenDB() - h.compactRangeAt(0, "", "") - - v = h.db.s.version() - if v.tLen(0) > 0 { - t.Errorf("level-0 tables more than 0, got %d", v.tLen(0)) - } - if v.tLen(1) <= 1 { - t.Errorf("level-1 tables less than 1, got %d", v.tLen(1)) - } - v.release() - - for i := 0; i < n; i++ { - h.getVal(numKey(i), strings.Repeat(fmt.Sprintf("v%09d", i), 100000/10)) - } -} - -func TestDB_RepeatedWritesToSameKey(t *testing.T) { - h := newDbHarnessWopt(t, &opt.Options{DisableLargeBatchTransaction: true, WriteBuffer: 100000}) - defer h.close() - - maxTables := h.o.GetWriteL0PauseTrigger() + 7 - - value := strings.Repeat("v", 2*h.o.GetWriteBuffer()) - for i := 0; i < 5*maxTables; i++ { - h.put("key", value) - n := h.totalTables() - if n > maxTables { - t.Errorf("total tables exceed %d, got=%d, iter=%d", maxTables, n, i) - } - } -} - -func TestDB_RepeatedWritesToSameKeyAfterReopen(t *testing.T) { - h := newDbHarnessWopt(t, &opt.Options{ - DisableLargeBatchTransaction: true, - WriteBuffer: 100000, - }) - defer h.close() - - h.reopenDB() - - maxTables := h.o.GetWriteL0PauseTrigger() + 7 - - value := strings.Repeat("v", 2*h.o.GetWriteBuffer()) - for i := 0; i < 5*maxTables; i++ { - h.put("key", value) - n := h.totalTables() - if n > maxTables { - t.Errorf("total tables exceed %d, got=%d, iter=%d", maxTables, n, i) - } - } -} - -func TestDB_SparseMerge(t *testing.T) { - h := newDbHarnessWopt(t, &opt.Options{DisableLargeBatchTransaction: true, Compression: opt.NoCompression}) - defer h.close() - - h.putMulti(7, "A", "Z") - - // Suppose there is: - // small amount of data with prefix A - // large amount of data with prefix B - // small amount of data with prefix C - // and that recent updates have made small changes to all three prefixes. - // Check that we do not do a compaction that merges all of B in one shot. - h.put("A", "va") - value := strings.Repeat("x", 1000) - for i := 0; i < 100000; i++ { - h.put(fmt.Sprintf("B%010d", i), value) - } - h.put("C", "vc") - h.compactMem() - h.compactRangeAt(0, "", "") - h.waitCompaction() - - // Make sparse update - h.put("A", "va2") - h.put("B100", "bvalue2") - h.put("C", "vc2") - h.compactMem() - - h.waitCompaction() - h.maxNextLevelOverlappingBytes(20 * 1048576) - h.compactRangeAt(0, "", "") - h.waitCompaction() - h.maxNextLevelOverlappingBytes(20 * 1048576) - h.compactRangeAt(1, "", "") - h.waitCompaction() - h.maxNextLevelOverlappingBytes(20 * 1048576) -} - -func TestDB_SizeOf(t *testing.T) { - h := newDbHarnessWopt(t, &opt.Options{ - DisableLargeBatchTransaction: true, - Compression: opt.NoCompression, - WriteBuffer: 10000000, - }) - defer h.close() - - h.sizeAssert("", "xyz", 0, 0) - h.reopenDB() - h.sizeAssert("", "xyz", 0, 0) - - // Write 8MB (80 values, each 100K) - n := 80 - s1 := 100000 - s2 := 105000 - - for i := 0; i < n; i++ { - h.put(numKey(i), strings.Repeat(fmt.Sprintf("v%09d", i), s1/10)) - } - - // 0 because SizeOf() does not account for memtable space - h.sizeAssert("", numKey(50), 0, 0) - - for r := 0; r < 3; r++ { - h.reopenDB() - - for cs := 0; cs < n; cs += 10 { - for i := 0; i < n; i += 10 { - h.sizeAssert("", numKey(i), int64(s1*i), int64(s2*i)) - h.sizeAssert("", numKey(i)+".suffix", int64(s1*(i+1)), int64(s2*(i+1))) - h.sizeAssert(numKey(i), numKey(i+10), int64(s1*10), int64(s2*10)) - } - - h.sizeAssert("", numKey(50), int64(s1*50), int64(s2*50)) - h.sizeAssert("", numKey(50)+".suffix", int64(s1*50), int64(s2*50)) - - h.compactRangeAt(0, numKey(cs), numKey(cs+9)) - } - - v := h.db.s.version() - if v.tLen(0) != 0 { - t.Errorf("level-0 tables was not zero, got %d", v.tLen(0)) - } - if v.tLen(1) == 0 { - t.Error("level-1 tables was zero") - } - v.release() - } -} - -func TestDB_SizeOf_MixOfSmallAndLarge(t *testing.T) { - h := newDbHarnessWopt(t, &opt.Options{ - DisableLargeBatchTransaction: true, - Compression: opt.NoCompression, - }) - defer h.close() - - sizes := []int64{ - 10000, - 10000, - 100000, - 10000, - 100000, - 10000, - 300000, - 10000, - } - - for i, n := range sizes { - h.put(numKey(i), strings.Repeat(fmt.Sprintf("v%09d", i), int(n)/10)) - } - - for r := 0; r < 3; r++ { - h.reopenDB() - - var x int64 - for i, n := range sizes { - y := x - if i > 0 { - y += 1000 - } - h.sizeAssert("", numKey(i), x, y) - x += n - } - - h.sizeAssert(numKey(3), numKey(5), 110000, 111000) - - h.compactRangeAt(0, "", "") - } -} - -func TestDB_Snapshot(t *testing.T) { - trun(t, func(h *dbHarness) { - h.put("foo", "v1") - s1 := h.getSnapshot() - h.put("foo", "v2") - s2 := h.getSnapshot() - h.put("foo", "v3") - s3 := h.getSnapshot() - h.put("foo", "v4") - - h.getValr(s1, "foo", "v1") - h.getValr(s2, "foo", "v2") - h.getValr(s3, "foo", "v3") - h.getVal("foo", "v4") - - s3.Release() - h.getValr(s1, "foo", "v1") - h.getValr(s2, "foo", "v2") - h.getVal("foo", "v4") - - s1.Release() - h.getValr(s2, "foo", "v2") - h.getVal("foo", "v4") - - s2.Release() - h.getVal("foo", "v4") - }) -} - -func TestDB_SnapshotList(t *testing.T) { - db := &DB{snapsList: list.New()} - e0a := db.acquireSnapshot() - e0b := db.acquireSnapshot() - db.seq = 1 - e1 := db.acquireSnapshot() - db.seq = 2 - e2 := db.acquireSnapshot() - - if db.minSeq() != 0 { - t.Fatalf("invalid sequence number, got=%d", db.minSeq()) - } - db.releaseSnapshot(e0a) - if db.minSeq() != 0 { - t.Fatalf("invalid sequence number, got=%d", db.minSeq()) - } - db.releaseSnapshot(e2) - if db.minSeq() != 0 { - t.Fatalf("invalid sequence number, got=%d", db.minSeq()) - } - db.releaseSnapshot(e0b) - if db.minSeq() != 1 { - t.Fatalf("invalid sequence number, got=%d", db.minSeq()) - } - e2 = db.acquireSnapshot() - if db.minSeq() != 1 { - t.Fatalf("invalid sequence number, got=%d", db.minSeq()) - } - db.releaseSnapshot(e1) - if db.minSeq() != 2 { - t.Fatalf("invalid sequence number, got=%d", db.minSeq()) - } - db.releaseSnapshot(e2) - if db.minSeq() != 2 { - t.Fatalf("invalid sequence number, got=%d", db.minSeq()) - } -} - -func TestDB_HiddenValuesAreRemoved(t *testing.T) { - trun(t, func(h *dbHarness) { - s := h.db.s - - m := 2 - h.db.memdbMaxLevel = m - - h.put("foo", "v1") - h.compactMem() - v := s.version() - num := v.tLen(m) - v.release() - if num != 1 { - t.Errorf("invalid level-%d len, want=1 got=%d", m, num) - } - - // Place a table at level last-1 to prevent merging with preceding mutation - h.put("a", "begin") - h.put("z", "end") - h.compactMem() - v = s.version() - if v.tLen(m) != 1 { - t.Errorf("invalid level-%d len, want=1 got=%d", m, v.tLen(m)) - } - if v.tLen(m-1) != 1 { - t.Errorf("invalid level-%d len, want=1 got=%d", m-1, v.tLen(m-1)) - } - v.release() - - h.delete("foo") - h.put("foo", "v2") - h.allEntriesFor("foo", "[ v2, DEL, v1 ]") - h.compactMem() - h.allEntriesFor("foo", "[ v2, DEL, v1 ]") - h.compactRangeAt(m-2, "", "z") - // DEL eliminated, but v1 remains because we aren't compacting that level - // (DEL can be eliminated because v2 hides v1). - h.allEntriesFor("foo", "[ v2, v1 ]") - h.compactRangeAt(m-1, "", "") - // Merging last-1 w/ last, so we are the base level for "foo", so - // DEL is removed. (as is v1). - h.allEntriesFor("foo", "[ v2 ]") - }) -} - -func TestDB_DeletionMarkers2(t *testing.T) { - h := newDbHarness(t) - defer h.close() - s := h.db.s - - m := 2 - h.db.memdbMaxLevel = m - - h.put("foo", "v1") - h.compactMem() - v := s.version() - num := v.tLen(m) - v.release() - if num != 1 { - t.Errorf("invalid level-%d len, want=1 got=%d", m, num) - } - - // Place a table at level last-1 to prevent merging with preceding mutation - h.put("a", "begin") - h.put("z", "end") - h.compactMem() - v = s.version() - if v.tLen(m) != 1 { - t.Errorf("invalid level-%d len, want=1 got=%d", m, v.tLen(m)) - } - if v.tLen(m-1) != 1 { - t.Errorf("invalid level-%d len, want=1 got=%d", m-1, v.tLen(m-1)) - } - v.release() - - h.delete("foo") - h.allEntriesFor("foo", "[ DEL, v1 ]") - h.compactMem() // Moves to level last-2 - h.allEntriesFor("foo", "[ DEL, v1 ]") - h.compactRangeAt(m-2, "", "") - // DEL kept: "last" file overlaps - h.allEntriesFor("foo", "[ DEL, v1 ]") - h.compactRangeAt(m-1, "", "") - // Merging last-1 w/ last, so we are the base level for "foo", so - // DEL is removed. (as is v1). - h.allEntriesFor("foo", "[ ]") -} - -func TestDB_CompactionTableOpenError(t *testing.T) { - h := newDbHarnessWopt(t, &opt.Options{ - DisableLargeBatchTransaction: true, - OpenFilesCacheCapacity: -1, - }) - defer h.close() - - h.db.memdbMaxLevel = 2 - - im := 10 - jm := 10 - for r := 0; r < 2; r++ { - for i := 0; i < im; i++ { - for j := 0; j < jm; j++ { - h.put(fmt.Sprintf("k%d,%d", i, j), fmt.Sprintf("v%d,%d", i, j)) - } - h.compactMem() - } - } - - if n := h.totalTables(); n != im*2 { - t.Errorf("total tables is %d, want %d", n, im*2) - } - - h.stor.EmulateError(testutil.ModeOpen, storage.TypeTable, errors.New("open error during table compaction")) - go h.db.CompactRange(util.Range{}) - if err := h.db.compTriggerWait(h.db.tcompCmdC); err != nil { - t.Log("compaction error: ", err) - } - h.closeDB0() - h.openDB() - h.stor.EmulateError(testutil.ModeOpen, storage.TypeTable, nil) - - for i := 0; i < im; i++ { - for j := 0; j < jm; j++ { - h.getVal(fmt.Sprintf("k%d,%d", i, j), fmt.Sprintf("v%d,%d", i, j)) - } - } -} - -func TestDB_OverlapInLevel0(t *testing.T) { - trun(t, func(h *dbHarness) { - h.db.memdbMaxLevel = 2 - - // Fill levels 1 and 2 to disable the pushing of new memtables to levels > 0. - h.put("100", "v100") - h.put("999", "v999") - h.compactMem() - h.delete("100") - h.delete("999") - h.compactMem() - h.tablesPerLevel("0,1,1") - - // Make files spanning the following ranges in level-0: - // files[0] 200 .. 900 - // files[1] 300 .. 500 - // Note that files are sorted by min key. - h.put("300", "v300") - h.put("500", "v500") - h.compactMem() - h.put("200", "v200") - h.put("600", "v600") - h.put("900", "v900") - h.compactMem() - h.tablesPerLevel("2,1,1") - - // Compact away the placeholder files we created initially - h.compactRangeAt(1, "", "") - h.compactRangeAt(2, "", "") - h.tablesPerLevel("2") - - // Do a memtable compaction. Before bug-fix, the compaction would - // not detect the overlap with level-0 files and would incorrectly place - // the deletion in a deeper level. - h.delete("600") - h.compactMem() - h.tablesPerLevel("3") - h.get("600", false) - }) -} - -func TestDB_L0_CompactionBug_Issue44_a(t *testing.T) { - h := newDbHarness(t) - defer h.close() - - h.reopenDB() - h.put("b", "v") - h.reopenDB() - h.delete("b") - h.delete("a") - h.reopenDB() - h.delete("a") - h.reopenDB() - h.put("a", "v") - h.reopenDB() - h.reopenDB() - h.getKeyVal("(a->v)") - h.waitCompaction() - h.getKeyVal("(a->v)") -} - -func TestDB_L0_CompactionBug_Issue44_b(t *testing.T) { - h := newDbHarness(t) - defer h.close() - - h.reopenDB() - h.put("", "") - h.reopenDB() - h.delete("e") - h.put("", "") - h.reopenDB() - h.put("c", "cv") - h.reopenDB() - h.put("", "") - h.reopenDB() - h.put("", "") - h.waitCompaction() - h.reopenDB() - h.put("d", "dv") - h.reopenDB() - h.put("", "") - h.reopenDB() - h.delete("d") - h.delete("b") - h.reopenDB() - h.getKeyVal("(->)(c->cv)") - h.waitCompaction() - h.getKeyVal("(->)(c->cv)") -} - -func TestDB_SingleEntryMemCompaction(t *testing.T) { - trun(t, func(h *dbHarness) { - for i := 0; i < 10; i++ { - h.put("big", strings.Repeat("v", opt.DefaultWriteBuffer)) - h.compactMem() - h.put("key", strings.Repeat("v", opt.DefaultBlockSize)) - h.compactMem() - h.put("k", "v") - h.compactMem() - h.put("", "") - h.compactMem() - h.put("verybig", strings.Repeat("v", opt.DefaultWriteBuffer*2)) - h.compactMem() - } - }) -} - -func TestDB_ManifestWriteError(t *testing.T) { - for i := 0; i < 2; i++ { - func() { - h := newDbHarness(t) - defer h.close() - - h.put("foo", "bar") - h.getVal("foo", "bar") - - // Mem compaction (will succeed) - h.compactMem() - h.getVal("foo", "bar") - v := h.db.s.version() - if n := v.tLen(0); n != 1 { - t.Errorf("invalid total tables, want=1 got=%d", n) - } - v.release() - - if i == 0 { - h.stor.EmulateError(testutil.ModeWrite, storage.TypeManifest, errors.New("manifest write error")) - } else { - h.stor.EmulateError(testutil.ModeSync, storage.TypeManifest, errors.New("manifest sync error")) - } - - // Merging compaction (will fail) - h.compactRangeAtErr(0, "", "", true) - - h.db.Close() - h.stor.EmulateError(testutil.ModeWrite, storage.TypeManifest, nil) - h.stor.EmulateError(testutil.ModeSync, storage.TypeManifest, nil) - - // Should not lose data - h.openDB() - h.getVal("foo", "bar") - }() - } -} - -func assertErr(t *testing.T, err error, wanterr bool) { - if err != nil { - if wanterr { - t.Log("AssertErr: got error (expected): ", err) - } else { - t.Error("AssertErr: got error: ", err) - } - } else if wanterr { - t.Error("AssertErr: expect error") - } -} - -func TestDB_ClosedIsClosed(t *testing.T) { - h := newDbHarness(t) - db := h.db - - var iter, iter2 iterator.Iterator - var snap *Snapshot - func() { - defer h.close() - - h.put("k", "v") - h.getVal("k", "v") - - iter = db.NewIterator(nil, h.ro) - iter.Seek([]byte("k")) - testKeyVal(t, iter, "k->v") - - var err error - snap, err = db.GetSnapshot() - if err != nil { - t.Fatal("GetSnapshot: got error: ", err) - } - - h.getValr(snap, "k", "v") - - iter2 = snap.NewIterator(nil, h.ro) - iter2.Seek([]byte("k")) - testKeyVal(t, iter2, "k->v") - - h.put("foo", "v2") - h.delete("foo") - - // closing DB - iter.Release() - iter2.Release() - }() - - assertErr(t, db.Put([]byte("x"), []byte("y"), h.wo), true) - _, err := db.Get([]byte("k"), h.ro) - assertErr(t, err, true) - - if iter.Valid() { - t.Errorf("iter.Valid should false") - } - assertErr(t, iter.Error(), false) - testKeyVal(t, iter, "->") - if iter.Seek([]byte("k")) { - t.Errorf("iter.Seek should false") - } - assertErr(t, iter.Error(), true) - - assertErr(t, iter2.Error(), false) - - _, err = snap.Get([]byte("k"), h.ro) - assertErr(t, err, true) - - _, err = db.GetSnapshot() - assertErr(t, err, true) - - iter3 := db.NewIterator(nil, h.ro) - assertErr(t, iter3.Error(), true) - - iter3 = snap.NewIterator(nil, h.ro) - assertErr(t, iter3.Error(), true) - - assertErr(t, db.Delete([]byte("k"), h.wo), true) - - _, err = db.GetProperty("leveldb.stats") - assertErr(t, err, true) - - _, err = db.SizeOf([]util.Range{{[]byte("a"), []byte("z")}}) - assertErr(t, err, true) - - assertErr(t, db.CompactRange(util.Range{}), true) - - assertErr(t, db.Close(), true) -} - -type numberComparer struct{} - -func (numberComparer) num(x []byte) (n int) { - fmt.Sscan(string(x[1:len(x)-1]), &n) - return -} - -func (numberComparer) Name() string { - return "test.NumberComparer" -} - -func (p numberComparer) Compare(a, b []byte) int { - return p.num(a) - p.num(b) -} - -func (numberComparer) Separator(dst, a, b []byte) []byte { return nil } -func (numberComparer) Successor(dst, b []byte) []byte { return nil } - -func TestDB_CustomComparer(t *testing.T) { - h := newDbHarnessWopt(t, &opt.Options{ - DisableLargeBatchTransaction: true, - Comparer: numberComparer{}, - WriteBuffer: 1000, - }) - defer h.close() - - h.put("[10]", "ten") - h.put("[0x14]", "twenty") - for i := 0; i < 2; i++ { - h.getVal("[10]", "ten") - h.getVal("[0xa]", "ten") - h.getVal("[20]", "twenty") - h.getVal("[0x14]", "twenty") - h.get("[15]", false) - h.get("[0xf]", false) - h.compactMem() - h.compactRange("[0]", "[9999]") - } - - for n := 0; n < 2; n++ { - for i := 0; i < 100; i++ { - v := fmt.Sprintf("[%d]", i*10) - h.put(v, v) - } - h.compactMem() - h.compactRange("[0]", "[1000000]") - } -} - -func TestDB_ManualCompaction(t *testing.T) { - h := newDbHarness(t) - defer h.close() - - h.db.memdbMaxLevel = 2 - - h.putMulti(3, "p", "q") - h.tablesPerLevel("1,1,1") - - // Compaction range falls before files - h.compactRange("", "c") - h.tablesPerLevel("1,1,1") - - // Compaction range falls after files - h.compactRange("r", "z") - h.tablesPerLevel("1,1,1") - - // Compaction range overlaps files - h.compactRange("p1", "p9") - h.tablesPerLevel("0,0,1") - - // Populate a different range - h.putMulti(3, "c", "e") - h.tablesPerLevel("1,1,2") - - // Compact just the new range - h.compactRange("b", "f") - h.tablesPerLevel("0,0,2") - - // Compact all - h.putMulti(1, "a", "z") - h.tablesPerLevel("0,1,2") - h.compactRange("", "") - h.tablesPerLevel("0,0,1") -} - -func TestDB_BloomFilter(t *testing.T) { - h := newDbHarnessWopt(t, &opt.Options{ - DisableLargeBatchTransaction: true, - DisableBlockCache: true, - Filter: filter.NewBloomFilter(10), - }) - defer h.close() - - key := func(i int) string { - return fmt.Sprintf("key%06d", i) - } - - const n = 10000 - - // Populate multiple layers - for i := 0; i < n; i++ { - h.put(key(i), key(i)) - } - h.compactMem() - h.compactRange("a", "z") - for i := 0; i < n; i += 100 { - h.put(key(i), key(i)) - } - h.compactMem() - - // Prevent auto compactions triggered by seeks - h.stor.Stall(testutil.ModeSync, storage.TypeTable) - - // Lookup present keys. Should rarely read from small sstable. - h.stor.ResetCounter(testutil.ModeRead, storage.TypeTable) - for i := 0; i < n; i++ { - h.getVal(key(i), key(i)) - } - cnt, _ := h.stor.Counter(testutil.ModeRead, storage.TypeTable) - t.Logf("lookup of %d present keys yield %d sstable I/O reads", n, cnt) - if min, max := n, n+2*n/100; cnt < min || cnt > max { - t.Errorf("num of sstable I/O reads of present keys not in range of %d - %d, got %d", min, max, cnt) - } - - // Lookup missing keys. Should rarely read from either sstable. - h.stor.ResetCounter(testutil.ModeRead, storage.TypeTable) - for i := 0; i < n; i++ { - h.get(key(i)+".missing", false) - } - cnt, _ = h.stor.Counter(testutil.ModeRead, storage.TypeTable) - t.Logf("lookup of %d missing keys yield %d sstable I/O reads", n, cnt) - if max := 3 * n / 100; cnt > max { - t.Errorf("num of sstable I/O reads of missing keys was more than %d, got %d", max, cnt) - } - - h.stor.Release(testutil.ModeSync, storage.TypeTable) -} - -func TestDB_Concurrent(t *testing.T) { - const n, secs, maxkey = 4, 6, 1000 - h := newDbHarness(t) - defer h.close() - - runtime.GOMAXPROCS(runtime.NumCPU()) - - var ( - closeWg sync.WaitGroup - stop uint32 - cnt [n]uint32 - ) - - for i := 0; i < n; i++ { - closeWg.Add(1) - go func(i int) { - var put, get, found uint - defer func() { - t.Logf("goroutine %d stopped after %d ops, put=%d get=%d found=%d missing=%d", - i, cnt[i], put, get, found, get-found) - closeWg.Done() - }() - - rnd := rand.New(rand.NewSource(int64(1000 + i))) - for atomic.LoadUint32(&stop) == 0 { - x := cnt[i] - - k := rnd.Intn(maxkey) - kstr := fmt.Sprintf("%016d", k) - - if (rnd.Int() % 2) > 0 { - put++ - h.put(kstr, fmt.Sprintf("%d.%d.%-1000d", k, i, x)) - } else { - get++ - v, err := h.db.Get([]byte(kstr), h.ro) - if err == nil { - found++ - rk, ri, rx := 0, -1, uint32(0) - fmt.Sscanf(string(v), "%d.%d.%d", &rk, &ri, &rx) - if rk != k { - t.Errorf("invalid key want=%d got=%d", k, rk) - } - if ri < 0 || ri >= n { - t.Error("invalid goroutine number: ", ri) - } else { - tx := atomic.LoadUint32(&(cnt[ri])) - if rx > tx { - t.Errorf("invalid seq number, %d > %d ", rx, tx) - } - } - } else if err != ErrNotFound { - t.Error("Get: got error: ", err) - return - } - } - atomic.AddUint32(&cnt[i], 1) - } - }(i) - } - - time.Sleep(secs * time.Second) - atomic.StoreUint32(&stop, 1) - closeWg.Wait() -} - -func TestDB_ConcurrentIterator(t *testing.T) { - const n, n2 = 4, 1000 - h := newDbHarnessWopt(t, &opt.Options{DisableLargeBatchTransaction: true, WriteBuffer: 30}) - defer h.close() - - runtime.GOMAXPROCS(runtime.NumCPU()) - - var ( - closeWg sync.WaitGroup - stop uint32 - ) - - for i := 0; i < n; i++ { - closeWg.Add(1) - go func(i int) { - for k := 0; atomic.LoadUint32(&stop) == 0; k++ { - h.put(fmt.Sprintf("k%d", k), fmt.Sprintf("%d.%d.", k, i)+strings.Repeat("x", 10)) - } - closeWg.Done() - }(i) - } - - for i := 0; i < n; i++ { - closeWg.Add(1) - go func(i int) { - for k := 1000000; k < 0 || atomic.LoadUint32(&stop) == 0; k-- { - h.put(fmt.Sprintf("k%d", k), fmt.Sprintf("%d.%d.", k, i)+strings.Repeat("x", 10)) - } - closeWg.Done() - }(i) - } - - cmp := comparer.DefaultComparer - for i := 0; i < n2; i++ { - closeWg.Add(1) - go func(i int) { - it := h.db.NewIterator(nil, nil) - var pk []byte - for it.Next() { - kk := it.Key() - if cmp.Compare(kk, pk) <= 0 { - t.Errorf("iter %d: %q is successor of %q", i, pk, kk) - } - pk = append(pk[:0], kk...) - var k, vk, vi int - if n, err := fmt.Sscanf(string(it.Key()), "k%d", &k); err != nil { - t.Errorf("iter %d: Scanf error on key %q: %v", i, it.Key(), err) - } else if n < 1 { - t.Errorf("iter %d: Cannot parse key %q", i, it.Key()) - } - if n, err := fmt.Sscanf(string(it.Value()), "%d.%d", &vk, &vi); err != nil { - t.Errorf("iter %d: Scanf error on value %q: %v", i, it.Value(), err) - } else if n < 2 { - t.Errorf("iter %d: Cannot parse value %q", i, it.Value()) - } - - if vk != k { - t.Errorf("iter %d: invalid value i=%d, want=%d got=%d", i, vi, k, vk) - } - } - if err := it.Error(); err != nil { - t.Errorf("iter %d: Got error: %v", i, err) - } - it.Release() - closeWg.Done() - }(i) - } - - atomic.StoreUint32(&stop, 1) - closeWg.Wait() -} - -func TestDB_ConcurrentWrite(t *testing.T) { - const n, bk, niter = 10, 3, 10000 - h := newDbHarness(t) - defer h.close() - - runtime.GOMAXPROCS(runtime.NumCPU()) - - var wg sync.WaitGroup - for i := 0; i < n; i++ { - wg.Add(1) - go func(i int) { - defer wg.Done() - for k := 0; k < niter; k++ { - kstr := fmt.Sprintf("put-%d.%d", i, k) - vstr := fmt.Sprintf("v%d", k) - h.put(kstr, vstr) - // Key should immediately available after put returns. - h.getVal(kstr, vstr) - } - }(i) - } - for i := 0; i < n; i++ { - wg.Add(1) - batch := &Batch{} - go func(i int) { - defer wg.Done() - for k := 0; k < niter; k++ { - batch.Reset() - for j := 0; j < bk; j++ { - batch.Put([]byte(fmt.Sprintf("batch-%d.%d.%d", i, k, j)), []byte(fmt.Sprintf("v%d", k))) - } - h.write(batch) - // Key should immediately available after put returns. - for j := 0; j < bk; j++ { - h.getVal(fmt.Sprintf("batch-%d.%d.%d", i, k, j), fmt.Sprintf("v%d", k)) - } - } - }(i) - } - wg.Wait() -} - -func TestDB_CreateReopenDbOnFile(t *testing.T) { - dbpath := filepath.Join(os.TempDir(), fmt.Sprintf("goleveldbtestCreateReopenDbOnFile-%d", os.Getuid())) - if err := os.RemoveAll(dbpath); err != nil { - t.Fatal("cannot remove old db: ", err) - } - defer os.RemoveAll(dbpath) - - for i := 0; i < 3; i++ { - stor, err := storage.OpenFile(dbpath, false) - if err != nil { - t.Fatalf("(%d) cannot open storage: %s", i, err) - } - db, err := Open(stor, nil) - if err != nil { - t.Fatalf("(%d) cannot open db: %s", i, err) - } - if err := db.Put([]byte("foo"), []byte("bar"), nil); err != nil { - t.Fatalf("(%d) cannot write to db: %s", i, err) - } - if err := db.Close(); err != nil { - t.Fatalf("(%d) cannot close db: %s", i, err) - } - if err := stor.Close(); err != nil { - t.Fatalf("(%d) cannot close storage: %s", i, err) - } - } -} - -func TestDB_CreateReopenDbOnFile2(t *testing.T) { - dbpath := filepath.Join(os.TempDir(), fmt.Sprintf("goleveldbtestCreateReopenDbOnFile2-%d", os.Getuid())) - if err := os.RemoveAll(dbpath); err != nil { - t.Fatal("cannot remove old db: ", err) - } - defer os.RemoveAll(dbpath) - - for i := 0; i < 3; i++ { - db, err := OpenFile(dbpath, nil) - if err != nil { - t.Fatalf("(%d) cannot open db: %s", i, err) - } - if err := db.Put([]byte("foo"), []byte("bar"), nil); err != nil { - t.Fatalf("(%d) cannot write to db: %s", i, err) - } - if err := db.Close(); err != nil { - t.Fatalf("(%d) cannot close db: %s", i, err) - } - } -} - -func TestDB_DeletionMarkersOnMemdb(t *testing.T) { - h := newDbHarness(t) - defer h.close() - - h.put("foo", "v1") - h.compactMem() - h.delete("foo") - h.get("foo", false) - h.getKeyVal("") -} - -func TestDB_LeveldbIssue178(t *testing.T) { - nKeys := (opt.DefaultCompactionTableSize / 30) * 5 - key1 := func(i int) string { - return fmt.Sprintf("my_key_%d", i) - } - key2 := func(i int) string { - return fmt.Sprintf("my_key_%d_xxx", i) - } - - // Disable compression since it affects the creation of layers and the - // code below is trying to test against a very specific scenario. - h := newDbHarnessWopt(t, &opt.Options{ - DisableLargeBatchTransaction: true, - Compression: opt.NoCompression, - }) - defer h.close() - - // Create first key range. - batch := new(Batch) - for i := 0; i < nKeys; i++ { - batch.Put([]byte(key1(i)), []byte("value for range 1 key")) - } - h.write(batch) - - // Create second key range. - batch.Reset() - for i := 0; i < nKeys; i++ { - batch.Put([]byte(key2(i)), []byte("value for range 2 key")) - } - h.write(batch) - - // Delete second key range. - batch.Reset() - for i := 0; i < nKeys; i++ { - batch.Delete([]byte(key2(i))) - } - h.write(batch) - h.waitMemCompaction() - - // Run manual compaction. - h.compactRange(key1(0), key1(nKeys-1)) - - // Checking the keys. - h.assertNumKeys(nKeys) -} - -func TestDB_LeveldbIssue200(t *testing.T) { - h := newDbHarness(t) - defer h.close() - - h.put("1", "b") - h.put("2", "c") - h.put("3", "d") - h.put("4", "e") - h.put("5", "f") - - iter := h.db.NewIterator(nil, h.ro) - - // Add an element that should not be reflected in the iterator. - h.put("25", "cd") - - iter.Seek([]byte("5")) - assertBytes(t, []byte("5"), iter.Key()) - iter.Prev() - assertBytes(t, []byte("4"), iter.Key()) - iter.Prev() - assertBytes(t, []byte("3"), iter.Key()) - iter.Next() - assertBytes(t, []byte("4"), iter.Key()) - iter.Next() - assertBytes(t, []byte("5"), iter.Key()) -} - -func TestDB_GoleveldbIssue74(t *testing.T) { - h := newDbHarnessWopt(t, &opt.Options{ - DisableLargeBatchTransaction: true, - WriteBuffer: 1 * opt.MiB, - }) - defer h.close() - - const n, dur = 10000, 5 * time.Second - - runtime.GOMAXPROCS(runtime.NumCPU()) - - until := time.Now().Add(dur) - wg := new(sync.WaitGroup) - wg.Add(2) - var done uint32 - go func() { - var i int - defer func() { - t.Logf("WRITER DONE #%d", i) - atomic.StoreUint32(&done, 1) - wg.Done() - }() - - b := new(Batch) - for ; time.Now().Before(until) && atomic.LoadUint32(&done) == 0; i++ { - iv := fmt.Sprintf("VAL%010d", i) - for k := 0; k < n; k++ { - key := fmt.Sprintf("KEY%06d", k) - b.Put([]byte(key), []byte(key+iv)) - b.Put([]byte(fmt.Sprintf("PTR%06d", k)), []byte(key)) - } - h.write(b) - - b.Reset() - snap := h.getSnapshot() - iter := snap.NewIterator(util.BytesPrefix([]byte("PTR")), nil) - var k int - for ; iter.Next(); k++ { - ptrKey := iter.Key() - key := iter.Value() - - if _, err := snap.Get(ptrKey, nil); err != nil { - t.Fatalf("WRITER #%d snapshot.Get %q: %v", i, ptrKey, err) - } - if value, err := snap.Get(key, nil); err != nil { - t.Fatalf("WRITER #%d snapshot.Get %q: %v", i, key, err) - } else if string(value) != string(key)+iv { - t.Fatalf("WRITER #%d snapshot.Get %q got invalid value, want %q got %q", i, key, string(key)+iv, value) - } - - b.Delete(key) - b.Delete(ptrKey) - } - h.write(b) - iter.Release() - snap.Release() - if k != n { - t.Fatalf("#%d %d != %d", i, k, n) - } - } - }() - go func() { - var i int - defer func() { - t.Logf("READER DONE #%d", i) - atomic.StoreUint32(&done, 1) - wg.Done() - }() - for ; time.Now().Before(until) && atomic.LoadUint32(&done) == 0; i++ { - snap := h.getSnapshot() - iter := snap.NewIterator(util.BytesPrefix([]byte("PTR")), nil) - var prevValue string - var k int - for ; iter.Next(); k++ { - ptrKey := iter.Key() - key := iter.Value() - - if _, err := snap.Get(ptrKey, nil); err != nil { - t.Fatalf("READER #%d snapshot.Get %q: %v", i, ptrKey, err) - } - - if value, err := snap.Get(key, nil); err != nil { - t.Fatalf("READER #%d snapshot.Get %q: %v", i, key, err) - } else if prevValue != "" && string(value) != string(key)+prevValue { - t.Fatalf("READER #%d snapshot.Get %q got invalid value, want %q got %q", i, key, string(key)+prevValue, value) - } else { - prevValue = string(value[len(key):]) - } - } - iter.Release() - snap.Release() - if k > 0 && k != n { - t.Fatalf("#%d %d != %d", i, k, n) - } - } - }() - wg.Wait() -} - -func TestDB_GetProperties(t *testing.T) { - h := newDbHarness(t) - defer h.close() - - _, err := h.db.GetProperty("leveldb.num-files-at-level") - if err == nil { - t.Error("GetProperty() failed to detect missing level") - } - - _, err = h.db.GetProperty("leveldb.num-files-at-level0") - if err != nil { - t.Error("got unexpected error", err) - } - - _, err = h.db.GetProperty("leveldb.num-files-at-level0x") - if err == nil { - t.Error("GetProperty() failed to detect invalid level") - } -} - -func TestDB_GoleveldbIssue72and83(t *testing.T) { - h := newDbHarnessWopt(t, &opt.Options{ - DisableLargeBatchTransaction: true, - WriteBuffer: 1 * opt.MiB, - OpenFilesCacheCapacity: 3, - }) - defer h.close() - - const n, wn, dur = 10000, 100, 30 * time.Second - - runtime.GOMAXPROCS(runtime.NumCPU()) - - randomData := func(prefix byte, i int) []byte { - data := make([]byte, 1+4+32+64+32) - _, err := crand.Reader.Read(data[1 : len(data)-8]) - if err != nil { - panic(err) - } - data[0] = prefix - binary.LittleEndian.PutUint32(data[len(data)-8:], uint32(i)) - binary.LittleEndian.PutUint32(data[len(data)-4:], util.NewCRC(data[:len(data)-4]).Value()) - return data - } - - keys := make([][]byte, n) - for i := range keys { - keys[i] = randomData(1, 0) - } - - until := time.Now().Add(dur) - wg := new(sync.WaitGroup) - wg.Add(3) - var done uint32 - go func() { - i := 0 - defer func() { - t.Logf("WRITER DONE #%d", i) - wg.Done() - }() - - b := new(Batch) - for ; i < wn && atomic.LoadUint32(&done) == 0; i++ { - b.Reset() - for _, k1 := range keys { - k2 := randomData(2, i) - b.Put(k2, randomData(42, i)) - b.Put(k1, k2) - } - if err := h.db.Write(b, h.wo); err != nil { - atomic.StoreUint32(&done, 1) - t.Fatalf("WRITER #%d db.Write: %v", i, err) - } - } - }() - go func() { - var i int - defer func() { - t.Logf("READER0 DONE #%d", i) - atomic.StoreUint32(&done, 1) - wg.Done() - }() - for ; time.Now().Before(until) && atomic.LoadUint32(&done) == 0; i++ { - snap := h.getSnapshot() - seq := snap.elem.seq - if seq == 0 { - snap.Release() - continue - } - iter := snap.NewIterator(util.BytesPrefix([]byte{1}), nil) - writei := int(seq/(n*2) - 1) - var k int - for ; iter.Next(); k++ { - k1 := iter.Key() - k2 := iter.Value() - k1checksum0 := binary.LittleEndian.Uint32(k1[len(k1)-4:]) - k1checksum1 := util.NewCRC(k1[:len(k1)-4]).Value() - if k1checksum0 != k1checksum1 { - t.Fatalf("READER0 #%d.%d W#%d invalid K1 checksum: %#x != %#x", i, k, k1checksum0, k1checksum0) - } - k2checksum0 := binary.LittleEndian.Uint32(k2[len(k2)-4:]) - k2checksum1 := util.NewCRC(k2[:len(k2)-4]).Value() - if k2checksum0 != k2checksum1 { - t.Fatalf("READER0 #%d.%d W#%d invalid K2 checksum: %#x != %#x", i, k, k2checksum0, k2checksum1) - } - kwritei := int(binary.LittleEndian.Uint32(k2[len(k2)-8:])) - if writei != kwritei { - t.Fatalf("READER0 #%d.%d W#%d invalid write iteration num: %d", i, k, writei, kwritei) - } - if _, err := snap.Get(k2, nil); err != nil { - t.Fatalf("READER0 #%d.%d W#%d snap.Get: %v\nk1: %x\n -> k2: %x", i, k, writei, err, k1, k2) - } - } - if err := iter.Error(); err != nil { - t.Fatalf("READER0 #%d.%d W#%d snap.Iterator: %v", i, k, writei, err) - } - iter.Release() - snap.Release() - if k > 0 && k != n { - t.Fatalf("READER0 #%d W#%d short read, got=%d want=%d", i, writei, k, n) - } - } - }() - go func() { - var i int - defer func() { - t.Logf("READER1 DONE #%d", i) - atomic.StoreUint32(&done, 1) - wg.Done() - }() - for ; time.Now().Before(until) && atomic.LoadUint32(&done) == 0; i++ { - iter := h.db.NewIterator(nil, nil) - seq := iter.(*dbIter).seq - if seq == 0 { - iter.Release() - continue - } - writei := int(seq/(n*2) - 1) - var k int - for ok := iter.Last(); ok; ok = iter.Prev() { - k++ - } - if err := iter.Error(); err != nil { - t.Fatalf("READER1 #%d.%d W#%d db.Iterator: %v", i, k, writei, err) - } - iter.Release() - if m := (writei+1)*n + n; k != m { - t.Fatalf("READER1 #%d W#%d short read, got=%d want=%d", i, writei, k, m) - } - } - }() - - wg.Wait() -} - -func TestDB_TransientError(t *testing.T) { - h := newDbHarnessWopt(t, &opt.Options{ - DisableLargeBatchTransaction: true, - WriteBuffer: 128 * opt.KiB, - OpenFilesCacheCapacity: 3, - DisableCompactionBackoff: true, - }) - defer h.close() - - const ( - nSnap = 20 - nKey = 10000 - ) - - var ( - snaps [nSnap]*Snapshot - b = &Batch{} - ) - for i := range snaps { - vtail := fmt.Sprintf("VAL%030d", i) - b.Reset() - for k := 0; k < nKey; k++ { - key := fmt.Sprintf("KEY%8d", k) - b.Put([]byte(key), []byte(key+vtail)) - } - h.stor.EmulateError(testutil.ModeOpen|testutil.ModeRead, storage.TypeTable, errors.New("table transient read error")) - if err := h.db.Write(b, nil); err != nil { - t.Logf("WRITE #%d error: %v", i, err) - h.stor.EmulateError(testutil.ModeOpen|testutil.ModeRead, storage.TypeTable, nil) - for { - if err := h.db.Write(b, nil); err == nil { - break - } else if errors.IsCorrupted(err) { - t.Fatalf("WRITE #%d corrupted: %v", i, err) - } - } - } - - snaps[i] = h.db.newSnapshot() - b.Reset() - for k := 0; k < nKey; k++ { - key := fmt.Sprintf("KEY%8d", k) - b.Delete([]byte(key)) - } - h.stor.EmulateError(testutil.ModeOpen|testutil.ModeRead, storage.TypeTable, errors.New("table transient read error")) - if err := h.db.Write(b, nil); err != nil { - t.Logf("WRITE #%d error: %v", i, err) - h.stor.EmulateError(testutil.ModeOpen|testutil.ModeRead, storage.TypeTable, nil) - for { - if err := h.db.Write(b, nil); err == nil { - break - } else if errors.IsCorrupted(err) { - t.Fatalf("WRITE #%d corrupted: %v", i, err) - } - } - } - } - h.stor.EmulateError(testutil.ModeOpen|testutil.ModeRead, storage.TypeTable, nil) - - runtime.GOMAXPROCS(runtime.NumCPU()) - - rnd := rand.New(rand.NewSource(0xecafdaed)) - wg := &sync.WaitGroup{} - for i, snap := range snaps { - wg.Add(2) - - go func(i int, snap *Snapshot, sk []int) { - defer wg.Done() - - vtail := fmt.Sprintf("VAL%030d", i) - for _, k := range sk { - key := fmt.Sprintf("KEY%8d", k) - xvalue, err := snap.Get([]byte(key), nil) - if err != nil { - t.Fatalf("READER_GET #%d SEQ=%d K%d error: %v", i, snap.elem.seq, k, err) - } - value := key + vtail - if !bytes.Equal([]byte(value), xvalue) { - t.Fatalf("READER_GET #%d SEQ=%d K%d invalid value: want %q, got %q", i, snap.elem.seq, k, value, xvalue) - } - } - }(i, snap, rnd.Perm(nKey)) - - go func(i int, snap *Snapshot) { - defer wg.Done() - - vtail := fmt.Sprintf("VAL%030d", i) - iter := snap.NewIterator(nil, nil) - defer iter.Release() - for k := 0; k < nKey; k++ { - if !iter.Next() { - if err := iter.Error(); err != nil { - t.Fatalf("READER_ITER #%d K%d error: %v", i, k, err) - } else { - t.Fatalf("READER_ITER #%d K%d eoi", i, k) - } - } - key := fmt.Sprintf("KEY%8d", k) - xkey := iter.Key() - if !bytes.Equal([]byte(key), xkey) { - t.Fatalf("READER_ITER #%d K%d invalid key: want %q, got %q", i, k, key, xkey) - } - value := key + vtail - xvalue := iter.Value() - if !bytes.Equal([]byte(value), xvalue) { - t.Fatalf("READER_ITER #%d K%d invalid value: want %q, got %q", i, k, value, xvalue) - } - } - }(i, snap) - } - - wg.Wait() -} - -func TestDB_UkeyShouldntHopAcrossTable(t *testing.T) { - h := newDbHarnessWopt(t, &opt.Options{ - DisableLargeBatchTransaction: true, - WriteBuffer: 112 * opt.KiB, - CompactionTableSize: 90 * opt.KiB, - CompactionExpandLimitFactor: 1, - }) - defer h.close() - - const ( - nSnap = 190 - nKey = 140 - ) - - var ( - snaps [nSnap]*Snapshot - b = &Batch{} - ) - for i := range snaps { - vtail := fmt.Sprintf("VAL%030d", i) - b.Reset() - for k := 0; k < nKey; k++ { - key := fmt.Sprintf("KEY%08d", k) - b.Put([]byte(key), []byte(key+vtail)) - } - if err := h.db.Write(b, nil); err != nil { - t.Fatalf("WRITE #%d error: %v", i, err) - } - - snaps[i] = h.db.newSnapshot() - b.Reset() - for k := 0; k < nKey; k++ { - key := fmt.Sprintf("KEY%08d", k) - b.Delete([]byte(key)) - } - if err := h.db.Write(b, nil); err != nil { - t.Fatalf("WRITE #%d error: %v", i, err) - } - } - - h.compactMem() - - h.waitCompaction() - for level, tables := range h.db.s.stVersion.levels { - for _, table := range tables { - t.Logf("L%d@%d %q:%q", level, table.fd.Num, table.imin, table.imax) - } - } - - h.compactRangeAt(0, "", "") - h.waitCompaction() - for level, tables := range h.db.s.stVersion.levels { - for _, table := range tables { - t.Logf("L%d@%d %q:%q", level, table.fd.Num, table.imin, table.imax) - } - } - h.compactRangeAt(1, "", "") - h.waitCompaction() - for level, tables := range h.db.s.stVersion.levels { - for _, table := range tables { - t.Logf("L%d@%d %q:%q", level, table.fd.Num, table.imin, table.imax) - } - } - runtime.GOMAXPROCS(runtime.NumCPU()) - - wg := &sync.WaitGroup{} - for i, snap := range snaps { - wg.Add(1) - - go func(i int, snap *Snapshot) { - defer wg.Done() - - vtail := fmt.Sprintf("VAL%030d", i) - for k := 0; k < nKey; k++ { - key := fmt.Sprintf("KEY%08d", k) - xvalue, err := snap.Get([]byte(key), nil) - if err != nil { - t.Fatalf("READER_GET #%d SEQ=%d K%d error: %v", i, snap.elem.seq, k, err) - } - value := key + vtail - if !bytes.Equal([]byte(value), xvalue) { - t.Fatalf("READER_GET #%d SEQ=%d K%d invalid value: want %q, got %q", i, snap.elem.seq, k, value, xvalue) - } - } - }(i, snap) - } - - wg.Wait() -} - -func TestDB_TableCompactionBuilder(t *testing.T) { - gomega.RegisterTestingT(t) - stor := testutil.NewStorage() - stor.OnLog(testingLogger(t)) - stor.OnClose(testingPreserveOnFailed(t)) - defer stor.Close() - - const nSeq = 99 - - o := &opt.Options{ - DisableLargeBatchTransaction: true, - WriteBuffer: 112 * opt.KiB, - CompactionTableSize: 43 * opt.KiB, - CompactionExpandLimitFactor: 1, - CompactionGPOverlapsFactor: 1, - DisableBlockCache: true, - } - s, err := newSession(stor, o) - if err != nil { - t.Fatal(err) - } - if err := s.create(); err != nil { - t.Fatal(err) - } - defer s.close() - var ( - seq uint64 - targetSize = 5 * o.CompactionTableSize - value = bytes.Repeat([]byte{'0'}, 100) - ) - for i := 0; i < 2; i++ { - tw, err := s.tops.create() - if err != nil { - t.Fatal(err) - } - for k := 0; tw.tw.BytesLen() < targetSize; k++ { - key := []byte(fmt.Sprintf("%09d", k)) - seq += nSeq - 1 - for x := uint64(0); x < nSeq; x++ { - if err := tw.append(makeInternalKey(nil, key, seq-x, keyTypeVal), value); err != nil { - t.Fatal(err) - } - } - } - tf, err := tw.finish() - if err != nil { - t.Fatal(err) - } - rec := &sessionRecord{} - rec.addTableFile(i, tf) - if err := s.commit(rec); err != nil { - t.Fatal(err) - } - } - - // Build grandparent. - v := s.version() - c := newCompaction(s, v, 1, append(tFiles{}, v.levels[1]...)) - rec := &sessionRecord{} - b := &tableCompactionBuilder{ - s: s, - c: c, - rec: rec, - stat1: new(cStatStaging), - minSeq: 0, - strict: true, - tableSize: o.CompactionTableSize/3 + 961, - } - if err := b.run(new(compactionTransactCounter)); err != nil { - t.Fatal(err) - } - for _, t := range c.levels[0] { - rec.delTable(c.sourceLevel, t.fd.Num) - } - if err := s.commit(rec); err != nil { - t.Fatal(err) - } - c.release() - - // Build level-1. - v = s.version() - c = newCompaction(s, v, 0, append(tFiles{}, v.levels[0]...)) - rec = &sessionRecord{} - b = &tableCompactionBuilder{ - s: s, - c: c, - rec: rec, - stat1: new(cStatStaging), - minSeq: 0, - strict: true, - tableSize: o.CompactionTableSize, - } - if err := b.run(new(compactionTransactCounter)); err != nil { - t.Fatal(err) - } - for _, t := range c.levels[0] { - rec.delTable(c.sourceLevel, t.fd.Num) - } - // Move grandparent to level-3 - for _, t := range v.levels[2] { - rec.delTable(2, t.fd.Num) - rec.addTableFile(3, t) - } - if err := s.commit(rec); err != nil { - t.Fatal(err) - } - c.release() - - v = s.version() - for level, want := range []bool{false, true, false, true} { - got := len(v.levels[level]) > 0 - if want != got { - t.Fatalf("invalid level-%d tables len: want %v, got %v", level, want, got) - } - } - for i, f := range v.levels[1][:len(v.levels[1])-1] { - nf := v.levels[1][i+1] - if bytes.Equal(f.imax.ukey(), nf.imin.ukey()) { - t.Fatalf("KEY %q hop across table %d .. %d", f.imax.ukey(), f.fd.Num, nf.fd.Num) - } - } - v.release() - - // Compaction with transient error. - v = s.version() - c = newCompaction(s, v, 1, append(tFiles{}, v.levels[1]...)) - rec = &sessionRecord{} - b = &tableCompactionBuilder{ - s: s, - c: c, - rec: rec, - stat1: new(cStatStaging), - minSeq: 0, - strict: true, - tableSize: o.CompactionTableSize, - } - stor.EmulateErrorOnce(testutil.ModeSync, storage.TypeTable, errors.New("table sync error (once)")) - stor.EmulateRandomError(testutil.ModeRead|testutil.ModeWrite, storage.TypeTable, 0.01, errors.New("table random IO error")) - for { - if err := b.run(new(compactionTransactCounter)); err != nil { - t.Logf("(expected) b.run: %v", err) - } else { - break - } - } - if err := s.commit(rec); err != nil { - t.Fatal(err) - } - c.release() - - stor.EmulateErrorOnce(testutil.ModeSync, storage.TypeTable, nil) - stor.EmulateRandomError(testutil.ModeRead|testutil.ModeWrite, storage.TypeTable, 0, nil) - - v = s.version() - if len(v.levels[1]) != len(v.levels[2]) { - t.Fatalf("invalid tables length, want %d, got %d", len(v.levels[1]), len(v.levels[2])) - } - for i, f0 := range v.levels[1] { - f1 := v.levels[2][i] - iter0 := s.tops.newIterator(f0, nil, nil) - iter1 := s.tops.newIterator(f1, nil, nil) - for j := 0; true; j++ { - next0 := iter0.Next() - next1 := iter1.Next() - if next0 != next1 { - t.Fatalf("#%d.%d invalid eoi: want %v, got %v", i, j, next0, next1) - } - key0 := iter0.Key() - key1 := iter1.Key() - if !bytes.Equal(key0, key1) { - t.Fatalf("#%d.%d invalid key: want %q, got %q", i, j, key0, key1) - } - if next0 == false { - break - } - } - iter0.Release() - iter1.Release() - } - v.release() -} - -func testDB_IterTriggeredCompaction(t *testing.T, limitDiv int) { - const ( - vSize = 200 * opt.KiB - tSize = 100 * opt.MiB - mIter = 100 - n = tSize / vSize - ) - - h := newDbHarnessWopt(t, &opt.Options{ - DisableLargeBatchTransaction: true, - Compression: opt.NoCompression, - DisableBlockCache: true, - }) - defer h.close() - - h.db.memdbMaxLevel = 2 - - key := func(x int) string { - return fmt.Sprintf("v%06d", x) - } - - // Fill. - value := strings.Repeat("x", vSize) - for i := 0; i < n; i++ { - h.put(key(i), value) - } - h.compactMem() - - // Delete all. - for i := 0; i < n; i++ { - h.delete(key(i)) - } - h.compactMem() - - var ( - limit = n / limitDiv - - startKey = key(0) - limitKey = key(limit) - maxKey = key(n) - slice = &util.Range{Limit: []byte(limitKey)} - - initialSize0 = h.sizeOf(startKey, limitKey) - initialSize1 = h.sizeOf(limitKey, maxKey) - ) - - t.Logf("inital size %s [rest %s]", shortenb(int(initialSize0)), shortenb(int(initialSize1))) - - for r := 0; true; r++ { - if r >= mIter { - t.Fatal("taking too long to compact") - } - - // Iterates. - iter := h.db.NewIterator(slice, h.ro) - for iter.Next() { - } - if err := iter.Error(); err != nil { - t.Fatalf("Iter err: %v", err) - } - iter.Release() - - // Wait compaction. - h.waitCompaction() - - // Check size. - size0 := h.sizeOf(startKey, limitKey) - size1 := h.sizeOf(limitKey, maxKey) - t.Logf("#%03d size %s [rest %s]", r, shortenb(int(size0)), shortenb(int(size1))) - if size0 < initialSize0/10 { - break - } - } - - if initialSize1 > 0 { - h.sizeAssert(limitKey, maxKey, initialSize1/4-opt.MiB, initialSize1+opt.MiB) - } -} - -func TestDB_IterTriggeredCompaction(t *testing.T) { - testDB_IterTriggeredCompaction(t, 1) -} - -func TestDB_IterTriggeredCompactionHalf(t *testing.T) { - testDB_IterTriggeredCompaction(t, 2) -} - -func TestDB_ReadOnly(t *testing.T) { - h := newDbHarness(t) - defer h.close() - - h.put("foo", "v1") - h.put("bar", "v2") - h.compactMem() - - h.put("xfoo", "v1") - h.put("xbar", "v2") - - t.Log("Trigger read-only") - if err := h.db.SetReadOnly(); err != nil { - h.close() - t.Fatalf("SetReadOnly error: %v", err) - } - - mode := testutil.ModeCreate | testutil.ModeRemove | testutil.ModeRename | testutil.ModeWrite | testutil.ModeSync - h.stor.EmulateError(mode, storage.TypeAll, errors.New("read-only DB shouldn't writes")) - - ro := func(key, value, wantValue string) { - if err := h.db.Put([]byte(key), []byte(value), h.wo); err != ErrReadOnly { - t.Fatalf("unexpected error: %v", err) - } - h.getVal(key, wantValue) - } - - ro("foo", "vx", "v1") - - h.o.ReadOnly = true - h.reopenDB() - - ro("foo", "vx", "v1") - ro("bar", "vx", "v2") - h.assertNumKeys(4) -} - -func TestDB_BulkInsertDelete(t *testing.T) { - h := newDbHarnessWopt(t, &opt.Options{ - DisableLargeBatchTransaction: true, - Compression: opt.NoCompression, - CompactionTableSize: 128 * opt.KiB, - CompactionTotalSize: 1 * opt.MiB, - WriteBuffer: 256 * opt.KiB, - }) - defer h.close() - - const R = 100 - const N = 2500 - key := make([]byte, 4) - value := make([]byte, 256) - for i := 0; i < R; i++ { - offset := N * i - for j := 0; j < N; j++ { - binary.BigEndian.PutUint32(key, uint32(offset+j)) - h.db.Put(key, value, nil) - } - for j := 0; j < N; j++ { - binary.BigEndian.PutUint32(key, uint32(offset+j)) - h.db.Delete(key, nil) - } - } - - if tot := h.totalTables(); tot > 10 { - t.Fatalf("too many uncompacted tables: %d (%s)", tot, h.getTablesPerLevel()) - } -} - -func TestDB_GracefulClose(t *testing.T) { - runtime.GOMAXPROCS(4) - h := newDbHarnessWopt(t, &opt.Options{ - DisableLargeBatchTransaction: true, - Compression: opt.NoCompression, - CompactionTableSize: 1 * opt.MiB, - WriteBuffer: 1 * opt.MiB, - }) - defer h.close() - - var closeWait sync.WaitGroup - - // During write. - n := 0 - closing := false - for i := 0; i < 1000000; i++ { - if !closing && h.totalTables() > 3 { - t.Logf("close db during write, index=%d", i) - closeWait.Add(1) - go func() { - h.closeDB() - closeWait.Done() - }() - closing = true - } - if err := h.db.Put([]byte(fmt.Sprintf("%09d", i)), []byte(fmt.Sprintf("VAL-%09d", i)), h.wo); err != nil { - t.Logf("Put error: %s (expected)", err) - n = i - break - } - } - closeWait.Wait() - - // During read. - h.openDB() - closing = false - for i := 0; i < n; i++ { - if !closing && i > n/2 { - t.Logf("close db during read, index=%d", i) - closeWait.Add(1) - go func() { - h.closeDB() - closeWait.Done() - }() - closing = true - } - if _, err := h.db.Get([]byte(fmt.Sprintf("%09d", i)), h.ro); err != nil { - t.Logf("Get error: %s (expected)", err) - break - } - } - closeWait.Wait() - - // During iterate. - h.openDB() - closing = false - iter := h.db.NewIterator(nil, h.ro) - for i := 0; iter.Next(); i++ { - if len(iter.Key()) == 0 || len(iter.Value()) == 0 { - t.Error("Key or value has zero length") - } - if !closing { - t.Logf("close db during iter, index=%d", i) - closeWait.Add(1) - go func() { - h.closeDB() - closeWait.Done() - }() - closing = true - } - time.Sleep(time.Millisecond) - } - if err := iter.Error(); err != nil { - t.Logf("Iter error: %s (expected)", err) - } - iter.Release() - closeWait.Wait() -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/db_transaction.go b/vendor/github.com/syndtr/goleveldb/leveldb/db_transaction.go deleted file mode 100644 index b8f7e7d..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/db_transaction.go +++ /dev/null @@ -1,325 +0,0 @@ -// Copyright (c) 2016, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package leveldb - -import ( - "errors" - "sync" - "time" - - "github.com/syndtr/goleveldb/leveldb/iterator" - "github.com/syndtr/goleveldb/leveldb/opt" - "github.com/syndtr/goleveldb/leveldb/util" -) - -var errTransactionDone = errors.New("leveldb: transaction already closed") - -// Transaction is the transaction handle. -type Transaction struct { - db *DB - lk sync.RWMutex - seq uint64 - mem *memDB - tables tFiles - ikScratch []byte - rec sessionRecord - stats cStatStaging - closed bool -} - -// Get gets the value for the given key. It returns ErrNotFound if the -// DB does not contains the key. -// -// The returned slice is its own copy, it is safe to modify the contents -// of the returned slice. -// It is safe to modify the contents of the argument after Get returns. -func (tr *Transaction) Get(key []byte, ro *opt.ReadOptions) ([]byte, error) { - tr.lk.RLock() - defer tr.lk.RUnlock() - if tr.closed { - return nil, errTransactionDone - } - return tr.db.get(tr.mem.DB, tr.tables, key, tr.seq, ro) -} - -// Has returns true if the DB does contains the given key. -// -// It is safe to modify the contents of the argument after Has returns. -func (tr *Transaction) Has(key []byte, ro *opt.ReadOptions) (bool, error) { - tr.lk.RLock() - defer tr.lk.RUnlock() - if tr.closed { - return false, errTransactionDone - } - return tr.db.has(tr.mem.DB, tr.tables, key, tr.seq, ro) -} - -// NewIterator returns an iterator for the latest snapshot of the transaction. -// The returned iterator is not safe for concurrent use, but it is safe to use -// multiple iterators concurrently, with each in a dedicated goroutine. -// It is also safe to use an iterator concurrently while writes to the -// transaction. The resultant key/value pairs are guaranteed to be consistent. -// -// Slice allows slicing the iterator to only contains keys in the given -// range. A nil Range.Start is treated as a key before all keys in the -// DB. And a nil Range.Limit is treated as a key after all keys in -// the DB. -// -// The iterator must be released after use, by calling Release method. -// -// Also read Iterator documentation of the leveldb/iterator package. -func (tr *Transaction) NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator { - tr.lk.RLock() - defer tr.lk.RUnlock() - if tr.closed { - return iterator.NewEmptyIterator(errTransactionDone) - } - tr.mem.incref() - return tr.db.newIterator(tr.mem, tr.tables, tr.seq, slice, ro) -} - -func (tr *Transaction) flush() error { - // Flush memdb. - if tr.mem.Len() != 0 { - tr.stats.startTimer() - iter := tr.mem.NewIterator(nil) - t, n, err := tr.db.s.tops.createFrom(iter) - iter.Release() - tr.stats.stopTimer() - if err != nil { - return err - } - if tr.mem.getref() == 1 { - tr.mem.Reset() - } else { - tr.mem.decref() - tr.mem = tr.db.mpoolGet(0) - tr.mem.incref() - } - tr.tables = append(tr.tables, t) - tr.rec.addTableFile(0, t) - tr.stats.write += t.size - tr.db.logf("transaction@flush created L0@%d N·%d S·%s %q:%q", t.fd.Num, n, shortenb(int(t.size)), t.imin, t.imax) - } - return nil -} - -func (tr *Transaction) put(kt keyType, key, value []byte) error { - tr.ikScratch = makeInternalKey(tr.ikScratch, key, tr.seq+1, kt) - if tr.mem.Free() < len(tr.ikScratch)+len(value) { - if err := tr.flush(); err != nil { - return err - } - } - if err := tr.mem.Put(tr.ikScratch, value); err != nil { - return err - } - tr.seq++ - return nil -} - -// Put sets the value for the given key. It overwrites any previous value -// for that key; a DB is not a multi-map. -// Please note that the transaction is not compacted until committed, so if you -// writes 10 same keys, then those 10 same keys are in the transaction. -// -// It is safe to modify the contents of the arguments after Put returns. -func (tr *Transaction) Put(key, value []byte, wo *opt.WriteOptions) error { - tr.lk.Lock() - defer tr.lk.Unlock() - if tr.closed { - return errTransactionDone - } - return tr.put(keyTypeVal, key, value) -} - -// Delete deletes the value for the given key. -// Please note that the transaction is not compacted until committed, so if you -// writes 10 same keys, then those 10 same keys are in the transaction. -// -// It is safe to modify the contents of the arguments after Delete returns. -func (tr *Transaction) Delete(key []byte, wo *opt.WriteOptions) error { - tr.lk.Lock() - defer tr.lk.Unlock() - if tr.closed { - return errTransactionDone - } - return tr.put(keyTypeDel, key, nil) -} - -// Write apply the given batch to the transaction. The batch will be applied -// sequentially. -// Please note that the transaction is not compacted until committed, so if you -// writes 10 same keys, then those 10 same keys are in the transaction. -// -// It is safe to modify the contents of the arguments after Write returns. -func (tr *Transaction) Write(b *Batch, wo *opt.WriteOptions) error { - if b == nil || b.Len() == 0 { - return nil - } - - tr.lk.Lock() - defer tr.lk.Unlock() - if tr.closed { - return errTransactionDone - } - return b.replayInternal(func(i int, kt keyType, k, v []byte) error { - return tr.put(kt, k, v) - }) -} - -func (tr *Transaction) setDone() { - tr.closed = true - tr.db.tr = nil - tr.mem.decref() - <-tr.db.writeLockC -} - -// Commit commits the transaction. If error is not nil, then the transaction is -// not committed, it can then either be retried or discarded. -// -// Other methods should not be called after transaction has been committed. -func (tr *Transaction) Commit() error { - if err := tr.db.ok(); err != nil { - return err - } - - tr.lk.Lock() - defer tr.lk.Unlock() - if tr.closed { - return errTransactionDone - } - if err := tr.flush(); err != nil { - // Return error, lets user decide either to retry or discard - // transaction. - return err - } - if len(tr.tables) != 0 { - // Committing transaction. - tr.rec.setSeqNum(tr.seq) - tr.db.compCommitLk.Lock() - tr.stats.startTimer() - var cerr error - for retry := 0; retry < 3; retry++ { - cerr = tr.db.s.commit(&tr.rec) - if cerr != nil { - tr.db.logf("transaction@commit error R·%d %q", retry, cerr) - select { - case <-time.After(time.Second): - case <-tr.db.closeC: - tr.db.logf("transaction@commit exiting") - tr.db.compCommitLk.Unlock() - return cerr - } - } else { - // Success. Set db.seq. - tr.db.setSeq(tr.seq) - break - } - } - tr.stats.stopTimer() - if cerr != nil { - // Return error, lets user decide either to retry or discard - // transaction. - return cerr - } - - // Update compaction stats. This is safe as long as we hold compCommitLk. - tr.db.compStats.addStat(0, &tr.stats) - - // Trigger table auto-compaction. - tr.db.compTrigger(tr.db.tcompCmdC) - tr.db.compCommitLk.Unlock() - - // Additionally, wait compaction when certain threshold reached. - // Ignore error, returns error only if transaction can't be committed. - tr.db.waitCompaction() - } - // Only mark as done if transaction committed successfully. - tr.setDone() - return nil -} - -func (tr *Transaction) discard() { - // Discard transaction. - for _, t := range tr.tables { - tr.db.logf("transaction@discard @%d", t.fd.Num) - if err1 := tr.db.s.stor.Remove(t.fd); err1 == nil { - tr.db.s.reuseFileNum(t.fd.Num) - } - } -} - -// Discard discards the transaction. -// -// Other methods should not be called after transaction has been discarded. -func (tr *Transaction) Discard() { - tr.lk.Lock() - if !tr.closed { - tr.discard() - tr.setDone() - } - tr.lk.Unlock() -} - -func (db *DB) waitCompaction() error { - if db.s.tLen(0) >= db.s.o.GetWriteL0PauseTrigger() { - return db.compTriggerWait(db.tcompCmdC) - } - return nil -} - -// OpenTransaction opens an atomic DB transaction. Only one transaction can be -// opened at a time. Subsequent call to Write and OpenTransaction will be blocked -// until in-flight transaction is committed or discarded. -// The returned transaction handle is safe for concurrent use. -// -// Transaction is expensive and can overwhelm compaction, especially if -// transaction size is small. Use with caution. -// -// The transaction must be closed once done, either by committing or discarding -// the transaction. -// Closing the DB will discard open transaction. -func (db *DB) OpenTransaction() (*Transaction, error) { - if err := db.ok(); err != nil { - return nil, err - } - - // The write happen synchronously. - select { - case db.writeLockC <- struct{}{}: - case err := <-db.compPerErrC: - return nil, err - case <-db.closeC: - return nil, ErrClosed - } - - if db.tr != nil { - panic("leveldb: has open transaction") - } - - // Flush current memdb. - if db.mem != nil && db.mem.Len() != 0 { - if _, err := db.rotateMem(0, true); err != nil { - return nil, err - } - } - - // Wait compaction when certain threshold reached. - if err := db.waitCompaction(); err != nil { - return nil, err - } - - tr := &Transaction{ - db: db, - seq: db.seq, - mem: db.mpoolGet(0), - } - tr.mem.incref() - db.tr = tr - return tr, nil -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/db_util.go b/vendor/github.com/syndtr/goleveldb/leveldb/db_util.go deleted file mode 100644 index 7ecd960..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/db_util.go +++ /dev/null @@ -1,102 +0,0 @@ -// Copyright (c) 2012, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package leveldb - -import ( - "github.com/syndtr/goleveldb/leveldb/errors" - "github.com/syndtr/goleveldb/leveldb/iterator" - "github.com/syndtr/goleveldb/leveldb/opt" - "github.com/syndtr/goleveldb/leveldb/storage" - "github.com/syndtr/goleveldb/leveldb/util" -) - -// Reader is the interface that wraps basic Get and NewIterator methods. -// This interface implemented by both DB and Snapshot. -type Reader interface { - Get(key []byte, ro *opt.ReadOptions) (value []byte, err error) - NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator -} - -// Sizes is list of size. -type Sizes []int64 - -// Sum returns sum of the sizes. -func (sizes Sizes) Sum() int64 { - var sum int64 - for _, size := range sizes { - sum += size - } - return sum -} - -// Logging. -func (db *DB) log(v ...interface{}) { db.s.log(v...) } -func (db *DB) logf(format string, v ...interface{}) { db.s.logf(format, v...) } - -// Check and clean files. -func (db *DB) checkAndCleanFiles() error { - v := db.s.version() - defer v.release() - - tmap := make(map[int64]bool) - for _, tables := range v.levels { - for _, t := range tables { - tmap[t.fd.Num] = false - } - } - - fds, err := db.s.stor.List(storage.TypeAll) - if err != nil { - return err - } - - var nt int - var rem []storage.FileDesc - for _, fd := range fds { - keep := true - switch fd.Type { - case storage.TypeManifest: - keep = fd.Num >= db.s.manifestFd.Num - case storage.TypeJournal: - if !db.frozenJournalFd.Zero() { - keep = fd.Num >= db.frozenJournalFd.Num - } else { - keep = fd.Num >= db.journalFd.Num - } - case storage.TypeTable: - _, keep = tmap[fd.Num] - if keep { - tmap[fd.Num] = true - nt++ - } - } - - if !keep { - rem = append(rem, fd) - } - } - - if nt != len(tmap) { - var mfds []storage.FileDesc - for num, present := range tmap { - if !present { - mfds = append(mfds, storage.FileDesc{storage.TypeTable, num}) - db.logf("db@janitor table missing @%d", num) - } - } - return errors.NewErrCorrupted(storage.FileDesc{}, &errors.ErrMissingFiles{Fds: mfds}) - } - - db.logf("db@janitor F·%d G·%d", len(fds), len(rem)) - for _, fd := range rem { - db.logf("db@janitor removing %s-%d", fd.Type, fd.Num) - if err := db.s.stor.Remove(fd); err != nil { - return err - } - } - return nil -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/db_write.go b/vendor/github.com/syndtr/goleveldb/leveldb/db_write.go deleted file mode 100644 index 5b6cb48..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/db_write.go +++ /dev/null @@ -1,452 +0,0 @@ -// Copyright (c) 2012, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package leveldb - -import ( - "time" - - "github.com/syndtr/goleveldb/leveldb/memdb" - "github.com/syndtr/goleveldb/leveldb/opt" - "github.com/syndtr/goleveldb/leveldb/util" -) - -func (db *DB) writeJournal(batches []*Batch, seq uint64, sync bool) error { - wr, err := db.journal.Next() - if err != nil { - return err - } - if err := writeBatchesWithHeader(wr, batches, seq); err != nil { - return err - } - if err := db.journal.Flush(); err != nil { - return err - } - if sync { - return db.journalWriter.Sync() - } - return nil -} - -func (db *DB) rotateMem(n int, wait bool) (mem *memDB, err error) { - retryLimit := 3 -retry: - // Wait for pending memdb compaction. - err = db.compTriggerWait(db.mcompCmdC) - if err != nil { - return - } - retryLimit-- - - // Create new memdb and journal. - mem, err = db.newMem(n) - if err != nil { - if err == errHasFrozenMem { - if retryLimit <= 0 { - panic("BUG: still has frozen memdb") - } - goto retry - } - return - } - - // Schedule memdb compaction. - if wait { - err = db.compTriggerWait(db.mcompCmdC) - } else { - db.compTrigger(db.mcompCmdC) - } - return -} - -func (db *DB) flush(n int) (mdb *memDB, mdbFree int, err error) { - delayed := false - slowdownTrigger := db.s.o.GetWriteL0SlowdownTrigger() - pauseTrigger := db.s.o.GetWriteL0PauseTrigger() - flush := func() (retry bool) { - mdb = db.getEffectiveMem() - if mdb == nil { - err = ErrClosed - return false - } - defer func() { - if retry { - mdb.decref() - mdb = nil - } - }() - tLen := db.s.tLen(0) - mdbFree = mdb.Free() - switch { - case tLen >= slowdownTrigger && !delayed: - delayed = true - time.Sleep(time.Millisecond) - case mdbFree >= n: - return false - case tLen >= pauseTrigger: - delayed = true - err = db.compTriggerWait(db.tcompCmdC) - if err != nil { - return false - } - default: - // Allow memdb to grow if it has no entry. - if mdb.Len() == 0 { - mdbFree = n - } else { - mdb.decref() - mdb, err = db.rotateMem(n, false) - if err == nil { - mdbFree = mdb.Free() - } else { - mdbFree = 0 - } - } - return false - } - return true - } - start := time.Now() - for flush() { - } - if delayed { - db.writeDelay += time.Since(start) - db.writeDelayN++ - } else if db.writeDelayN > 0 { - db.logf("db@write was delayed N·%d T·%v", db.writeDelayN, db.writeDelay) - db.writeDelay = 0 - db.writeDelayN = 0 - } - return -} - -type writeMerge struct { - sync bool - batch *Batch - keyType keyType - key, value []byte -} - -func (db *DB) unlockWrite(overflow bool, merged int, err error) { - for i := 0; i < merged; i++ { - db.writeAckC <- err - } - if overflow { - // Pass lock to the next write (that failed to merge). - db.writeMergedC <- false - } else { - // Release lock. - <-db.writeLockC - } -} - -// ourBatch if defined should equal with batch. -func (db *DB) writeLocked(batch, ourBatch *Batch, merge, sync bool) error { - // Try to flush memdb. This method would also trying to throttle writes - // if it is too fast and compaction cannot catch-up. - mdb, mdbFree, err := db.flush(batch.internalLen) - if err != nil { - db.unlockWrite(false, 0, err) - return err - } - defer mdb.decref() - - var ( - overflow bool - merged int - batches = []*Batch{batch} - ) - - if merge { - // Merge limit. - var mergeLimit int - if batch.internalLen > 128<<10 { - mergeLimit = (1 << 20) - batch.internalLen - } else { - mergeLimit = 128 << 10 - } - mergeCap := mdbFree - batch.internalLen - if mergeLimit > mergeCap { - mergeLimit = mergeCap - } - - merge: - for mergeLimit > 0 { - select { - case incoming := <-db.writeMergeC: - if incoming.batch != nil { - // Merge batch. - if incoming.batch.internalLen > mergeLimit { - overflow = true - break merge - } - batches = append(batches, incoming.batch) - mergeLimit -= incoming.batch.internalLen - } else { - // Merge put. - internalLen := len(incoming.key) + len(incoming.value) + 8 - if internalLen > mergeLimit { - overflow = true - break merge - } - if ourBatch == nil { - ourBatch = db.batchPool.Get().(*Batch) - ourBatch.Reset() - batches = append(batches, ourBatch) - } - // We can use same batch since concurrent write doesn't - // guarantee write order. - ourBatch.appendRec(incoming.keyType, incoming.key, incoming.value) - mergeLimit -= internalLen - } - sync = sync || incoming.sync - merged++ - db.writeMergedC <- true - - default: - break merge - } - } - } - - // Seq number. - seq := db.seq + 1 - - // Write journal. - if err := db.writeJournal(batches, seq, sync); err != nil { - db.unlockWrite(overflow, merged, err) - return err - } - - // Put batches. - for _, batch := range batches { - if err := batch.putMem(seq, mdb.DB); err != nil { - panic(err) - } - seq += uint64(batch.Len()) - } - - // Incr seq number. - db.addSeq(uint64(batchesLen(batches))) - - // Rotate memdb if it's reach the threshold. - if batch.internalLen >= mdbFree { - db.rotateMem(0, false) - } - - db.unlockWrite(overflow, merged, nil) - return nil -} - -// Write apply the given batch to the DB. The batch records will be applied -// sequentially. Write might be used concurrently, when used concurrently and -// batch is small enough, write will try to merge the batches. Set NoWriteMerge -// option to true to disable write merge. -// -// It is safe to modify the contents of the arguments after Write returns but -// not before. Write will not modify content of the batch. -func (db *DB) Write(batch *Batch, wo *opt.WriteOptions) error { - if err := db.ok(); err != nil || batch == nil || batch.Len() == 0 { - return err - } - - // If the batch size is larger than write buffer, it may justified to write - // using transaction instead. Using transaction the batch will be written - // into tables directly, skipping the journaling. - if batch.internalLen > db.s.o.GetWriteBuffer() && !db.s.o.GetDisableLargeBatchTransaction() { - tr, err := db.OpenTransaction() - if err != nil { - return err - } - if err := tr.Write(batch, wo); err != nil { - tr.Discard() - return err - } - return tr.Commit() - } - - merge := !wo.GetNoWriteMerge() && !db.s.o.GetNoWriteMerge() - sync := wo.GetSync() && !db.s.o.GetNoSync() - - // Acquire write lock. - if merge { - select { - case db.writeMergeC <- writeMerge{sync: sync, batch: batch}: - if <-db.writeMergedC { - // Write is merged. - return <-db.writeAckC - } - // Write is not merged, the write lock is handed to us. Continue. - case db.writeLockC <- struct{}{}: - // Write lock acquired. - case err := <-db.compPerErrC: - // Compaction error. - return err - case <-db.closeC: - // Closed - return ErrClosed - } - } else { - select { - case db.writeLockC <- struct{}{}: - // Write lock acquired. - case err := <-db.compPerErrC: - // Compaction error. - return err - case <-db.closeC: - // Closed - return ErrClosed - } - } - - return db.writeLocked(batch, nil, merge, sync) -} - -func (db *DB) putRec(kt keyType, key, value []byte, wo *opt.WriteOptions) error { - if err := db.ok(); err != nil { - return err - } - - merge := !wo.GetNoWriteMerge() && !db.s.o.GetNoWriteMerge() - sync := wo.GetSync() && !db.s.o.GetNoSync() - - // Acquire write lock. - if merge { - select { - case db.writeMergeC <- writeMerge{sync: sync, keyType: kt, key: key, value: value}: - if <-db.writeMergedC { - // Write is merged. - return <-db.writeAckC - } - // Write is not merged, the write lock is handed to us. Continue. - case db.writeLockC <- struct{}{}: - // Write lock acquired. - case err := <-db.compPerErrC: - // Compaction error. - return err - case <-db.closeC: - // Closed - return ErrClosed - } - } else { - select { - case db.writeLockC <- struct{}{}: - // Write lock acquired. - case err := <-db.compPerErrC: - // Compaction error. - return err - case <-db.closeC: - // Closed - return ErrClosed - } - } - - batch := db.batchPool.Get().(*Batch) - batch.Reset() - batch.appendRec(kt, key, value) - return db.writeLocked(batch, batch, merge, sync) -} - -// Put sets the value for the given key. It overwrites any previous value -// for that key; a DB is not a multi-map. Write merge also applies for Put, see -// Write. -// -// It is safe to modify the contents of the arguments after Put returns but not -// before. -func (db *DB) Put(key, value []byte, wo *opt.WriteOptions) error { - return db.putRec(keyTypeVal, key, value, wo) -} - -// Delete deletes the value for the given key. Delete will not returns error if -// key doesn't exist. Write merge also applies for Delete, see Write. -// -// It is safe to modify the contents of the arguments after Delete returns but -// not before. -func (db *DB) Delete(key []byte, wo *opt.WriteOptions) error { - return db.putRec(keyTypeDel, key, nil, wo) -} - -func isMemOverlaps(icmp *iComparer, mem *memdb.DB, min, max []byte) bool { - iter := mem.NewIterator(nil) - defer iter.Release() - return (max == nil || (iter.First() && icmp.uCompare(max, internalKey(iter.Key()).ukey()) >= 0)) && - (min == nil || (iter.Last() && icmp.uCompare(min, internalKey(iter.Key()).ukey()) <= 0)) -} - -// CompactRange compacts the underlying DB for the given key range. -// In particular, deleted and overwritten versions are discarded, -// and the data is rearranged to reduce the cost of operations -// needed to access the data. This operation should typically only -// be invoked by users who understand the underlying implementation. -// -// A nil Range.Start is treated as a key before all keys in the DB. -// And a nil Range.Limit is treated as a key after all keys in the DB. -// Therefore if both is nil then it will compact entire DB. -func (db *DB) CompactRange(r util.Range) error { - if err := db.ok(); err != nil { - return err - } - - // Lock writer. - select { - case db.writeLockC <- struct{}{}: - case err := <-db.compPerErrC: - return err - case <-db.closeC: - return ErrClosed - } - - // Check for overlaps in memdb. - mdb := db.getEffectiveMem() - if mdb == nil { - return ErrClosed - } - defer mdb.decref() - if isMemOverlaps(db.s.icmp, mdb.DB, r.Start, r.Limit) { - // Memdb compaction. - if _, err := db.rotateMem(0, false); err != nil { - <-db.writeLockC - return err - } - <-db.writeLockC - if err := db.compTriggerWait(db.mcompCmdC); err != nil { - return err - } - } else { - <-db.writeLockC - } - - // Table compaction. - return db.compTriggerRange(db.tcompCmdC, -1, r.Start, r.Limit) -} - -// SetReadOnly makes DB read-only. It will stay read-only until reopened. -func (db *DB) SetReadOnly() error { - if err := db.ok(); err != nil { - return err - } - - // Lock writer. - select { - case db.writeLockC <- struct{}{}: - db.compWriteLocking = true - case err := <-db.compPerErrC: - return err - case <-db.closeC: - return ErrClosed - } - - // Set compaction read-only. - select { - case db.compErrSetC <- ErrReadOnly: - case perr := <-db.compPerErrC: - return perr - case <-db.closeC: - return ErrClosed - } - - return nil -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/doc.go b/vendor/github.com/syndtr/goleveldb/leveldb/doc.go deleted file mode 100644 index 53f13bb..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/doc.go +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright (c) 2012, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -// Package leveldb provides implementation of LevelDB key/value database. -// -// Create or open a database: -// -// db, err := leveldb.OpenFile("path/to/db", nil) -// ... -// defer db.Close() -// ... -// -// Read or modify the database content: -// -// // Remember that the contents of the returned slice should not be modified. -// data, err := db.Get([]byte("key"), nil) -// ... -// err = db.Put([]byte("key"), []byte("value"), nil) -// ... -// err = db.Delete([]byte("key"), nil) -// ... -// -// Iterate over database content: -// -// iter := db.NewIterator(nil, nil) -// for iter.Next() { -// // Remember that the contents of the returned slice should not be modified, and -// // only valid until the next call to Next. -// key := iter.Key() -// value := iter.Value() -// ... -// } -// iter.Release() -// err = iter.Error() -// ... -// -// Iterate over subset of database content with a particular prefix: -// iter := db.NewIterator(util.BytesPrefix([]byte("foo-")), nil) -// for iter.Next() { -// // Use key/value. -// ... -// } -// iter.Release() -// err = iter.Error() -// ... -// -// Seek-then-Iterate: -// -// iter := db.NewIterator(nil, nil) -// for ok := iter.Seek(key); ok; ok = iter.Next() { -// // Use key/value. -// ... -// } -// iter.Release() -// err = iter.Error() -// ... -// -// Iterate over subset of database content: -// -// iter := db.NewIterator(&util.Range{Start: []byte("foo"), Limit: []byte("xoo")}, nil) -// for iter.Next() { -// // Use key/value. -// ... -// } -// iter.Release() -// err = iter.Error() -// ... -// -// Batch writes: -// -// batch := new(leveldb.Batch) -// batch.Put([]byte("foo"), []byte("value")) -// batch.Put([]byte("bar"), []byte("another value")) -// batch.Delete([]byte("baz")) -// err = db.Write(batch, nil) -// ... -// -// Use bloom filter: -// -// o := &opt.Options{ -// Filter: filter.NewBloomFilter(10), -// } -// db, err := leveldb.OpenFile("path/to/db", o) -// ... -// defer db.Close() -// ... -package leveldb diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/errors.go b/vendor/github.com/syndtr/goleveldb/leveldb/errors.go deleted file mode 100644 index de26498..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/errors.go +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright (c) 2014, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package leveldb - -import ( - "github.com/syndtr/goleveldb/leveldb/errors" -) - -// Common errors. -var ( - ErrNotFound = errors.ErrNotFound - ErrReadOnly = errors.New("leveldb: read-only mode") - ErrSnapshotReleased = errors.New("leveldb: snapshot released") - ErrIterReleased = errors.New("leveldb: iterator released") - ErrClosed = errors.New("leveldb: closed") -) diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/errors/errors.go b/vendor/github.com/syndtr/goleveldb/leveldb/errors/errors.go deleted file mode 100644 index 8d6146b..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/errors/errors.go +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright (c) 2014, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -// Package errors provides common error types used throughout leveldb. -package errors - -import ( - "errors" - "fmt" - - "github.com/syndtr/goleveldb/leveldb/storage" - "github.com/syndtr/goleveldb/leveldb/util" -) - -// Common errors. -var ( - ErrNotFound = New("leveldb: not found") - ErrReleased = util.ErrReleased - ErrHasReleaser = util.ErrHasReleaser -) - -// New returns an error that formats as the given text. -func New(text string) error { - return errors.New(text) -} - -// ErrCorrupted is the type that wraps errors that indicate corruption in -// the database. -type ErrCorrupted struct { - Fd storage.FileDesc - Err error -} - -func (e *ErrCorrupted) Error() string { - if !e.Fd.Zero() { - return fmt.Sprintf("%v [file=%v]", e.Err, e.Fd) - } - return e.Err.Error() -} - -// NewErrCorrupted creates new ErrCorrupted error. -func NewErrCorrupted(fd storage.FileDesc, err error) error { - return &ErrCorrupted{fd, err} -} - -// IsCorrupted returns a boolean indicating whether the error is indicating -// a corruption. -func IsCorrupted(err error) bool { - switch err.(type) { - case *ErrCorrupted: - return true - case *storage.ErrCorrupted: - return true - } - return false -} - -// ErrMissingFiles is the type that indicating a corruption due to missing -// files. ErrMissingFiles always wrapped with ErrCorrupted. -type ErrMissingFiles struct { - Fds []storage.FileDesc -} - -func (e *ErrMissingFiles) Error() string { return "file missing" } - -// SetFd sets 'file info' of the given error with the given file. -// Currently only ErrCorrupted is supported, otherwise will do nothing. -func SetFd(err error, fd storage.FileDesc) error { - switch x := err.(type) { - case *ErrCorrupted: - x.Fd = fd - return x - } - return err -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/external_test.go b/vendor/github.com/syndtr/goleveldb/leveldb/external_test.go deleted file mode 100644 index 669d77f..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/external_test.go +++ /dev/null @@ -1,117 +0,0 @@ -// Copyright (c) 2014, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package leveldb - -import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - - "github.com/syndtr/goleveldb/leveldb/opt" - "github.com/syndtr/goleveldb/leveldb/testutil" -) - -var _ = testutil.Defer(func() { - Describe("Leveldb external", func() { - o := &opt.Options{ - DisableBlockCache: true, - BlockRestartInterval: 5, - BlockSize: 80, - Compression: opt.NoCompression, - OpenFilesCacheCapacity: -1, - Strict: opt.StrictAll, - WriteBuffer: 1000, - CompactionTableSize: 2000, - } - - Describe("write test", func() { - It("should do write correctly", func(done Done) { - db := newTestingDB(o, nil, nil) - t := testutil.DBTesting{ - DB: db, - Deleted: testutil.KeyValue_Generate(nil, 500, 1, 1, 50, 5, 5).Clone(), - } - testutil.DoDBTesting(&t) - db.TestClose() - done <- true - }, 80.0) - }) - - Describe("read test", func() { - testutil.AllKeyValueTesting(nil, nil, func(kv testutil.KeyValue) testutil.DB { - // Building the DB. - db := newTestingDB(o, nil, nil) - kv.IterateShuffled(nil, func(i int, key, value []byte) { - err := db.TestPut(key, value) - Expect(err).NotTo(HaveOccurred()) - }) - - return db - }, func(db testutil.DB) { - db.(*testingDB).TestClose() - }) - }) - - Describe("transaction test", func() { - It("should do transaction correctly", func(done Done) { - db := newTestingDB(o, nil, nil) - - By("creating first transaction") - var err error - tr := &testingTransaction{} - tr.Transaction, err = db.OpenTransaction() - Expect(err).NotTo(HaveOccurred()) - t0 := &testutil.DBTesting{ - DB: tr, - Deleted: testutil.KeyValue_Generate(nil, 200, 1, 1, 50, 5, 5).Clone(), - } - testutil.DoDBTesting(t0) - testutil.TestGet(tr, t0.Present) - testutil.TestHas(tr, t0.Present) - - By("committing first transaction") - err = tr.Commit() - Expect(err).NotTo(HaveOccurred()) - testutil.TestIter(db, nil, t0.Present) - testutil.TestGet(db, t0.Present) - testutil.TestHas(db, t0.Present) - - By("manipulating DB without transaction") - t0.DB = db - testutil.DoDBTesting(t0) - - By("creating second transaction") - tr.Transaction, err = db.OpenTransaction() - Expect(err).NotTo(HaveOccurred()) - t1 := &testutil.DBTesting{ - DB: tr, - Deleted: t0.Deleted.Clone(), - Present: t0.Present.Clone(), - } - testutil.DoDBTesting(t1) - testutil.TestIter(db, nil, t0.Present) - - By("discarding second transaction") - tr.Discard() - testutil.TestIter(db, nil, t0.Present) - - By("creating third transaction") - tr.Transaction, err = db.OpenTransaction() - Expect(err).NotTo(HaveOccurred()) - t0.DB = tr - testutil.DoDBTesting(t0) - - By("committing third transaction") - err = tr.Commit() - Expect(err).NotTo(HaveOccurred()) - testutil.TestIter(db, nil, t0.Present) - - db.TestClose() - done <- true - }, 240.0) - }) - }) -}) diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/filter.go b/vendor/github.com/syndtr/goleveldb/leveldb/filter.go deleted file mode 100644 index e961e42..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/filter.go +++ /dev/null @@ -1,31 +0,0 @@ -// Copyright (c) 2012, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package leveldb - -import ( - "github.com/syndtr/goleveldb/leveldb/filter" -) - -type iFilter struct { - filter.Filter -} - -func (f iFilter) Contains(filter, key []byte) bool { - return f.Filter.Contains(filter, internalKey(key).ukey()) -} - -func (f iFilter) NewGenerator() filter.FilterGenerator { - return iFilterGenerator{f.Filter.NewGenerator()} -} - -type iFilterGenerator struct { - filter.FilterGenerator -} - -func (g iFilterGenerator) Add(key []byte) { - g.FilterGenerator.Add(internalKey(key).ukey()) -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/filter/bloom.go b/vendor/github.com/syndtr/goleveldb/leveldb/filter/bloom.go deleted file mode 100644 index bab0e99..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/filter/bloom.go +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright (c) 2012, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package filter - -import ( - "github.com/syndtr/goleveldb/leveldb/util" -) - -func bloomHash(key []byte) uint32 { - return util.Hash(key, 0xbc9f1d34) -} - -type bloomFilter int - -// The bloom filter serializes its parameters and is backward compatible -// with respect to them. Therefor, its parameters are not added to its -// name. -func (bloomFilter) Name() string { - return "leveldb.BuiltinBloomFilter" -} - -func (f bloomFilter) Contains(filter, key []byte) bool { - nBytes := len(filter) - 1 - if nBytes < 1 { - return false - } - nBits := uint32(nBytes * 8) - - // Use the encoded k so that we can read filters generated by - // bloom filters created using different parameters. - k := filter[nBytes] - if k > 30 { - // Reserved for potentially new encodings for short bloom filters. - // Consider it a match. - return true - } - - kh := bloomHash(key) - delta := (kh >> 17) | (kh << 15) // Rotate right 17 bits - for j := uint8(0); j < k; j++ { - bitpos := kh % nBits - if (uint32(filter[bitpos/8]) & (1 << (bitpos % 8))) == 0 { - return false - } - kh += delta - } - return true -} - -func (f bloomFilter) NewGenerator() FilterGenerator { - // Round down to reduce probing cost a little bit. - k := uint8(f * 69 / 100) // 0.69 =~ ln(2) - if k < 1 { - k = 1 - } else if k > 30 { - k = 30 - } - return &bloomFilterGenerator{ - n: int(f), - k: k, - } -} - -type bloomFilterGenerator struct { - n int - k uint8 - - keyHashes []uint32 -} - -func (g *bloomFilterGenerator) Add(key []byte) { - // Use double-hashing to generate a sequence of hash values. - // See analysis in [Kirsch,Mitzenmacher 2006]. - g.keyHashes = append(g.keyHashes, bloomHash(key)) -} - -func (g *bloomFilterGenerator) Generate(b Buffer) { - // Compute bloom filter size (in both bits and bytes) - nBits := uint32(len(g.keyHashes) * g.n) - // For small n, we can see a very high false positive rate. Fix it - // by enforcing a minimum bloom filter length. - if nBits < 64 { - nBits = 64 - } - nBytes := (nBits + 7) / 8 - nBits = nBytes * 8 - - dest := b.Alloc(int(nBytes) + 1) - dest[nBytes] = g.k - for _, kh := range g.keyHashes { - delta := (kh >> 17) | (kh << 15) // Rotate right 17 bits - for j := uint8(0); j < g.k; j++ { - bitpos := kh % nBits - dest[bitpos/8] |= (1 << (bitpos % 8)) - kh += delta - } - } - - g.keyHashes = g.keyHashes[:0] -} - -// NewBloomFilter creates a new initialized bloom filter for given -// bitsPerKey. -// -// Since bitsPerKey is persisted individually for each bloom filter -// serialization, bloom filters are backwards compatible with respect to -// changing bitsPerKey. This means that no big performance penalty will -// be experienced when changing the parameter. See documentation for -// opt.Options.Filter for more information. -func NewBloomFilter(bitsPerKey int) Filter { - return bloomFilter(bitsPerKey) -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/filter/bloom_test.go b/vendor/github.com/syndtr/goleveldb/leveldb/filter/bloom_test.go deleted file mode 100644 index 1fb56f0..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/filter/bloom_test.go +++ /dev/null @@ -1,142 +0,0 @@ -// Copyright (c) 2012, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package filter - -import ( - "encoding/binary" - "github.com/syndtr/goleveldb/leveldb/util" - "testing" -) - -type harness struct { - t *testing.T - - bloom Filter - generator FilterGenerator - filter []byte -} - -func newHarness(t *testing.T) *harness { - bloom := NewBloomFilter(10) - return &harness{ - t: t, - bloom: bloom, - generator: bloom.NewGenerator(), - } -} - -func (h *harness) add(key []byte) { - h.generator.Add(key) -} - -func (h *harness) addNum(key uint32) { - var b [4]byte - binary.LittleEndian.PutUint32(b[:], key) - h.add(b[:]) -} - -func (h *harness) build() { - b := &util.Buffer{} - h.generator.Generate(b) - h.filter = b.Bytes() -} - -func (h *harness) reset() { - h.filter = nil -} - -func (h *harness) filterLen() int { - return len(h.filter) -} - -func (h *harness) assert(key []byte, want, silent bool) bool { - got := h.bloom.Contains(h.filter, key) - if !silent && got != want { - h.t.Errorf("assert on '%v' failed got '%v', want '%v'", key, got, want) - } - return got -} - -func (h *harness) assertNum(key uint32, want, silent bool) bool { - var b [4]byte - binary.LittleEndian.PutUint32(b[:], key) - return h.assert(b[:], want, silent) -} - -func TestBloomFilter_Empty(t *testing.T) { - h := newHarness(t) - h.build() - h.assert([]byte("hello"), false, false) - h.assert([]byte("world"), false, false) -} - -func TestBloomFilter_Small(t *testing.T) { - h := newHarness(t) - h.add([]byte("hello")) - h.add([]byte("world")) - h.build() - h.assert([]byte("hello"), true, false) - h.assert([]byte("world"), true, false) - h.assert([]byte("x"), false, false) - h.assert([]byte("foo"), false, false) -} - -func nextN(n int) int { - switch { - case n < 10: - n += 1 - case n < 100: - n += 10 - case n < 1000: - n += 100 - default: - n += 1000 - } - return n -} - -func TestBloomFilter_VaryingLengths(t *testing.T) { - h := newHarness(t) - var mediocre, good int - for n := 1; n < 10000; n = nextN(n) { - h.reset() - for i := 0; i < n; i++ { - h.addNum(uint32(i)) - } - h.build() - - got := h.filterLen() - want := (n * 10 / 8) + 40 - if got > want { - t.Errorf("filter len test failed, '%d' > '%d'", got, want) - } - - for i := 0; i < n; i++ { - h.assertNum(uint32(i), true, false) - } - - var rate float32 - for i := 0; i < 10000; i++ { - if h.assertNum(uint32(i+1000000000), true, true) { - rate++ - } - } - rate /= 10000 - if rate > 0.02 { - t.Errorf("false positive rate is more than 2%%, got %v, at len %d", rate, n) - } - if rate > 0.0125 { - mediocre++ - } else { - good++ - } - } - t.Logf("false positive rate: %d good, %d mediocre", good, mediocre) - if mediocre > good/5 { - t.Error("mediocre false positive rate is more than expected") - } -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/filter/filter.go b/vendor/github.com/syndtr/goleveldb/leveldb/filter/filter.go deleted file mode 100644 index 7a925c5..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/filter/filter.go +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright (c) 2012, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -// Package filter provides interface and implementation of probabilistic -// data structure. -// -// The filter is resposible for creating small filter from a set of keys. -// These filter will then used to test whether a key is a member of the set. -// In many cases, a filter can cut down the number of disk seeks from a -// handful to a single disk seek per DB.Get call. -package filter - -// Buffer is the interface that wraps basic Alloc, Write and WriteByte methods. -type Buffer interface { - // Alloc allocs n bytes of slice from the buffer. This also advancing - // write offset. - Alloc(n int) []byte - - // Write appends the contents of p to the buffer. - Write(p []byte) (n int, err error) - - // WriteByte appends the byte c to the buffer. - WriteByte(c byte) error -} - -// Filter is the filter. -type Filter interface { - // Name returns the name of this policy. - // - // Note that if the filter encoding changes in an incompatible way, - // the name returned by this method must be changed. Otherwise, old - // incompatible filters may be passed to methods of this type. - Name() string - - // NewGenerator creates a new filter generator. - NewGenerator() FilterGenerator - - // Contains returns true if the filter contains the given key. - // - // The filter are filters generated by the filter generator. - Contains(filter, key []byte) bool -} - -// FilterGenerator is the filter generator. -type FilterGenerator interface { - // Add adds a key to the filter generator. - // - // The key may become invalid after call to this method end, therefor - // key must be copied if implementation require keeping key for later - // use. The key should not modified directly, doing so may cause - // undefined results. - Add(key []byte) - - // Generate generates filters based on keys passed so far. After call - // to Generate the filter generator maybe resetted, depends on implementation. - Generate(b Buffer) -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/iterator/array_iter.go b/vendor/github.com/syndtr/goleveldb/leveldb/iterator/array_iter.go deleted file mode 100644 index a23ab05..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/iterator/array_iter.go +++ /dev/null @@ -1,184 +0,0 @@ -// Copyright (c) 2014, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package iterator - -import ( - "github.com/syndtr/goleveldb/leveldb/util" -) - -// BasicArray is the interface that wraps basic Len and Search method. -type BasicArray interface { - // Len returns length of the array. - Len() int - - // Search finds smallest index that point to a key that is greater - // than or equal to the given key. - Search(key []byte) int -} - -// Array is the interface that wraps BasicArray and basic Index method. -type Array interface { - BasicArray - - // Index returns key/value pair with index of i. - Index(i int) (key, value []byte) -} - -// Array is the interface that wraps BasicArray and basic Get method. -type ArrayIndexer interface { - BasicArray - - // Get returns a new data iterator with index of i. - Get(i int) Iterator -} - -type basicArrayIterator struct { - util.BasicReleaser - array BasicArray - pos int - err error -} - -func (i *basicArrayIterator) Valid() bool { - return i.pos >= 0 && i.pos < i.array.Len() && !i.Released() -} - -func (i *basicArrayIterator) First() bool { - if i.Released() { - i.err = ErrIterReleased - return false - } - - if i.array.Len() == 0 { - i.pos = -1 - return false - } - i.pos = 0 - return true -} - -func (i *basicArrayIterator) Last() bool { - if i.Released() { - i.err = ErrIterReleased - return false - } - - n := i.array.Len() - if n == 0 { - i.pos = 0 - return false - } - i.pos = n - 1 - return true -} - -func (i *basicArrayIterator) Seek(key []byte) bool { - if i.Released() { - i.err = ErrIterReleased - return false - } - - n := i.array.Len() - if n == 0 { - i.pos = 0 - return false - } - i.pos = i.array.Search(key) - if i.pos >= n { - return false - } - return true -} - -func (i *basicArrayIterator) Next() bool { - if i.Released() { - i.err = ErrIterReleased - return false - } - - i.pos++ - if n := i.array.Len(); i.pos >= n { - i.pos = n - return false - } - return true -} - -func (i *basicArrayIterator) Prev() bool { - if i.Released() { - i.err = ErrIterReleased - return false - } - - i.pos-- - if i.pos < 0 { - i.pos = -1 - return false - } - return true -} - -func (i *basicArrayIterator) Error() error { return i.err } - -type arrayIterator struct { - basicArrayIterator - array Array - pos int - key, value []byte -} - -func (i *arrayIterator) updateKV() { - if i.pos == i.basicArrayIterator.pos { - return - } - i.pos = i.basicArrayIterator.pos - if i.Valid() { - i.key, i.value = i.array.Index(i.pos) - } else { - i.key = nil - i.value = nil - } -} - -func (i *arrayIterator) Key() []byte { - i.updateKV() - return i.key -} - -func (i *arrayIterator) Value() []byte { - i.updateKV() - return i.value -} - -type arrayIteratorIndexer struct { - basicArrayIterator - array ArrayIndexer -} - -func (i *arrayIteratorIndexer) Get() Iterator { - if i.Valid() { - return i.array.Get(i.basicArrayIterator.pos) - } - return nil -} - -// NewArrayIterator returns an iterator from the given array. -func NewArrayIterator(array Array) Iterator { - return &arrayIterator{ - basicArrayIterator: basicArrayIterator{array: array, pos: -1}, - array: array, - pos: -1, - } -} - -// NewArrayIndexer returns an index iterator from the given array. -func NewArrayIndexer(array ArrayIndexer) IteratorIndexer { - return &arrayIteratorIndexer{ - basicArrayIterator: basicArrayIterator{array: array, pos: -1}, - array: array, - } -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/iterator/array_iter_test.go b/vendor/github.com/syndtr/goleveldb/leveldb/iterator/array_iter_test.go deleted file mode 100644 index f16d014..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/iterator/array_iter_test.go +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright (c) 2014, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package iterator_test - -import ( - . "github.com/onsi/ginkgo" - - . "github.com/syndtr/goleveldb/leveldb/iterator" - "github.com/syndtr/goleveldb/leveldb/testutil" -) - -var _ = testutil.Defer(func() { - Describe("Array iterator", func() { - It("Should iterates and seeks correctly", func() { - // Build key/value. - kv := testutil.KeyValue_Generate(nil, 70, 1, 1, 5, 3, 3) - - // Test the iterator. - t := testutil.IteratorTesting{ - KeyValue: kv.Clone(), - Iter: NewArrayIterator(kv), - } - testutil.DoIteratorTesting(&t) - }) - }) -}) diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/iterator/indexed_iter.go b/vendor/github.com/syndtr/goleveldb/leveldb/iterator/indexed_iter.go deleted file mode 100644 index 939adbb..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/iterator/indexed_iter.go +++ /dev/null @@ -1,242 +0,0 @@ -// Copyright (c) 2012, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package iterator - -import ( - "github.com/syndtr/goleveldb/leveldb/errors" - "github.com/syndtr/goleveldb/leveldb/util" -) - -// IteratorIndexer is the interface that wraps CommonIterator and basic Get -// method. IteratorIndexer provides index for indexed iterator. -type IteratorIndexer interface { - CommonIterator - - // Get returns a new data iterator for the current position, or nil if - // done. - Get() Iterator -} - -type indexedIterator struct { - util.BasicReleaser - index IteratorIndexer - strict bool - - data Iterator - err error - errf func(err error) - closed bool -} - -func (i *indexedIterator) setData() { - if i.data != nil { - i.data.Release() - } - i.data = i.index.Get() -} - -func (i *indexedIterator) clearData() { - if i.data != nil { - i.data.Release() - } - i.data = nil -} - -func (i *indexedIterator) indexErr() { - if err := i.index.Error(); err != nil { - if i.errf != nil { - i.errf(err) - } - i.err = err - } -} - -func (i *indexedIterator) dataErr() bool { - if err := i.data.Error(); err != nil { - if i.errf != nil { - i.errf(err) - } - if i.strict || !errors.IsCorrupted(err) { - i.err = err - return true - } - } - return false -} - -func (i *indexedIterator) Valid() bool { - return i.data != nil && i.data.Valid() -} - -func (i *indexedIterator) First() bool { - if i.err != nil { - return false - } else if i.Released() { - i.err = ErrIterReleased - return false - } - - if !i.index.First() { - i.indexErr() - i.clearData() - return false - } - i.setData() - return i.Next() -} - -func (i *indexedIterator) Last() bool { - if i.err != nil { - return false - } else if i.Released() { - i.err = ErrIterReleased - return false - } - - if !i.index.Last() { - i.indexErr() - i.clearData() - return false - } - i.setData() - if !i.data.Last() { - if i.dataErr() { - return false - } - i.clearData() - return i.Prev() - } - return true -} - -func (i *indexedIterator) Seek(key []byte) bool { - if i.err != nil { - return false - } else if i.Released() { - i.err = ErrIterReleased - return false - } - - if !i.index.Seek(key) { - i.indexErr() - i.clearData() - return false - } - i.setData() - if !i.data.Seek(key) { - if i.dataErr() { - return false - } - i.clearData() - return i.Next() - } - return true -} - -func (i *indexedIterator) Next() bool { - if i.err != nil { - return false - } else if i.Released() { - i.err = ErrIterReleased - return false - } - - switch { - case i.data != nil && !i.data.Next(): - if i.dataErr() { - return false - } - i.clearData() - fallthrough - case i.data == nil: - if !i.index.Next() { - i.indexErr() - return false - } - i.setData() - return i.Next() - } - return true -} - -func (i *indexedIterator) Prev() bool { - if i.err != nil { - return false - } else if i.Released() { - i.err = ErrIterReleased - return false - } - - switch { - case i.data != nil && !i.data.Prev(): - if i.dataErr() { - return false - } - i.clearData() - fallthrough - case i.data == nil: - if !i.index.Prev() { - i.indexErr() - return false - } - i.setData() - if !i.data.Last() { - if i.dataErr() { - return false - } - i.clearData() - return i.Prev() - } - } - return true -} - -func (i *indexedIterator) Key() []byte { - if i.data == nil { - return nil - } - return i.data.Key() -} - -func (i *indexedIterator) Value() []byte { - if i.data == nil { - return nil - } - return i.data.Value() -} - -func (i *indexedIterator) Release() { - i.clearData() - i.index.Release() - i.BasicReleaser.Release() -} - -func (i *indexedIterator) Error() error { - if i.err != nil { - return i.err - } - if err := i.index.Error(); err != nil { - return err - } - return nil -} - -func (i *indexedIterator) SetErrorCallback(f func(err error)) { - i.errf = f -} - -// NewIndexedIterator returns an 'indexed iterator'. An index is iterator -// that returns another iterator, a 'data iterator'. A 'data iterator' is the -// iterator that contains actual key/value pairs. -// -// If strict is true the any 'corruption errors' (i.e errors.IsCorrupted(err) == true) -// won't be ignored and will halt 'indexed iterator', otherwise the iterator will -// continue to the next 'data iterator'. Corruption on 'index iterator' will not be -// ignored and will halt the iterator. -func NewIndexedIterator(index IteratorIndexer, strict bool) Iterator { - return &indexedIterator{index: index, strict: strict} -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/iterator/indexed_iter_test.go b/vendor/github.com/syndtr/goleveldb/leveldb/iterator/indexed_iter_test.go deleted file mode 100644 index fde8016..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/iterator/indexed_iter_test.go +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright (c) 2014, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package iterator_test - -import ( - "sort" - - . "github.com/onsi/ginkgo" - - "github.com/syndtr/goleveldb/leveldb/comparer" - . "github.com/syndtr/goleveldb/leveldb/iterator" - "github.com/syndtr/goleveldb/leveldb/testutil" -) - -type keyValue struct { - key []byte - testutil.KeyValue -} - -type keyValueIndex []keyValue - -func (x keyValueIndex) Search(key []byte) int { - return sort.Search(x.Len(), func(i int) bool { - return comparer.DefaultComparer.Compare(x[i].key, key) >= 0 - }) -} - -func (x keyValueIndex) Len() int { return len(x) } -func (x keyValueIndex) Index(i int) (key, value []byte) { return x[i].key, nil } -func (x keyValueIndex) Get(i int) Iterator { return NewArrayIterator(x[i]) } - -var _ = testutil.Defer(func() { - Describe("Indexed iterator", func() { - Test := func(n ...int) func() { - if len(n) == 0 { - rnd := testutil.NewRand() - n = make([]int, rnd.Intn(17)+3) - for i := range n { - n[i] = rnd.Intn(19) + 1 - } - } - - return func() { - It("Should iterates and seeks correctly", func(done Done) { - // Build key/value. - index := make(keyValueIndex, len(n)) - sum := 0 - for _, x := range n { - sum += x - } - kv := testutil.KeyValue_Generate(nil, sum, 1, 1, 10, 4, 4) - for i, j := 0, 0; i < len(n); i++ { - for x := n[i]; x > 0; x-- { - key, value := kv.Index(j) - index[i].key = key - index[i].Put(key, value) - j++ - } - } - - // Test the iterator. - t := testutil.IteratorTesting{ - KeyValue: kv.Clone(), - Iter: NewIndexedIterator(NewArrayIndexer(index), true), - } - testutil.DoIteratorTesting(&t) - done <- true - }, 15.0) - } - } - - Describe("with 100 keys", Test(100)) - Describe("with 50-50 keys", Test(50, 50)) - Describe("with 50-1 keys", Test(50, 1)) - Describe("with 50-1-50 keys", Test(50, 1, 50)) - Describe("with 1-50 keys", Test(1, 50)) - Describe("with random N-keys", Test()) - }) -}) diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/iterator/iter.go b/vendor/github.com/syndtr/goleveldb/leveldb/iterator/iter.go deleted file mode 100644 index 3b55532..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/iterator/iter.go +++ /dev/null @@ -1,132 +0,0 @@ -// Copyright (c) 2012, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -// Package iterator provides interface and implementation to traverse over -// contents of a database. -package iterator - -import ( - "errors" - - "github.com/syndtr/goleveldb/leveldb/util" -) - -var ( - ErrIterReleased = errors.New("leveldb/iterator: iterator released") -) - -// IteratorSeeker is the interface that wraps the 'seeks method'. -type IteratorSeeker interface { - // First moves the iterator to the first key/value pair. If the iterator - // only contains one key/value pair then First and Last would moves - // to the same key/value pair. - // It returns whether such pair exist. - First() bool - - // Last moves the iterator to the last key/value pair. If the iterator - // only contains one key/value pair then First and Last would moves - // to the same key/value pair. - // It returns whether such pair exist. - Last() bool - - // Seek moves the iterator to the first key/value pair whose key is greater - // than or equal to the given key. - // It returns whether such pair exist. - // - // It is safe to modify the contents of the argument after Seek returns. - Seek(key []byte) bool - - // Next moves the iterator to the next key/value pair. - // It returns whether the iterator is exhausted. - Next() bool - - // Prev moves the iterator to the previous key/value pair. - // It returns whether the iterator is exhausted. - Prev() bool -} - -// CommonIterator is the interface that wraps common iterator methods. -type CommonIterator interface { - IteratorSeeker - - // util.Releaser is the interface that wraps basic Release method. - // When called Release will releases any resources associated with the - // iterator. - util.Releaser - - // util.ReleaseSetter is the interface that wraps the basic SetReleaser - // method. - util.ReleaseSetter - - // TODO: Remove this when ready. - Valid() bool - - // Error returns any accumulated error. Exhausting all the key/value pairs - // is not considered to be an error. - Error() error -} - -// Iterator iterates over a DB's key/value pairs in key order. -// -// When encounter an error any 'seeks method' will return false and will -// yield no key/value pairs. The error can be queried by calling the Error -// method. Calling Release is still necessary. -// -// An iterator must be released after use, but it is not necessary to read -// an iterator until exhaustion. -// Also, an iterator is not necessarily safe for concurrent use, but it is -// safe to use multiple iterators concurrently, with each in a dedicated -// goroutine. -type Iterator interface { - CommonIterator - - // Key returns the key of the current key/value pair, or nil if done. - // The caller should not modify the contents of the returned slice, and - // its contents may change on the next call to any 'seeks method'. - Key() []byte - - // Value returns the key of the current key/value pair, or nil if done. - // The caller should not modify the contents of the returned slice, and - // its contents may change on the next call to any 'seeks method'. - Value() []byte -} - -// ErrorCallbackSetter is the interface that wraps basic SetErrorCallback -// method. -// -// ErrorCallbackSetter implemented by indexed and merged iterator. -type ErrorCallbackSetter interface { - // SetErrorCallback allows set an error callback of the corresponding - // iterator. Use nil to clear the callback. - SetErrorCallback(f func(err error)) -} - -type emptyIterator struct { - util.BasicReleaser - err error -} - -func (i *emptyIterator) rErr() { - if i.err == nil && i.Released() { - i.err = ErrIterReleased - } -} - -func (*emptyIterator) Valid() bool { return false } -func (i *emptyIterator) First() bool { i.rErr(); return false } -func (i *emptyIterator) Last() bool { i.rErr(); return false } -func (i *emptyIterator) Seek(key []byte) bool { i.rErr(); return false } -func (i *emptyIterator) Next() bool { i.rErr(); return false } -func (i *emptyIterator) Prev() bool { i.rErr(); return false } -func (*emptyIterator) Key() []byte { return nil } -func (*emptyIterator) Value() []byte { return nil } -func (i *emptyIterator) Error() error { return i.err } - -// NewEmptyIterator creates an empty iterator. The err parameter can be -// nil, but if not nil the given err will be returned by Error method. -func NewEmptyIterator(err error) Iterator { - return &emptyIterator{err: err} -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/iterator/iter_suite_test.go b/vendor/github.com/syndtr/goleveldb/leveldb/iterator/iter_suite_test.go deleted file mode 100644 index 5ef8d5b..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/iterator/iter_suite_test.go +++ /dev/null @@ -1,11 +0,0 @@ -package iterator_test - -import ( - "testing" - - "github.com/syndtr/goleveldb/leveldb/testutil" -) - -func TestIterator(t *testing.T) { - testutil.RunSuite(t, "Iterator Suite") -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/iterator/merged_iter.go b/vendor/github.com/syndtr/goleveldb/leveldb/iterator/merged_iter.go deleted file mode 100644 index 1a7e29d..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/iterator/merged_iter.go +++ /dev/null @@ -1,304 +0,0 @@ -// Copyright (c) 2012, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package iterator - -import ( - "github.com/syndtr/goleveldb/leveldb/comparer" - "github.com/syndtr/goleveldb/leveldb/errors" - "github.com/syndtr/goleveldb/leveldb/util" -) - -type dir int - -const ( - dirReleased dir = iota - 1 - dirSOI - dirEOI - dirBackward - dirForward -) - -type mergedIterator struct { - cmp comparer.Comparer - iters []Iterator - strict bool - - keys [][]byte - index int - dir dir - err error - errf func(err error) - releaser util.Releaser -} - -func assertKey(key []byte) []byte { - if key == nil { - panic("leveldb/iterator: nil key") - } - return key -} - -func (i *mergedIterator) iterErr(iter Iterator) bool { - if err := iter.Error(); err != nil { - if i.errf != nil { - i.errf(err) - } - if i.strict || !errors.IsCorrupted(err) { - i.err = err - return true - } - } - return false -} - -func (i *mergedIterator) Valid() bool { - return i.err == nil && i.dir > dirEOI -} - -func (i *mergedIterator) First() bool { - if i.err != nil { - return false - } else if i.dir == dirReleased { - i.err = ErrIterReleased - return false - } - - for x, iter := range i.iters { - switch { - case iter.First(): - i.keys[x] = assertKey(iter.Key()) - case i.iterErr(iter): - return false - default: - i.keys[x] = nil - } - } - i.dir = dirSOI - return i.next() -} - -func (i *mergedIterator) Last() bool { - if i.err != nil { - return false - } else if i.dir == dirReleased { - i.err = ErrIterReleased - return false - } - - for x, iter := range i.iters { - switch { - case iter.Last(): - i.keys[x] = assertKey(iter.Key()) - case i.iterErr(iter): - return false - default: - i.keys[x] = nil - } - } - i.dir = dirEOI - return i.prev() -} - -func (i *mergedIterator) Seek(key []byte) bool { - if i.err != nil { - return false - } else if i.dir == dirReleased { - i.err = ErrIterReleased - return false - } - - for x, iter := range i.iters { - switch { - case iter.Seek(key): - i.keys[x] = assertKey(iter.Key()) - case i.iterErr(iter): - return false - default: - i.keys[x] = nil - } - } - i.dir = dirSOI - return i.next() -} - -func (i *mergedIterator) next() bool { - var key []byte - if i.dir == dirForward { - key = i.keys[i.index] - } - for x, tkey := range i.keys { - if tkey != nil && (key == nil || i.cmp.Compare(tkey, key) < 0) { - key = tkey - i.index = x - } - } - if key == nil { - i.dir = dirEOI - return false - } - i.dir = dirForward - return true -} - -func (i *mergedIterator) Next() bool { - if i.dir == dirEOI || i.err != nil { - return false - } else if i.dir == dirReleased { - i.err = ErrIterReleased - return false - } - - switch i.dir { - case dirSOI: - return i.First() - case dirBackward: - key := append([]byte{}, i.keys[i.index]...) - if !i.Seek(key) { - return false - } - return i.Next() - } - - x := i.index - iter := i.iters[x] - switch { - case iter.Next(): - i.keys[x] = assertKey(iter.Key()) - case i.iterErr(iter): - return false - default: - i.keys[x] = nil - } - return i.next() -} - -func (i *mergedIterator) prev() bool { - var key []byte - if i.dir == dirBackward { - key = i.keys[i.index] - } - for x, tkey := range i.keys { - if tkey != nil && (key == nil || i.cmp.Compare(tkey, key) > 0) { - key = tkey - i.index = x - } - } - if key == nil { - i.dir = dirSOI - return false - } - i.dir = dirBackward - return true -} - -func (i *mergedIterator) Prev() bool { - if i.dir == dirSOI || i.err != nil { - return false - } else if i.dir == dirReleased { - i.err = ErrIterReleased - return false - } - - switch i.dir { - case dirEOI: - return i.Last() - case dirForward: - key := append([]byte{}, i.keys[i.index]...) - for x, iter := range i.iters { - if x == i.index { - continue - } - seek := iter.Seek(key) - switch { - case seek && iter.Prev(), !seek && iter.Last(): - i.keys[x] = assertKey(iter.Key()) - case i.iterErr(iter): - return false - default: - i.keys[x] = nil - } - } - } - - x := i.index - iter := i.iters[x] - switch { - case iter.Prev(): - i.keys[x] = assertKey(iter.Key()) - case i.iterErr(iter): - return false - default: - i.keys[x] = nil - } - return i.prev() -} - -func (i *mergedIterator) Key() []byte { - if i.err != nil || i.dir <= dirEOI { - return nil - } - return i.keys[i.index] -} - -func (i *mergedIterator) Value() []byte { - if i.err != nil || i.dir <= dirEOI { - return nil - } - return i.iters[i.index].Value() -} - -func (i *mergedIterator) Release() { - if i.dir != dirReleased { - i.dir = dirReleased - for _, iter := range i.iters { - iter.Release() - } - i.iters = nil - i.keys = nil - if i.releaser != nil { - i.releaser.Release() - i.releaser = nil - } - } -} - -func (i *mergedIterator) SetReleaser(releaser util.Releaser) { - if i.dir == dirReleased { - panic(util.ErrReleased) - } - if i.releaser != nil && releaser != nil { - panic(util.ErrHasReleaser) - } - i.releaser = releaser -} - -func (i *mergedIterator) Error() error { - return i.err -} - -func (i *mergedIterator) SetErrorCallback(f func(err error)) { - i.errf = f -} - -// NewMergedIterator returns an iterator that merges its input. Walking the -// resultant iterator will return all key/value pairs of all input iterators -// in strictly increasing key order, as defined by cmp. -// The input's key ranges may overlap, but there are assumed to be no duplicate -// keys: if iters[i] contains a key k then iters[j] will not contain that key k. -// None of the iters may be nil. -// -// If strict is true the any 'corruption errors' (i.e errors.IsCorrupted(err) == true) -// won't be ignored and will halt 'merged iterator', otherwise the iterator will -// continue to the next 'input iterator'. -func NewMergedIterator(iters []Iterator, cmp comparer.Comparer, strict bool) Iterator { - return &mergedIterator{ - iters: iters, - cmp: cmp, - strict: strict, - keys: make([][]byte, len(iters)), - } -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/iterator/merged_iter_test.go b/vendor/github.com/syndtr/goleveldb/leveldb/iterator/merged_iter_test.go deleted file mode 100644 index ee40881..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/iterator/merged_iter_test.go +++ /dev/null @@ -1,60 +0,0 @@ -// Copyright (c) 2014, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package iterator_test - -import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - - "github.com/syndtr/goleveldb/leveldb/comparer" - . "github.com/syndtr/goleveldb/leveldb/iterator" - "github.com/syndtr/goleveldb/leveldb/testutil" -) - -var _ = testutil.Defer(func() { - Describe("Merged iterator", func() { - Test := func(filled int, empty int) func() { - return func() { - It("Should iterates and seeks correctly", func(done Done) { - rnd := testutil.NewRand() - - // Build key/value. - filledKV := make([]testutil.KeyValue, filled) - kv := testutil.KeyValue_Generate(nil, 100, 1, 1, 10, 4, 4) - kv.Iterate(func(i int, key, value []byte) { - filledKV[rnd.Intn(filled)].Put(key, value) - }) - - // Create itearators. - iters := make([]Iterator, filled+empty) - for i := range iters { - if empty == 0 || (rnd.Int()%2 == 0 && filled > 0) { - filled-- - Expect(filledKV[filled].Len()).ShouldNot(BeZero()) - iters[i] = NewArrayIterator(filledKV[filled]) - } else { - empty-- - iters[i] = NewEmptyIterator(nil) - } - } - - // Test the iterator. - t := testutil.IteratorTesting{ - KeyValue: kv.Clone(), - Iter: NewMergedIterator(iters, comparer.DefaultComparer, true), - } - testutil.DoIteratorTesting(&t) - done <- true - }, 15.0) - } - } - - Describe("with three, all filled iterators", Test(3, 0)) - Describe("with one filled, one empty iterators", Test(1, 1)) - Describe("with one filled, two empty iterators", Test(1, 2)) - }) -}) diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/journal/journal.go b/vendor/github.com/syndtr/goleveldb/leveldb/journal/journal.go deleted file mode 100644 index d094c3d..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/journal/journal.go +++ /dev/null @@ -1,524 +0,0 @@ -// Copyright 2011 The LevelDB-Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Taken from: https://code.google.com/p/leveldb-go/source/browse/leveldb/record/record.go?r=1d5ccbe03246da926391ee12d1c6caae054ff4b0 -// License, authors and contributors informations can be found at bellow URLs respectively: -// https://code.google.com/p/leveldb-go/source/browse/LICENSE -// https://code.google.com/p/leveldb-go/source/browse/AUTHORS -// https://code.google.com/p/leveldb-go/source/browse/CONTRIBUTORS - -// Package journal reads and writes sequences of journals. Each journal is a stream -// of bytes that completes before the next journal starts. -// -// When reading, call Next to obtain an io.Reader for the next journal. Next will -// return io.EOF when there are no more journals. It is valid to call Next -// without reading the current journal to exhaustion. -// -// When writing, call Next to obtain an io.Writer for the next journal. Calling -// Next finishes the current journal. Call Close to finish the final journal. -// -// Optionally, call Flush to finish the current journal and flush the underlying -// writer without starting a new journal. To start a new journal after flushing, -// call Next. -// -// Neither Readers or Writers are safe to use concurrently. -// -// Example code: -// func read(r io.Reader) ([]string, error) { -// var ss []string -// journals := journal.NewReader(r, nil, true, true) -// for { -// j, err := journals.Next() -// if err == io.EOF { -// break -// } -// if err != nil { -// return nil, err -// } -// s, err := ioutil.ReadAll(j) -// if err != nil { -// return nil, err -// } -// ss = append(ss, string(s)) -// } -// return ss, nil -// } -// -// func write(w io.Writer, ss []string) error { -// journals := journal.NewWriter(w) -// for _, s := range ss { -// j, err := journals.Next() -// if err != nil { -// return err -// } -// if _, err := j.Write([]byte(s)), err != nil { -// return err -// } -// } -// return journals.Close() -// } -// -// The wire format is that the stream is divided into 32KiB blocks, and each -// block contains a number of tightly packed chunks. Chunks cannot cross block -// boundaries. The last block may be shorter than 32 KiB. Any unused bytes in a -// block must be zero. -// -// A journal maps to one or more chunks. Each chunk has a 7 byte header (a 4 -// byte checksum, a 2 byte little-endian uint16 length, and a 1 byte chunk type) -// followed by a payload. The checksum is over the chunk type and the payload. -// -// There are four chunk types: whether the chunk is the full journal, or the -// first, middle or last chunk of a multi-chunk journal. A multi-chunk journal -// has one first chunk, zero or more middle chunks, and one last chunk. -// -// The wire format allows for limited recovery in the face of data corruption: -// on a format error (such as a checksum mismatch), the reader moves to the -// next block and looks for the next full or first chunk. -package journal - -import ( - "encoding/binary" - "fmt" - "io" - - "github.com/syndtr/goleveldb/leveldb/errors" - "github.com/syndtr/goleveldb/leveldb/storage" - "github.com/syndtr/goleveldb/leveldb/util" -) - -// These constants are part of the wire format and should not be changed. -const ( - fullChunkType = 1 - firstChunkType = 2 - middleChunkType = 3 - lastChunkType = 4 -) - -const ( - blockSize = 32 * 1024 - headerSize = 7 -) - -type flusher interface { - Flush() error -} - -// ErrCorrupted is the error type that generated by corrupted block or chunk. -type ErrCorrupted struct { - Size int - Reason string -} - -func (e *ErrCorrupted) Error() string { - return fmt.Sprintf("leveldb/journal: block/chunk corrupted: %s (%d bytes)", e.Reason, e.Size) -} - -// Dropper is the interface that wrap simple Drop method. The Drop -// method will be called when the journal reader dropping a block or chunk. -type Dropper interface { - Drop(err error) -} - -// Reader reads journals from an underlying io.Reader. -type Reader struct { - // r is the underlying reader. - r io.Reader - // the dropper. - dropper Dropper - // strict flag. - strict bool - // checksum flag. - checksum bool - // seq is the sequence number of the current journal. - seq int - // buf[i:j] is the unread portion of the current chunk's payload. - // The low bound, i, excludes the chunk header. - i, j int - // n is the number of bytes of buf that are valid. Once reading has started, - // only the final block can have n < blockSize. - n int - // last is whether the current chunk is the last chunk of the journal. - last bool - // err is any accumulated error. - err error - // buf is the buffer. - buf [blockSize]byte -} - -// NewReader returns a new reader. The dropper may be nil, and if -// strict is true then corrupted or invalid chunk will halt the journal -// reader entirely. -func NewReader(r io.Reader, dropper Dropper, strict, checksum bool) *Reader { - return &Reader{ - r: r, - dropper: dropper, - strict: strict, - checksum: checksum, - last: true, - } -} - -var errSkip = errors.New("leveldb/journal: skipped") - -func (r *Reader) corrupt(n int, reason string, skip bool) error { - if r.dropper != nil { - r.dropper.Drop(&ErrCorrupted{n, reason}) - } - if r.strict && !skip { - r.err = errors.NewErrCorrupted(storage.FileDesc{}, &ErrCorrupted{n, reason}) - return r.err - } - return errSkip -} - -// nextChunk sets r.buf[r.i:r.j] to hold the next chunk's payload, reading the -// next block into the buffer if necessary. -func (r *Reader) nextChunk(first bool) error { - for { - if r.j+headerSize <= r.n { - checksum := binary.LittleEndian.Uint32(r.buf[r.j+0 : r.j+4]) - length := binary.LittleEndian.Uint16(r.buf[r.j+4 : r.j+6]) - chunkType := r.buf[r.j+6] - unprocBlock := r.n - r.j - if checksum == 0 && length == 0 && chunkType == 0 { - // Drop entire block. - r.i = r.n - r.j = r.n - return r.corrupt(unprocBlock, "zero header", false) - } - if chunkType < fullChunkType || chunkType > lastChunkType { - // Drop entire block. - r.i = r.n - r.j = r.n - return r.corrupt(unprocBlock, fmt.Sprintf("invalid chunk type %#x", chunkType), false) - } - r.i = r.j + headerSize - r.j = r.j + headerSize + int(length) - if r.j > r.n { - // Drop entire block. - r.i = r.n - r.j = r.n - return r.corrupt(unprocBlock, "chunk length overflows block", false) - } else if r.checksum && checksum != util.NewCRC(r.buf[r.i-1:r.j]).Value() { - // Drop entire block. - r.i = r.n - r.j = r.n - return r.corrupt(unprocBlock, "checksum mismatch", false) - } - if first && chunkType != fullChunkType && chunkType != firstChunkType { - chunkLength := (r.j - r.i) + headerSize - r.i = r.j - // Report the error, but skip it. - return r.corrupt(chunkLength, "orphan chunk", true) - } - r.last = chunkType == fullChunkType || chunkType == lastChunkType - return nil - } - - // The last block. - if r.n < blockSize && r.n > 0 { - if !first { - return r.corrupt(0, "missing chunk part", false) - } - r.err = io.EOF - return r.err - } - - // Read block. - n, err := io.ReadFull(r.r, r.buf[:]) - if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF { - return err - } - if n == 0 { - if !first { - return r.corrupt(0, "missing chunk part", false) - } - r.err = io.EOF - return r.err - } - r.i, r.j, r.n = 0, 0, n - } -} - -// Next returns a reader for the next journal. It returns io.EOF if there are no -// more journals. The reader returned becomes stale after the next Next call, -// and should no longer be used. If strict is false, the reader will returns -// io.ErrUnexpectedEOF error when found corrupted journal. -func (r *Reader) Next() (io.Reader, error) { - r.seq++ - if r.err != nil { - return nil, r.err - } - r.i = r.j - for { - if err := r.nextChunk(true); err == nil { - break - } else if err != errSkip { - return nil, err - } - } - return &singleReader{r, r.seq, nil}, nil -} - -// Reset resets the journal reader, allows reuse of the journal reader. Reset returns -// last accumulated error. -func (r *Reader) Reset(reader io.Reader, dropper Dropper, strict, checksum bool) error { - r.seq++ - err := r.err - r.r = reader - r.dropper = dropper - r.strict = strict - r.checksum = checksum - r.i = 0 - r.j = 0 - r.n = 0 - r.last = true - r.err = nil - return err -} - -type singleReader struct { - r *Reader - seq int - err error -} - -func (x *singleReader) Read(p []byte) (int, error) { - r := x.r - if r.seq != x.seq { - return 0, errors.New("leveldb/journal: stale reader") - } - if x.err != nil { - return 0, x.err - } - if r.err != nil { - return 0, r.err - } - for r.i == r.j { - if r.last { - return 0, io.EOF - } - x.err = r.nextChunk(false) - if x.err != nil { - if x.err == errSkip { - x.err = io.ErrUnexpectedEOF - } - return 0, x.err - } - } - n := copy(p, r.buf[r.i:r.j]) - r.i += n - return n, nil -} - -func (x *singleReader) ReadByte() (byte, error) { - r := x.r - if r.seq != x.seq { - return 0, errors.New("leveldb/journal: stale reader") - } - if x.err != nil { - return 0, x.err - } - if r.err != nil { - return 0, r.err - } - for r.i == r.j { - if r.last { - return 0, io.EOF - } - x.err = r.nextChunk(false) - if x.err != nil { - if x.err == errSkip { - x.err = io.ErrUnexpectedEOF - } - return 0, x.err - } - } - c := r.buf[r.i] - r.i++ - return c, nil -} - -// Writer writes journals to an underlying io.Writer. -type Writer struct { - // w is the underlying writer. - w io.Writer - // seq is the sequence number of the current journal. - seq int - // f is w as a flusher. - f flusher - // buf[i:j] is the bytes that will become the current chunk. - // The low bound, i, includes the chunk header. - i, j int - // buf[:written] has already been written to w. - // written is zero unless Flush has been called. - written int - // first is whether the current chunk is the first chunk of the journal. - first bool - // pending is whether a chunk is buffered but not yet written. - pending bool - // err is any accumulated error. - err error - // buf is the buffer. - buf [blockSize]byte -} - -// NewWriter returns a new Writer. -func NewWriter(w io.Writer) *Writer { - f, _ := w.(flusher) - return &Writer{ - w: w, - f: f, - } -} - -// fillHeader fills in the header for the pending chunk. -func (w *Writer) fillHeader(last bool) { - if w.i+headerSize > w.j || w.j > blockSize { - panic("leveldb/journal: bad writer state") - } - if last { - if w.first { - w.buf[w.i+6] = fullChunkType - } else { - w.buf[w.i+6] = lastChunkType - } - } else { - if w.first { - w.buf[w.i+6] = firstChunkType - } else { - w.buf[w.i+6] = middleChunkType - } - } - binary.LittleEndian.PutUint32(w.buf[w.i+0:w.i+4], util.NewCRC(w.buf[w.i+6:w.j]).Value()) - binary.LittleEndian.PutUint16(w.buf[w.i+4:w.i+6], uint16(w.j-w.i-headerSize)) -} - -// writeBlock writes the buffered block to the underlying writer, and reserves -// space for the next chunk's header. -func (w *Writer) writeBlock() { - _, w.err = w.w.Write(w.buf[w.written:]) - w.i = 0 - w.j = headerSize - w.written = 0 -} - -// writePending finishes the current journal and writes the buffer to the -// underlying writer. -func (w *Writer) writePending() { - if w.err != nil { - return - } - if w.pending { - w.fillHeader(true) - w.pending = false - } - _, w.err = w.w.Write(w.buf[w.written:w.j]) - w.written = w.j -} - -// Close finishes the current journal and closes the writer. -func (w *Writer) Close() error { - w.seq++ - w.writePending() - if w.err != nil { - return w.err - } - w.err = errors.New("leveldb/journal: closed Writer") - return nil -} - -// Flush finishes the current journal, writes to the underlying writer, and -// flushes it if that writer implements interface{ Flush() error }. -func (w *Writer) Flush() error { - w.seq++ - w.writePending() - if w.err != nil { - return w.err - } - if w.f != nil { - w.err = w.f.Flush() - return w.err - } - return nil -} - -// Reset resets the journal writer, allows reuse of the journal writer. Reset -// will also closes the journal writer if not already. -func (w *Writer) Reset(writer io.Writer) (err error) { - w.seq++ - if w.err == nil { - w.writePending() - err = w.err - } - w.w = writer - w.f, _ = writer.(flusher) - w.i = 0 - w.j = 0 - w.written = 0 - w.first = false - w.pending = false - w.err = nil - return -} - -// Next returns a writer for the next journal. The writer returned becomes stale -// after the next Close, Flush or Next call, and should no longer be used. -func (w *Writer) Next() (io.Writer, error) { - w.seq++ - if w.err != nil { - return nil, w.err - } - if w.pending { - w.fillHeader(true) - } - w.i = w.j - w.j = w.j + headerSize - // Check if there is room in the block for the header. - if w.j > blockSize { - // Fill in the rest of the block with zeroes. - for k := w.i; k < blockSize; k++ { - w.buf[k] = 0 - } - w.writeBlock() - if w.err != nil { - return nil, w.err - } - } - w.first = true - w.pending = true - return singleWriter{w, w.seq}, nil -} - -type singleWriter struct { - w *Writer - seq int -} - -func (x singleWriter) Write(p []byte) (int, error) { - w := x.w - if w.seq != x.seq { - return 0, errors.New("leveldb/journal: stale writer") - } - if w.err != nil { - return 0, w.err - } - n0 := len(p) - for len(p) > 0 { - // Write a block, if it is full. - if w.j == blockSize { - w.fillHeader(false) - w.writeBlock() - if w.err != nil { - return 0, w.err - } - w.first = false - } - // Copy bytes into the buffer. - n := copy(w.buf[w.j:], p) - w.j += n - p = p[n:] - } - return n0, nil -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/journal/journal_test.go b/vendor/github.com/syndtr/goleveldb/leveldb/journal/journal_test.go deleted file mode 100644 index 0fcf225..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/journal/journal_test.go +++ /dev/null @@ -1,818 +0,0 @@ -// Copyright 2011 The LevelDB-Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -// Taken from: https://code.google.com/p/leveldb-go/source/browse/leveldb/record/record_test.go?r=df1fa28f7f3be6c3935548169002309c12967135 -// License, authors and contributors informations can be found at bellow URLs respectively: -// https://code.google.com/p/leveldb-go/source/browse/LICENSE -// https://code.google.com/p/leveldb-go/source/browse/AUTHORS -// https://code.google.com/p/leveldb-go/source/browse/CONTRIBUTORS - -package journal - -import ( - "bytes" - "encoding/binary" - "fmt" - "io" - "io/ioutil" - "math/rand" - "strings" - "testing" -) - -type dropper struct { - t *testing.T -} - -func (d dropper) Drop(err error) { - d.t.Log(err) -} - -func short(s string) string { - if len(s) < 64 { - return s - } - return fmt.Sprintf("%s...(skipping %d bytes)...%s", s[:20], len(s)-40, s[len(s)-20:]) -} - -// big returns a string of length n, composed of repetitions of partial. -func big(partial string, n int) string { - return strings.Repeat(partial, n/len(partial)+1)[:n] -} - -func TestEmpty(t *testing.T) { - buf := new(bytes.Buffer) - r := NewReader(buf, dropper{t}, true, true) - if _, err := r.Next(); err != io.EOF { - t.Fatalf("got %v, want %v", err, io.EOF) - } -} - -func testGenerator(t *testing.T, reset func(), gen func() (string, bool)) { - buf := new(bytes.Buffer) - - reset() - w := NewWriter(buf) - for { - s, ok := gen() - if !ok { - break - } - ww, err := w.Next() - if err != nil { - t.Fatal(err) - } - if _, err := ww.Write([]byte(s)); err != nil { - t.Fatal(err) - } - } - if err := w.Close(); err != nil { - t.Fatal(err) - } - - reset() - r := NewReader(buf, dropper{t}, true, true) - for { - s, ok := gen() - if !ok { - break - } - rr, err := r.Next() - if err != nil { - t.Fatal(err) - } - x, err := ioutil.ReadAll(rr) - if err != nil { - t.Fatal(err) - } - if string(x) != s { - t.Fatalf("got %q, want %q", short(string(x)), short(s)) - } - } - if _, err := r.Next(); err != io.EOF { - t.Fatalf("got %v, want %v", err, io.EOF) - } -} - -func testLiterals(t *testing.T, s []string) { - var i int - reset := func() { - i = 0 - } - gen := func() (string, bool) { - if i == len(s) { - return "", false - } - i++ - return s[i-1], true - } - testGenerator(t, reset, gen) -} - -func TestMany(t *testing.T) { - const n = 1e5 - var i int - reset := func() { - i = 0 - } - gen := func() (string, bool) { - if i == n { - return "", false - } - i++ - return fmt.Sprintf("%d.", i-1), true - } - testGenerator(t, reset, gen) -} - -func TestRandom(t *testing.T) { - const n = 1e2 - var ( - i int - r *rand.Rand - ) - reset := func() { - i, r = 0, rand.New(rand.NewSource(0)) - } - gen := func() (string, bool) { - if i == n { - return "", false - } - i++ - return strings.Repeat(string(uint8(i)), r.Intn(2*blockSize+16)), true - } - testGenerator(t, reset, gen) -} - -func TestBasic(t *testing.T) { - testLiterals(t, []string{ - strings.Repeat("a", 1000), - strings.Repeat("b", 97270), - strings.Repeat("c", 8000), - }) -} - -func TestBoundary(t *testing.T) { - for i := blockSize - 16; i < blockSize+16; i++ { - s0 := big("abcd", i) - for j := blockSize - 16; j < blockSize+16; j++ { - s1 := big("ABCDE", j) - testLiterals(t, []string{s0, s1}) - testLiterals(t, []string{s0, "", s1}) - testLiterals(t, []string{s0, "x", s1}) - } - } -} - -func TestFlush(t *testing.T) { - buf := new(bytes.Buffer) - w := NewWriter(buf) - // Write a couple of records. Everything should still be held - // in the record.Writer buffer, so that buf.Len should be 0. - w0, _ := w.Next() - w0.Write([]byte("0")) - w1, _ := w.Next() - w1.Write([]byte("11")) - if got, want := buf.Len(), 0; got != want { - t.Fatalf("buffer length #0: got %d want %d", got, want) - } - // Flush the record.Writer buffer, which should yield 17 bytes. - // 17 = 2*7 + 1 + 2, which is two headers and 1 + 2 payload bytes. - if err := w.Flush(); err != nil { - t.Fatal(err) - } - if got, want := buf.Len(), 17; got != want { - t.Fatalf("buffer length #1: got %d want %d", got, want) - } - // Do another write, one that isn't large enough to complete the block. - // The write should not have flowed through to buf. - w2, _ := w.Next() - w2.Write(bytes.Repeat([]byte("2"), 10000)) - if got, want := buf.Len(), 17; got != want { - t.Fatalf("buffer length #2: got %d want %d", got, want) - } - // Flushing should get us up to 10024 bytes written. - // 10024 = 17 + 7 + 10000. - if err := w.Flush(); err != nil { - t.Fatal(err) - } - if got, want := buf.Len(), 10024; got != want { - t.Fatalf("buffer length #3: got %d want %d", got, want) - } - // Do a bigger write, one that completes the current block. - // We should now have 32768 bytes (a complete block), without - // an explicit flush. - w3, _ := w.Next() - w3.Write(bytes.Repeat([]byte("3"), 40000)) - if got, want := buf.Len(), 32768; got != want { - t.Fatalf("buffer length #4: got %d want %d", got, want) - } - // Flushing should get us up to 50038 bytes written. - // 50038 = 10024 + 2*7 + 40000. There are two headers because - // the one record was split into two chunks. - if err := w.Flush(); err != nil { - t.Fatal(err) - } - if got, want := buf.Len(), 50038; got != want { - t.Fatalf("buffer length #5: got %d want %d", got, want) - } - // Check that reading those records give the right lengths. - r := NewReader(buf, dropper{t}, true, true) - wants := []int64{1, 2, 10000, 40000} - for i, want := range wants { - rr, _ := r.Next() - n, err := io.Copy(ioutil.Discard, rr) - if err != nil { - t.Fatalf("read #%d: %v", i, err) - } - if n != want { - t.Fatalf("read #%d: got %d bytes want %d", i, n, want) - } - } -} - -func TestNonExhaustiveRead(t *testing.T) { - const n = 100 - buf := new(bytes.Buffer) - p := make([]byte, 10) - rnd := rand.New(rand.NewSource(1)) - - w := NewWriter(buf) - for i := 0; i < n; i++ { - length := len(p) + rnd.Intn(3*blockSize) - s := string(uint8(i)) + "123456789abcdefgh" - ww, _ := w.Next() - ww.Write([]byte(big(s, length))) - } - if err := w.Close(); err != nil { - t.Fatal(err) - } - - r := NewReader(buf, dropper{t}, true, true) - for i := 0; i < n; i++ { - rr, _ := r.Next() - _, err := io.ReadFull(rr, p) - if err != nil { - t.Fatal(err) - } - want := string(uint8(i)) + "123456789" - if got := string(p); got != want { - t.Fatalf("read #%d: got %q want %q", i, got, want) - } - } -} - -func TestStaleReader(t *testing.T) { - buf := new(bytes.Buffer) - - w := NewWriter(buf) - w0, err := w.Next() - if err != nil { - t.Fatal(err) - } - w0.Write([]byte("0")) - w1, err := w.Next() - if err != nil { - t.Fatal(err) - } - w1.Write([]byte("11")) - if err := w.Close(); err != nil { - t.Fatal(err) - } - - r := NewReader(buf, dropper{t}, true, true) - r0, err := r.Next() - if err != nil { - t.Fatal(err) - } - r1, err := r.Next() - if err != nil { - t.Fatal(err) - } - p := make([]byte, 1) - if _, err := r0.Read(p); err == nil || !strings.Contains(err.Error(), "stale") { - t.Fatalf("stale read #0: unexpected error: %v", err) - } - if _, err := r1.Read(p); err != nil { - t.Fatalf("fresh read #1: got %v want nil error", err) - } - if p[0] != '1' { - t.Fatalf("fresh read #1: byte contents: got '%c' want '1'", p[0]) - } -} - -func TestStaleWriter(t *testing.T) { - buf := new(bytes.Buffer) - - w := NewWriter(buf) - w0, err := w.Next() - if err != nil { - t.Fatal(err) - } - w1, err := w.Next() - if err != nil { - t.Fatal(err) - } - if _, err := w0.Write([]byte("0")); err == nil || !strings.Contains(err.Error(), "stale") { - t.Fatalf("stale write #0: unexpected error: %v", err) - } - if _, err := w1.Write([]byte("11")); err != nil { - t.Fatalf("fresh write #1: got %v want nil error", err) - } - if err := w.Flush(); err != nil { - t.Fatalf("flush: %v", err) - } - if _, err := w1.Write([]byte("0")); err == nil || !strings.Contains(err.Error(), "stale") { - t.Fatalf("stale write #1: unexpected error: %v", err) - } -} - -func TestCorrupt_MissingLastBlock(t *testing.T) { - buf := new(bytes.Buffer) - - w := NewWriter(buf) - - // First record. - ww, err := w.Next() - if err != nil { - t.Fatal(err) - } - if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize-1024)); err != nil { - t.Fatalf("write #0: unexpected error: %v", err) - } - - // Second record. - ww, err = w.Next() - if err != nil { - t.Fatal(err) - } - if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize-headerSize)); err != nil { - t.Fatalf("write #1: unexpected error: %v", err) - } - - if err := w.Close(); err != nil { - t.Fatal(err) - } - - // Cut the last block. - b := buf.Bytes()[:blockSize] - r := NewReader(bytes.NewReader(b), dropper{t}, false, true) - - // First read. - rr, err := r.Next() - if err != nil { - t.Fatal(err) - } - n, err := io.Copy(ioutil.Discard, rr) - if err != nil { - t.Fatalf("read #0: %v", err) - } - if n != blockSize-1024 { - t.Fatalf("read #0: got %d bytes want %d", n, blockSize-1024) - } - - // Second read. - rr, err = r.Next() - if err != nil { - t.Fatal(err) - } - n, err = io.Copy(ioutil.Discard, rr) - if err != io.ErrUnexpectedEOF { - t.Fatalf("read #1: unexpected error: %v", err) - } - - if _, err := r.Next(); err != io.EOF { - t.Fatalf("last next: unexpected error: %v", err) - } -} - -func TestCorrupt_CorruptedFirstBlock(t *testing.T) { - buf := new(bytes.Buffer) - - w := NewWriter(buf) - - // First record. - ww, err := w.Next() - if err != nil { - t.Fatal(err) - } - if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize/2)); err != nil { - t.Fatalf("write #0: unexpected error: %v", err) - } - - // Second record. - ww, err = w.Next() - if err != nil { - t.Fatal(err) - } - if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize-headerSize)); err != nil { - t.Fatalf("write #1: unexpected error: %v", err) - } - - // Third record. - ww, err = w.Next() - if err != nil { - t.Fatal(err) - } - if _, err := ww.Write(bytes.Repeat([]byte("0"), (blockSize-headerSize)+1)); err != nil { - t.Fatalf("write #2: unexpected error: %v", err) - } - - // Fourth record. - ww, err = w.Next() - if err != nil { - t.Fatal(err) - } - if _, err := ww.Write(bytes.Repeat([]byte("0"), (blockSize-headerSize)+2)); err != nil { - t.Fatalf("write #3: unexpected error: %v", err) - } - - if err := w.Close(); err != nil { - t.Fatal(err) - } - - b := buf.Bytes() - // Corrupting block #0. - for i := 0; i < 1024; i++ { - b[i] = '1' - } - - r := NewReader(bytes.NewReader(b), dropper{t}, false, true) - - // First read (third record). - rr, err := r.Next() - if err != nil { - t.Fatal(err) - } - n, err := io.Copy(ioutil.Discard, rr) - if err != nil { - t.Fatalf("read #0: %v", err) - } - if want := int64(blockSize-headerSize) + 1; n != want { - t.Fatalf("read #0: got %d bytes want %d", n, want) - } - - // Second read (fourth record). - rr, err = r.Next() - if err != nil { - t.Fatal(err) - } - n, err = io.Copy(ioutil.Discard, rr) - if err != nil { - t.Fatalf("read #1: %v", err) - } - if want := int64(blockSize-headerSize) + 2; n != want { - t.Fatalf("read #1: got %d bytes want %d", n, want) - } - - if _, err := r.Next(); err != io.EOF { - t.Fatalf("last next: unexpected error: %v", err) - } -} - -func TestCorrupt_CorruptedMiddleBlock(t *testing.T) { - buf := new(bytes.Buffer) - - w := NewWriter(buf) - - // First record. - ww, err := w.Next() - if err != nil { - t.Fatal(err) - } - if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize/2)); err != nil { - t.Fatalf("write #0: unexpected error: %v", err) - } - - // Second record. - ww, err = w.Next() - if err != nil { - t.Fatal(err) - } - if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize-headerSize)); err != nil { - t.Fatalf("write #1: unexpected error: %v", err) - } - - // Third record. - ww, err = w.Next() - if err != nil { - t.Fatal(err) - } - if _, err := ww.Write(bytes.Repeat([]byte("0"), (blockSize-headerSize)+1)); err != nil { - t.Fatalf("write #2: unexpected error: %v", err) - } - - // Fourth record. - ww, err = w.Next() - if err != nil { - t.Fatal(err) - } - if _, err := ww.Write(bytes.Repeat([]byte("0"), (blockSize-headerSize)+2)); err != nil { - t.Fatalf("write #3: unexpected error: %v", err) - } - - if err := w.Close(); err != nil { - t.Fatal(err) - } - - b := buf.Bytes() - // Corrupting block #1. - for i := 0; i < 1024; i++ { - b[blockSize+i] = '1' - } - - r := NewReader(bytes.NewReader(b), dropper{t}, false, true) - - // First read (first record). - rr, err := r.Next() - if err != nil { - t.Fatal(err) - } - n, err := io.Copy(ioutil.Discard, rr) - if err != nil { - t.Fatalf("read #0: %v", err) - } - if want := int64(blockSize / 2); n != want { - t.Fatalf("read #0: got %d bytes want %d", n, want) - } - - // Second read (second record). - rr, err = r.Next() - if err != nil { - t.Fatal(err) - } - n, err = io.Copy(ioutil.Discard, rr) - if err != io.ErrUnexpectedEOF { - t.Fatalf("read #1: unexpected error: %v", err) - } - - // Third read (fourth record). - rr, err = r.Next() - if err != nil { - t.Fatal(err) - } - n, err = io.Copy(ioutil.Discard, rr) - if err != nil { - t.Fatalf("read #2: %v", err) - } - if want := int64(blockSize-headerSize) + 2; n != want { - t.Fatalf("read #2: got %d bytes want %d", n, want) - } - - if _, err := r.Next(); err != io.EOF { - t.Fatalf("last next: unexpected error: %v", err) - } -} - -func TestCorrupt_CorruptedLastBlock(t *testing.T) { - buf := new(bytes.Buffer) - - w := NewWriter(buf) - - // First record. - ww, err := w.Next() - if err != nil { - t.Fatal(err) - } - if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize/2)); err != nil { - t.Fatalf("write #0: unexpected error: %v", err) - } - - // Second record. - ww, err = w.Next() - if err != nil { - t.Fatal(err) - } - if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize-headerSize)); err != nil { - t.Fatalf("write #1: unexpected error: %v", err) - } - - // Third record. - ww, err = w.Next() - if err != nil { - t.Fatal(err) - } - if _, err := ww.Write(bytes.Repeat([]byte("0"), (blockSize-headerSize)+1)); err != nil { - t.Fatalf("write #2: unexpected error: %v", err) - } - - // Fourth record. - ww, err = w.Next() - if err != nil { - t.Fatal(err) - } - if _, err := ww.Write(bytes.Repeat([]byte("0"), (blockSize-headerSize)+2)); err != nil { - t.Fatalf("write #3: unexpected error: %v", err) - } - - if err := w.Close(); err != nil { - t.Fatal(err) - } - - b := buf.Bytes() - // Corrupting block #3. - for i := len(b) - 1; i > len(b)-1024; i-- { - b[i] = '1' - } - - r := NewReader(bytes.NewReader(b), dropper{t}, false, true) - - // First read (first record). - rr, err := r.Next() - if err != nil { - t.Fatal(err) - } - n, err := io.Copy(ioutil.Discard, rr) - if err != nil { - t.Fatalf("read #0: %v", err) - } - if want := int64(blockSize / 2); n != want { - t.Fatalf("read #0: got %d bytes want %d", n, want) - } - - // Second read (second record). - rr, err = r.Next() - if err != nil { - t.Fatal(err) - } - n, err = io.Copy(ioutil.Discard, rr) - if err != nil { - t.Fatalf("read #1: %v", err) - } - if want := int64(blockSize - headerSize); n != want { - t.Fatalf("read #1: got %d bytes want %d", n, want) - } - - // Third read (third record). - rr, err = r.Next() - if err != nil { - t.Fatal(err) - } - n, err = io.Copy(ioutil.Discard, rr) - if err != nil { - t.Fatalf("read #2: %v", err) - } - if want := int64(blockSize-headerSize) + 1; n != want { - t.Fatalf("read #2: got %d bytes want %d", n, want) - } - - // Fourth read (fourth record). - rr, err = r.Next() - if err != nil { - t.Fatal(err) - } - n, err = io.Copy(ioutil.Discard, rr) - if err != io.ErrUnexpectedEOF { - t.Fatalf("read #3: unexpected error: %v", err) - } - - if _, err := r.Next(); err != io.EOF { - t.Fatalf("last next: unexpected error: %v", err) - } -} - -func TestCorrupt_FirstChuckLengthOverflow(t *testing.T) { - buf := new(bytes.Buffer) - - w := NewWriter(buf) - - // First record. - ww, err := w.Next() - if err != nil { - t.Fatal(err) - } - if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize/2)); err != nil { - t.Fatalf("write #0: unexpected error: %v", err) - } - - // Second record. - ww, err = w.Next() - if err != nil { - t.Fatal(err) - } - if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize-headerSize)); err != nil { - t.Fatalf("write #1: unexpected error: %v", err) - } - - // Third record. - ww, err = w.Next() - if err != nil { - t.Fatal(err) - } - if _, err := ww.Write(bytes.Repeat([]byte("0"), (blockSize-headerSize)+1)); err != nil { - t.Fatalf("write #2: unexpected error: %v", err) - } - - if err := w.Close(); err != nil { - t.Fatal(err) - } - - b := buf.Bytes() - // Corrupting record #1. - x := blockSize - binary.LittleEndian.PutUint16(b[x+4:], 0xffff) - - r := NewReader(bytes.NewReader(b), dropper{t}, false, true) - - // First read (first record). - rr, err := r.Next() - if err != nil { - t.Fatal(err) - } - n, err := io.Copy(ioutil.Discard, rr) - if err != nil { - t.Fatalf("read #0: %v", err) - } - if want := int64(blockSize / 2); n != want { - t.Fatalf("read #0: got %d bytes want %d", n, want) - } - - // Second read (second record). - rr, err = r.Next() - if err != nil { - t.Fatal(err) - } - n, err = io.Copy(ioutil.Discard, rr) - if err != io.ErrUnexpectedEOF { - t.Fatalf("read #1: unexpected error: %v", err) - } - - if _, err := r.Next(); err != io.EOF { - t.Fatalf("last next: unexpected error: %v", err) - } -} - -func TestCorrupt_MiddleChuckLengthOverflow(t *testing.T) { - buf := new(bytes.Buffer) - - w := NewWriter(buf) - - // First record. - ww, err := w.Next() - if err != nil { - t.Fatal(err) - } - if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize/2)); err != nil { - t.Fatalf("write #0: unexpected error: %v", err) - } - - // Second record. - ww, err = w.Next() - if err != nil { - t.Fatal(err) - } - if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize-headerSize)); err != nil { - t.Fatalf("write #1: unexpected error: %v", err) - } - - // Third record. - ww, err = w.Next() - if err != nil { - t.Fatal(err) - } - if _, err := ww.Write(bytes.Repeat([]byte("0"), (blockSize-headerSize)+1)); err != nil { - t.Fatalf("write #2: unexpected error: %v", err) - } - - if err := w.Close(); err != nil { - t.Fatal(err) - } - - b := buf.Bytes() - // Corrupting record #1. - x := blockSize/2 + headerSize - binary.LittleEndian.PutUint16(b[x+4:], 0xffff) - - r := NewReader(bytes.NewReader(b), dropper{t}, false, true) - - // First read (first record). - rr, err := r.Next() - if err != nil { - t.Fatal(err) - } - n, err := io.Copy(ioutil.Discard, rr) - if err != nil { - t.Fatalf("read #0: %v", err) - } - if want := int64(blockSize / 2); n != want { - t.Fatalf("read #0: got %d bytes want %d", n, want) - } - - // Second read (third record). - rr, err = r.Next() - if err != nil { - t.Fatal(err) - } - n, err = io.Copy(ioutil.Discard, rr) - if err != nil { - t.Fatalf("read #1: %v", err) - } - if want := int64(blockSize-headerSize) + 1; n != want { - t.Fatalf("read #1: got %d bytes want %d", n, want) - } - - if _, err := r.Next(); err != io.EOF { - t.Fatalf("last next: unexpected error: %v", err) - } -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/key.go b/vendor/github.com/syndtr/goleveldb/leveldb/key.go deleted file mode 100644 index ad8f51e..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/key.go +++ /dev/null @@ -1,143 +0,0 @@ -// Copyright (c) 2012, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package leveldb - -import ( - "encoding/binary" - "fmt" - - "github.com/syndtr/goleveldb/leveldb/errors" - "github.com/syndtr/goleveldb/leveldb/storage" -) - -// ErrInternalKeyCorrupted records internal key corruption. -type ErrInternalKeyCorrupted struct { - Ikey []byte - Reason string -} - -func (e *ErrInternalKeyCorrupted) Error() string { - return fmt.Sprintf("leveldb: internal key %q corrupted: %s", e.Ikey, e.Reason) -} - -func newErrInternalKeyCorrupted(ikey []byte, reason string) error { - return errors.NewErrCorrupted(storage.FileDesc{}, &ErrInternalKeyCorrupted{append([]byte{}, ikey...), reason}) -} - -type keyType uint - -func (kt keyType) String() string { - switch kt { - case keyTypeDel: - return "d" - case keyTypeVal: - return "v" - } - return fmt.Sprintf("", uint(kt)) -} - -// Value types encoded as the last component of internal keys. -// Don't modify; this value are saved to disk. -const ( - keyTypeDel = keyType(0) - keyTypeVal = keyType(1) -) - -// keyTypeSeek defines the keyType that should be passed when constructing an -// internal key for seeking to a particular sequence number (since we -// sort sequence numbers in decreasing order and the value type is -// embedded as the low 8 bits in the sequence number in internal keys, -// we need to use the highest-numbered ValueType, not the lowest). -const keyTypeSeek = keyTypeVal - -const ( - // Maximum value possible for sequence number; the 8-bits are - // used by value type, so its can packed together in single - // 64-bit integer. - keyMaxSeq = (uint64(1) << 56) - 1 - // Maximum value possible for packed sequence number and type. - keyMaxNum = (keyMaxSeq << 8) | uint64(keyTypeSeek) -) - -// Maximum number encoded in bytes. -var keyMaxNumBytes = make([]byte, 8) - -func init() { - binary.LittleEndian.PutUint64(keyMaxNumBytes, keyMaxNum) -} - -type internalKey []byte - -func makeInternalKey(dst, ukey []byte, seq uint64, kt keyType) internalKey { - if seq > keyMaxSeq { - panic("leveldb: invalid sequence number") - } else if kt > keyTypeVal { - panic("leveldb: invalid type") - } - - dst = ensureBuffer(dst, len(ukey)+8) - copy(dst, ukey) - binary.LittleEndian.PutUint64(dst[len(ukey):], (seq<<8)|uint64(kt)) - return internalKey(dst) -} - -func parseInternalKey(ik []byte) (ukey []byte, seq uint64, kt keyType, err error) { - if len(ik) < 8 { - return nil, 0, 0, newErrInternalKeyCorrupted(ik, "invalid length") - } - num := binary.LittleEndian.Uint64(ik[len(ik)-8:]) - seq, kt = uint64(num>>8), keyType(num&0xff) - if kt > keyTypeVal { - return nil, 0, 0, newErrInternalKeyCorrupted(ik, "invalid type") - } - ukey = ik[:len(ik)-8] - return -} - -func validInternalKey(ik []byte) bool { - _, _, _, err := parseInternalKey(ik) - return err == nil -} - -func (ik internalKey) assert() { - if ik == nil { - panic("leveldb: nil internalKey") - } - if len(ik) < 8 { - panic(fmt.Sprintf("leveldb: internal key %q, len=%d: invalid length", []byte(ik), len(ik))) - } -} - -func (ik internalKey) ukey() []byte { - ik.assert() - return ik[:len(ik)-8] -} - -func (ik internalKey) num() uint64 { - ik.assert() - return binary.LittleEndian.Uint64(ik[len(ik)-8:]) -} - -func (ik internalKey) parseNum() (seq uint64, kt keyType) { - num := ik.num() - seq, kt = uint64(num>>8), keyType(num&0xff) - if kt > keyTypeVal { - panic(fmt.Sprintf("leveldb: internal key %q, len=%d: invalid type %#x", []byte(ik), len(ik), kt)) - } - return -} - -func (ik internalKey) String() string { - if ik == nil { - return "" - } - - if ukey, seq, kt, err := parseInternalKey(ik); err == nil { - return fmt.Sprintf("%s,%s%d", shorten(string(ukey)), kt, seq) - } - return fmt.Sprintf("", []byte(ik)) -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/key_test.go b/vendor/github.com/syndtr/goleveldb/leveldb/key_test.go deleted file mode 100644 index 2f33ccb..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/key_test.go +++ /dev/null @@ -1,133 +0,0 @@ -// Copyright (c) 2012, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package leveldb - -import ( - "bytes" - "testing" - - "github.com/syndtr/goleveldb/leveldb/comparer" -) - -var defaultIComparer = &iComparer{comparer.DefaultComparer} - -func ikey(key string, seq uint64, kt keyType) internalKey { - return makeInternalKey(nil, []byte(key), uint64(seq), kt) -} - -func shortSep(a, b []byte) []byte { - dst := make([]byte, len(a)) - dst = defaultIComparer.Separator(dst[:0], a, b) - if dst == nil { - return a - } - return dst -} - -func shortSuccessor(b []byte) []byte { - dst := make([]byte, len(b)) - dst = defaultIComparer.Successor(dst[:0], b) - if dst == nil { - return b - } - return dst -} - -func testSingleKey(t *testing.T, key string, seq uint64, kt keyType) { - ik := ikey(key, seq, kt) - - if !bytes.Equal(ik.ukey(), []byte(key)) { - t.Errorf("user key does not equal, got %v, want %v", string(ik.ukey()), key) - } - - rseq, rt := ik.parseNum() - if rseq != seq { - t.Errorf("seq number does not equal, got %v, want %v", rseq, seq) - } - if rt != kt { - t.Errorf("type does not equal, got %v, want %v", rt, kt) - } - - if rukey, rseq, rt, kerr := parseInternalKey(ik); kerr == nil { - if !bytes.Equal(rukey, []byte(key)) { - t.Errorf("user key does not equal, got %v, want %v", string(ik.ukey()), key) - } - if rseq != seq { - t.Errorf("seq number does not equal, got %v, want %v", rseq, seq) - } - if rt != kt { - t.Errorf("type does not equal, got %v, want %v", rt, kt) - } - } else { - t.Errorf("key error: %v", kerr) - } -} - -func TestInternalKey_EncodeDecode(t *testing.T) { - keys := []string{"", "k", "hello", "longggggggggggggggggggggg"} - seqs := []uint64{ - 1, 2, 3, - (1 << 8) - 1, 1 << 8, (1 << 8) + 1, - (1 << 16) - 1, 1 << 16, (1 << 16) + 1, - (1 << 32) - 1, 1 << 32, (1 << 32) + 1, - } - for _, key := range keys { - for _, seq := range seqs { - testSingleKey(t, key, seq, keyTypeVal) - testSingleKey(t, "hello", 1, keyTypeDel) - } - } -} - -func assertBytes(t *testing.T, want, got []byte) { - if !bytes.Equal(got, want) { - t.Errorf("assert failed, got %v, want %v", got, want) - } -} - -func TestInternalKeyShortSeparator(t *testing.T) { - // When user keys are same - assertBytes(t, ikey("foo", 100, keyTypeVal), - shortSep(ikey("foo", 100, keyTypeVal), - ikey("foo", 99, keyTypeVal))) - assertBytes(t, ikey("foo", 100, keyTypeVal), - shortSep(ikey("foo", 100, keyTypeVal), - ikey("foo", 101, keyTypeVal))) - assertBytes(t, ikey("foo", 100, keyTypeVal), - shortSep(ikey("foo", 100, keyTypeVal), - ikey("foo", 100, keyTypeVal))) - assertBytes(t, ikey("foo", 100, keyTypeVal), - shortSep(ikey("foo", 100, keyTypeVal), - ikey("foo", 100, keyTypeDel))) - - // When user keys are misordered - assertBytes(t, ikey("foo", 100, keyTypeVal), - shortSep(ikey("foo", 100, keyTypeVal), - ikey("bar", 99, keyTypeVal))) - - // When user keys are different, but correctly ordered - assertBytes(t, ikey("g", uint64(keyMaxSeq), keyTypeSeek), - shortSep(ikey("foo", 100, keyTypeVal), - ikey("hello", 200, keyTypeVal))) - - // When start user key is prefix of limit user key - assertBytes(t, ikey("foo", 100, keyTypeVal), - shortSep(ikey("foo", 100, keyTypeVal), - ikey("foobar", 200, keyTypeVal))) - - // When limit user key is prefix of start user key - assertBytes(t, ikey("foobar", 100, keyTypeVal), - shortSep(ikey("foobar", 100, keyTypeVal), - ikey("foo", 200, keyTypeVal))) -} - -func TestInternalKeyShortestSuccessor(t *testing.T) { - assertBytes(t, ikey("g", uint64(keyMaxSeq), keyTypeSeek), - shortSuccessor(ikey("foo", 100, keyTypeVal))) - assertBytes(t, ikey("\xff\xff", 100, keyTypeVal), - shortSuccessor(ikey("\xff\xff", 100, keyTypeVal))) -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/leveldb_suite_test.go b/vendor/github.com/syndtr/goleveldb/leveldb/leveldb_suite_test.go deleted file mode 100644 index fefa007..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/leveldb_suite_test.go +++ /dev/null @@ -1,11 +0,0 @@ -package leveldb - -import ( - "testing" - - "github.com/syndtr/goleveldb/leveldb/testutil" -) - -func TestLevelDB(t *testing.T) { - testutil.RunSuite(t, "LevelDB Suite") -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/memdb/bench_test.go b/vendor/github.com/syndtr/goleveldb/leveldb/memdb/bench_test.go deleted file mode 100644 index b05084c..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/memdb/bench_test.go +++ /dev/null @@ -1,75 +0,0 @@ -// Copyright (c) 2012, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package memdb - -import ( - "encoding/binary" - "math/rand" - "testing" - - "github.com/syndtr/goleveldb/leveldb/comparer" -) - -func BenchmarkPut(b *testing.B) { - buf := make([][4]byte, b.N) - for i := range buf { - binary.LittleEndian.PutUint32(buf[i][:], uint32(i)) - } - - b.ResetTimer() - p := New(comparer.DefaultComparer, 0) - for i := range buf { - p.Put(buf[i][:], nil) - } -} - -func BenchmarkPutRandom(b *testing.B) { - buf := make([][4]byte, b.N) - for i := range buf { - binary.LittleEndian.PutUint32(buf[i][:], uint32(rand.Int())) - } - - b.ResetTimer() - p := New(comparer.DefaultComparer, 0) - for i := range buf { - p.Put(buf[i][:], nil) - } -} - -func BenchmarkGet(b *testing.B) { - buf := make([][4]byte, b.N) - for i := range buf { - binary.LittleEndian.PutUint32(buf[i][:], uint32(i)) - } - - p := New(comparer.DefaultComparer, 0) - for i := range buf { - p.Put(buf[i][:], nil) - } - - b.ResetTimer() - for i := range buf { - p.Get(buf[i][:]) - } -} - -func BenchmarkGetRandom(b *testing.B) { - buf := make([][4]byte, b.N) - for i := range buf { - binary.LittleEndian.PutUint32(buf[i][:], uint32(i)) - } - - p := New(comparer.DefaultComparer, 0) - for i := range buf { - p.Put(buf[i][:], nil) - } - - b.ResetTimer() - for i := 0; i < b.N; i++ { - p.Get(buf[rand.Int()%b.N][:]) - } -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/memdb/memdb.go b/vendor/github.com/syndtr/goleveldb/leveldb/memdb/memdb.go deleted file mode 100644 index 18a19ed..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/memdb/memdb.go +++ /dev/null @@ -1,475 +0,0 @@ -// Copyright (c) 2012, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -// Package memdb provides in-memory key/value database implementation. -package memdb - -import ( - "math/rand" - "sync" - - "github.com/syndtr/goleveldb/leveldb/comparer" - "github.com/syndtr/goleveldb/leveldb/errors" - "github.com/syndtr/goleveldb/leveldb/iterator" - "github.com/syndtr/goleveldb/leveldb/util" -) - -// Common errors. -var ( - ErrNotFound = errors.ErrNotFound - ErrIterReleased = errors.New("leveldb/memdb: iterator released") -) - -const tMaxHeight = 12 - -type dbIter struct { - util.BasicReleaser - p *DB - slice *util.Range - node int - forward bool - key, value []byte - err error -} - -func (i *dbIter) fill(checkStart, checkLimit bool) bool { - if i.node != 0 { - n := i.p.nodeData[i.node] - m := n + i.p.nodeData[i.node+nKey] - i.key = i.p.kvData[n:m] - if i.slice != nil { - switch { - case checkLimit && i.slice.Limit != nil && i.p.cmp.Compare(i.key, i.slice.Limit) >= 0: - fallthrough - case checkStart && i.slice.Start != nil && i.p.cmp.Compare(i.key, i.slice.Start) < 0: - i.node = 0 - goto bail - } - } - i.value = i.p.kvData[m : m+i.p.nodeData[i.node+nVal]] - return true - } -bail: - i.key = nil - i.value = nil - return false -} - -func (i *dbIter) Valid() bool { - return i.node != 0 -} - -func (i *dbIter) First() bool { - if i.Released() { - i.err = ErrIterReleased - return false - } - - i.forward = true - i.p.mu.RLock() - defer i.p.mu.RUnlock() - if i.slice != nil && i.slice.Start != nil { - i.node, _ = i.p.findGE(i.slice.Start, false) - } else { - i.node = i.p.nodeData[nNext] - } - return i.fill(false, true) -} - -func (i *dbIter) Last() bool { - if i.Released() { - i.err = ErrIterReleased - return false - } - - i.forward = false - i.p.mu.RLock() - defer i.p.mu.RUnlock() - if i.slice != nil && i.slice.Limit != nil { - i.node = i.p.findLT(i.slice.Limit) - } else { - i.node = i.p.findLast() - } - return i.fill(true, false) -} - -func (i *dbIter) Seek(key []byte) bool { - if i.Released() { - i.err = ErrIterReleased - return false - } - - i.forward = true - i.p.mu.RLock() - defer i.p.mu.RUnlock() - if i.slice != nil && i.slice.Start != nil && i.p.cmp.Compare(key, i.slice.Start) < 0 { - key = i.slice.Start - } - i.node, _ = i.p.findGE(key, false) - return i.fill(false, true) -} - -func (i *dbIter) Next() bool { - if i.Released() { - i.err = ErrIterReleased - return false - } - - if i.node == 0 { - if !i.forward { - return i.First() - } - return false - } - i.forward = true - i.p.mu.RLock() - defer i.p.mu.RUnlock() - i.node = i.p.nodeData[i.node+nNext] - return i.fill(false, true) -} - -func (i *dbIter) Prev() bool { - if i.Released() { - i.err = ErrIterReleased - return false - } - - if i.node == 0 { - if i.forward { - return i.Last() - } - return false - } - i.forward = false - i.p.mu.RLock() - defer i.p.mu.RUnlock() - i.node = i.p.findLT(i.key) - return i.fill(true, false) -} - -func (i *dbIter) Key() []byte { - return i.key -} - -func (i *dbIter) Value() []byte { - return i.value -} - -func (i *dbIter) Error() error { return i.err } - -func (i *dbIter) Release() { - if !i.Released() { - i.p = nil - i.node = 0 - i.key = nil - i.value = nil - i.BasicReleaser.Release() - } -} - -const ( - nKV = iota - nKey - nVal - nHeight - nNext -) - -// DB is an in-memory key/value database. -type DB struct { - cmp comparer.BasicComparer - rnd *rand.Rand - - mu sync.RWMutex - kvData []byte - // Node data: - // [0] : KV offset - // [1] : Key length - // [2] : Value length - // [3] : Height - // [3..height] : Next nodes - nodeData []int - prevNode [tMaxHeight]int - maxHeight int - n int - kvSize int -} - -func (p *DB) randHeight() (h int) { - const branching = 4 - h = 1 - for h < tMaxHeight && p.rnd.Int()%branching == 0 { - h++ - } - return -} - -// Must hold RW-lock if prev == true, as it use shared prevNode slice. -func (p *DB) findGE(key []byte, prev bool) (int, bool) { - node := 0 - h := p.maxHeight - 1 - for { - next := p.nodeData[node+nNext+h] - cmp := 1 - if next != 0 { - o := p.nodeData[next] - cmp = p.cmp.Compare(p.kvData[o:o+p.nodeData[next+nKey]], key) - } - if cmp < 0 { - // Keep searching in this list - node = next - } else { - if prev { - p.prevNode[h] = node - } else if cmp == 0 { - return next, true - } - if h == 0 { - return next, cmp == 0 - } - h-- - } - } -} - -func (p *DB) findLT(key []byte) int { - node := 0 - h := p.maxHeight - 1 - for { - next := p.nodeData[node+nNext+h] - o := p.nodeData[next] - if next == 0 || p.cmp.Compare(p.kvData[o:o+p.nodeData[next+nKey]], key) >= 0 { - if h == 0 { - break - } - h-- - } else { - node = next - } - } - return node -} - -func (p *DB) findLast() int { - node := 0 - h := p.maxHeight - 1 - for { - next := p.nodeData[node+nNext+h] - if next == 0 { - if h == 0 { - break - } - h-- - } else { - node = next - } - } - return node -} - -// Put sets the value for the given key. It overwrites any previous value -// for that key; a DB is not a multi-map. -// -// It is safe to modify the contents of the arguments after Put returns. -func (p *DB) Put(key []byte, value []byte) error { - p.mu.Lock() - defer p.mu.Unlock() - - if node, exact := p.findGE(key, true); exact { - kvOffset := len(p.kvData) - p.kvData = append(p.kvData, key...) - p.kvData = append(p.kvData, value...) - p.nodeData[node] = kvOffset - m := p.nodeData[node+nVal] - p.nodeData[node+nVal] = len(value) - p.kvSize += len(value) - m - return nil - } - - h := p.randHeight() - if h > p.maxHeight { - for i := p.maxHeight; i < h; i++ { - p.prevNode[i] = 0 - } - p.maxHeight = h - } - - kvOffset := len(p.kvData) - p.kvData = append(p.kvData, key...) - p.kvData = append(p.kvData, value...) - // Node - node := len(p.nodeData) - p.nodeData = append(p.nodeData, kvOffset, len(key), len(value), h) - for i, n := range p.prevNode[:h] { - m := n + nNext + i - p.nodeData = append(p.nodeData, p.nodeData[m]) - p.nodeData[m] = node - } - - p.kvSize += len(key) + len(value) - p.n++ - return nil -} - -// Delete deletes the value for the given key. It returns ErrNotFound if -// the DB does not contain the key. -// -// It is safe to modify the contents of the arguments after Delete returns. -func (p *DB) Delete(key []byte) error { - p.mu.Lock() - defer p.mu.Unlock() - - node, exact := p.findGE(key, true) - if !exact { - return ErrNotFound - } - - h := p.nodeData[node+nHeight] - for i, n := range p.prevNode[:h] { - m := n + 4 + i - p.nodeData[m] = p.nodeData[p.nodeData[m]+nNext+i] - } - - p.kvSize -= p.nodeData[node+nKey] + p.nodeData[node+nVal] - p.n-- - return nil -} - -// Contains returns true if the given key are in the DB. -// -// It is safe to modify the contents of the arguments after Contains returns. -func (p *DB) Contains(key []byte) bool { - p.mu.RLock() - _, exact := p.findGE(key, false) - p.mu.RUnlock() - return exact -} - -// Get gets the value for the given key. It returns error.ErrNotFound if the -// DB does not contain the key. -// -// The caller should not modify the contents of the returned slice, but -// it is safe to modify the contents of the argument after Get returns. -func (p *DB) Get(key []byte) (value []byte, err error) { - p.mu.RLock() - if node, exact := p.findGE(key, false); exact { - o := p.nodeData[node] + p.nodeData[node+nKey] - value = p.kvData[o : o+p.nodeData[node+nVal]] - } else { - err = ErrNotFound - } - p.mu.RUnlock() - return -} - -// Find finds key/value pair whose key is greater than or equal to the -// given key. It returns ErrNotFound if the table doesn't contain -// such pair. -// -// The caller should not modify the contents of the returned slice, but -// it is safe to modify the contents of the argument after Find returns. -func (p *DB) Find(key []byte) (rkey, value []byte, err error) { - p.mu.RLock() - if node, _ := p.findGE(key, false); node != 0 { - n := p.nodeData[node] - m := n + p.nodeData[node+nKey] - rkey = p.kvData[n:m] - value = p.kvData[m : m+p.nodeData[node+nVal]] - } else { - err = ErrNotFound - } - p.mu.RUnlock() - return -} - -// NewIterator returns an iterator of the DB. -// The returned iterator is not safe for concurrent use, but it is safe to use -// multiple iterators concurrently, with each in a dedicated goroutine. -// It is also safe to use an iterator concurrently with modifying its -// underlying DB. However, the resultant key/value pairs are not guaranteed -// to be a consistent snapshot of the DB at a particular point in time. -// -// Slice allows slicing the iterator to only contains keys in the given -// range. A nil Range.Start is treated as a key before all keys in the -// DB. And a nil Range.Limit is treated as a key after all keys in -// the DB. -// -// The iterator must be released after use, by calling Release method. -// -// Also read Iterator documentation of the leveldb/iterator package. -func (p *DB) NewIterator(slice *util.Range) iterator.Iterator { - return &dbIter{p: p, slice: slice} -} - -// Capacity returns keys/values buffer capacity. -func (p *DB) Capacity() int { - p.mu.RLock() - defer p.mu.RUnlock() - return cap(p.kvData) -} - -// Size returns sum of keys and values length. Note that deleted -// key/value will not be accounted for, but it will still consume -// the buffer, since the buffer is append only. -func (p *DB) Size() int { - p.mu.RLock() - defer p.mu.RUnlock() - return p.kvSize -} - -// Free returns keys/values free buffer before need to grow. -func (p *DB) Free() int { - p.mu.RLock() - defer p.mu.RUnlock() - return cap(p.kvData) - len(p.kvData) -} - -// Len returns the number of entries in the DB. -func (p *DB) Len() int { - p.mu.RLock() - defer p.mu.RUnlock() - return p.n -} - -// Reset resets the DB to initial empty state. Allows reuse the buffer. -func (p *DB) Reset() { - p.mu.Lock() - p.rnd = rand.New(rand.NewSource(0xdeadbeef)) - p.maxHeight = 1 - p.n = 0 - p.kvSize = 0 - p.kvData = p.kvData[:0] - p.nodeData = p.nodeData[:nNext+tMaxHeight] - p.nodeData[nKV] = 0 - p.nodeData[nKey] = 0 - p.nodeData[nVal] = 0 - p.nodeData[nHeight] = tMaxHeight - for n := 0; n < tMaxHeight; n++ { - p.nodeData[nNext+n] = 0 - p.prevNode[n] = 0 - } - p.mu.Unlock() -} - -// New creates a new initialized in-memory key/value DB. The capacity -// is the initial key/value buffer capacity. The capacity is advisory, -// not enforced. -// -// This DB is append-only, deleting an entry would remove entry node but not -// reclaim KV buffer. -// -// The returned DB instance is safe for concurrent use. -func New(cmp comparer.BasicComparer, capacity int) *DB { - p := &DB{ - cmp: cmp, - rnd: rand.New(rand.NewSource(0xdeadbeef)), - maxHeight: 1, - kvData: make([]byte, 0, capacity), - nodeData: make([]int, 4+tMaxHeight), - } - p.nodeData[nHeight] = tMaxHeight - return p -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/memdb/memdb_suite_test.go b/vendor/github.com/syndtr/goleveldb/leveldb/memdb/memdb_suite_test.go deleted file mode 100644 index 18c304b..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/memdb/memdb_suite_test.go +++ /dev/null @@ -1,11 +0,0 @@ -package memdb - -import ( - "testing" - - "github.com/syndtr/goleveldb/leveldb/testutil" -) - -func TestMemDB(t *testing.T) { - testutil.RunSuite(t, "MemDB Suite") -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/memdb/memdb_test.go b/vendor/github.com/syndtr/goleveldb/leveldb/memdb/memdb_test.go deleted file mode 100644 index 3f0a31e..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/memdb/memdb_test.go +++ /dev/null @@ -1,135 +0,0 @@ -// Copyright (c) 2014, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package memdb - -import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - - "github.com/syndtr/goleveldb/leveldb/comparer" - "github.com/syndtr/goleveldb/leveldb/iterator" - "github.com/syndtr/goleveldb/leveldb/testutil" - "github.com/syndtr/goleveldb/leveldb/util" -) - -func (p *DB) TestFindLT(key []byte) (rkey, value []byte, err error) { - p.mu.RLock() - if node := p.findLT(key); node != 0 { - n := p.nodeData[node] - m := n + p.nodeData[node+nKey] - rkey = p.kvData[n:m] - value = p.kvData[m : m+p.nodeData[node+nVal]] - } else { - err = ErrNotFound - } - p.mu.RUnlock() - return -} - -func (p *DB) TestFindLast() (rkey, value []byte, err error) { - p.mu.RLock() - if node := p.findLast(); node != 0 { - n := p.nodeData[node] - m := n + p.nodeData[node+nKey] - rkey = p.kvData[n:m] - value = p.kvData[m : m+p.nodeData[node+nVal]] - } else { - err = ErrNotFound - } - p.mu.RUnlock() - return -} - -func (p *DB) TestPut(key []byte, value []byte) error { - p.Put(key, value) - return nil -} - -func (p *DB) TestDelete(key []byte) error { - p.Delete(key) - return nil -} - -func (p *DB) TestFind(key []byte) (rkey, rvalue []byte, err error) { - return p.Find(key) -} - -func (p *DB) TestGet(key []byte) (value []byte, err error) { - return p.Get(key) -} - -func (p *DB) TestNewIterator(slice *util.Range) iterator.Iterator { - return p.NewIterator(slice) -} - -var _ = testutil.Defer(func() { - Describe("Memdb", func() { - Describe("write test", func() { - It("should do write correctly", func() { - db := New(comparer.DefaultComparer, 0) - t := testutil.DBTesting{ - DB: db, - Deleted: testutil.KeyValue_Generate(nil, 1000, 1, 1, 30, 5, 5).Clone(), - PostFn: func(t *testutil.DBTesting) { - Expect(db.Len()).Should(Equal(t.Present.Len())) - Expect(db.Size()).Should(Equal(t.Present.Size())) - switch t.Act { - case testutil.DBPut, testutil.DBOverwrite: - Expect(db.Contains(t.ActKey)).Should(BeTrue()) - default: - Expect(db.Contains(t.ActKey)).Should(BeFalse()) - } - }, - } - testutil.DoDBTesting(&t) - }) - }) - - Describe("read test", func() { - testutil.AllKeyValueTesting(nil, func(kv testutil.KeyValue) testutil.DB { - // Building the DB. - db := New(comparer.DefaultComparer, 0) - kv.IterateShuffled(nil, func(i int, key, value []byte) { - db.Put(key, value) - }) - - if kv.Len() > 1 { - It("Should find correct keys with findLT", func() { - testutil.ShuffledIndex(nil, kv.Len()-1, 1, func(i int) { - key_, key, _ := kv.IndexInexact(i + 1) - expectedKey, expectedValue := kv.Index(i) - - // Using key that exist. - rkey, rvalue, err := db.TestFindLT(key) - Expect(err).ShouldNot(HaveOccurred(), "Error for key %q -> %q", key, expectedKey) - Expect(rkey).Should(Equal(expectedKey), "Key") - Expect(rvalue).Should(Equal(expectedValue), "Value for key %q -> %q", key, expectedKey) - - // Using key that doesn't exist. - rkey, rvalue, err = db.TestFindLT(key_) - Expect(err).ShouldNot(HaveOccurred(), "Error for key %q (%q) -> %q", key_, key, expectedKey) - Expect(rkey).Should(Equal(expectedKey)) - Expect(rvalue).Should(Equal(expectedValue), "Value for key %q (%q) -> %q", key_, key, expectedKey) - }) - }) - } - - if kv.Len() > 0 { - It("Should find last key with findLast", func() { - key, value := kv.Index(kv.Len() - 1) - rkey, rvalue, err := db.TestFindLast() - Expect(err).ShouldNot(HaveOccurred()) - Expect(rkey).Should(Equal(key)) - Expect(rvalue).Should(Equal(value)) - }) - } - - return db - }, nil, nil) - }) - }) -}) diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/opt/options.go b/vendor/github.com/syndtr/goleveldb/leveldb/opt/options.go deleted file mode 100644 index 44e7d9a..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/opt/options.go +++ /dev/null @@ -1,684 +0,0 @@ -// Copyright (c) 2012, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -// Package opt provides sets of options used by LevelDB. -package opt - -import ( - "math" - - "github.com/syndtr/goleveldb/leveldb/cache" - "github.com/syndtr/goleveldb/leveldb/comparer" - "github.com/syndtr/goleveldb/leveldb/filter" -) - -const ( - KiB = 1024 - MiB = KiB * 1024 - GiB = MiB * 1024 -) - -var ( - DefaultBlockCacher = LRUCacher - DefaultBlockCacheCapacity = 8 * MiB - DefaultBlockRestartInterval = 16 - DefaultBlockSize = 4 * KiB - DefaultCompactionExpandLimitFactor = 25 - DefaultCompactionGPOverlapsFactor = 10 - DefaultCompactionL0Trigger = 4 - DefaultCompactionSourceLimitFactor = 1 - DefaultCompactionTableSize = 2 * MiB - DefaultCompactionTableSizeMultiplier = 1.0 - DefaultCompactionTotalSize = 10 * MiB - DefaultCompactionTotalSizeMultiplier = 10.0 - DefaultCompressionType = SnappyCompression - DefaultIteratorSamplingRate = 1 * MiB - DefaultOpenFilesCacher = LRUCacher - DefaultOpenFilesCacheCapacity = 500 - DefaultWriteBuffer = 4 * MiB - DefaultWriteL0PauseTrigger = 12 - DefaultWriteL0SlowdownTrigger = 8 -) - -// Cacher is a caching algorithm. -type Cacher interface { - New(capacity int) cache.Cacher -} - -type CacherFunc struct { - NewFunc func(capacity int) cache.Cacher -} - -func (f *CacherFunc) New(capacity int) cache.Cacher { - if f.NewFunc != nil { - return f.NewFunc(capacity) - } - return nil -} - -func noCacher(int) cache.Cacher { return nil } - -var ( - // LRUCacher is the LRU-cache algorithm. - LRUCacher = &CacherFunc{cache.NewLRU} - - // NoCacher is the value to disable caching algorithm. - NoCacher = &CacherFunc{} -) - -// Compression is the 'sorted table' block compression algorithm to use. -type Compression uint - -func (c Compression) String() string { - switch c { - case DefaultCompression: - return "default" - case NoCompression: - return "none" - case SnappyCompression: - return "snappy" - } - return "invalid" -} - -const ( - DefaultCompression Compression = iota - NoCompression - SnappyCompression - nCompression -) - -// Strict is the DB 'strict level'. -type Strict uint - -const ( - // If present then a corrupted or invalid chunk or block in manifest - // journal will cause an error instead of being dropped. - // This will prevent database with corrupted manifest to be opened. - StrictManifest Strict = 1 << iota - - // If present then journal chunk checksum will be verified. - StrictJournalChecksum - - // If present then a corrupted or invalid chunk or block in journal - // will cause an error instead of being dropped. - // This will prevent database with corrupted journal to be opened. - StrictJournal - - // If present then 'sorted table' block checksum will be verified. - // This has effect on both 'read operation' and compaction. - StrictBlockChecksum - - // If present then a corrupted 'sorted table' will fails compaction. - // The database will enter read-only mode. - StrictCompaction - - // If present then a corrupted 'sorted table' will halts 'read operation'. - StrictReader - - // If present then leveldb.Recover will drop corrupted 'sorted table'. - StrictRecovery - - // This only applicable for ReadOptions, if present then this ReadOptions - // 'strict level' will override global ones. - StrictOverride - - // StrictAll enables all strict flags. - StrictAll = StrictManifest | StrictJournalChecksum | StrictJournal | StrictBlockChecksum | StrictCompaction | StrictReader | StrictRecovery - - // DefaultStrict is the default strict flags. Specify any strict flags - // will override default strict flags as whole (i.e. not OR'ed). - DefaultStrict = StrictJournalChecksum | StrictBlockChecksum | StrictCompaction | StrictReader - - // NoStrict disables all strict flags. Override default strict flags. - NoStrict = ^StrictAll -) - -// Options holds the optional parameters for the DB at large. -type Options struct { - // AltFilters defines one or more 'alternative filters'. - // 'alternative filters' will be used during reads if a filter block - // does not match with the 'effective filter'. - // - // The default value is nil - AltFilters []filter.Filter - - // BlockCacher provides cache algorithm for LevelDB 'sorted table' block caching. - // Specify NoCacher to disable caching algorithm. - // - // The default value is LRUCacher. - BlockCacher Cacher - - // BlockCacheCapacity defines the capacity of the 'sorted table' block caching. - // Use -1 for zero, this has same effect as specifying NoCacher to BlockCacher. - // - // The default value is 8MiB. - BlockCacheCapacity int - - // BlockRestartInterval is the number of keys between restart points for - // delta encoding of keys. - // - // The default value is 16. - BlockRestartInterval int - - // BlockSize is the minimum uncompressed size in bytes of each 'sorted table' - // block. - // - // The default value is 4KiB. - BlockSize int - - // CompactionExpandLimitFactor limits compaction size after expanded. - // This will be multiplied by table size limit at compaction target level. - // - // The default value is 25. - CompactionExpandLimitFactor int - - // CompactionGPOverlapsFactor limits overlaps in grandparent (Level + 2) that a - // single 'sorted table' generates. - // This will be multiplied by table size limit at grandparent level. - // - // The default value is 10. - CompactionGPOverlapsFactor int - - // CompactionL0Trigger defines number of 'sorted table' at level-0 that will - // trigger compaction. - // - // The default value is 4. - CompactionL0Trigger int - - // CompactionSourceLimitFactor limits compaction source size. This doesn't apply to - // level-0. - // This will be multiplied by table size limit at compaction target level. - // - // The default value is 1. - CompactionSourceLimitFactor int - - // CompactionTableSize limits size of 'sorted table' that compaction generates. - // The limits for each level will be calculated as: - // CompactionTableSize * (CompactionTableSizeMultiplier ^ Level) - // The multiplier for each level can also fine-tuned using CompactionTableSizeMultiplierPerLevel. - // - // The default value is 2MiB. - CompactionTableSize int - - // CompactionTableSizeMultiplier defines multiplier for CompactionTableSize. - // - // The default value is 1. - CompactionTableSizeMultiplier float64 - - // CompactionTableSizeMultiplierPerLevel defines per-level multiplier for - // CompactionTableSize. - // Use zero to skip a level. - // - // The default value is nil. - CompactionTableSizeMultiplierPerLevel []float64 - - // CompactionTotalSize limits total size of 'sorted table' for each level. - // The limits for each level will be calculated as: - // CompactionTotalSize * (CompactionTotalSizeMultiplier ^ Level) - // The multiplier for each level can also fine-tuned using - // CompactionTotalSizeMultiplierPerLevel. - // - // The default value is 10MiB. - CompactionTotalSize int - - // CompactionTotalSizeMultiplier defines multiplier for CompactionTotalSize. - // - // The default value is 10. - CompactionTotalSizeMultiplier float64 - - // CompactionTotalSizeMultiplierPerLevel defines per-level multiplier for - // CompactionTotalSize. - // Use zero to skip a level. - // - // The default value is nil. - CompactionTotalSizeMultiplierPerLevel []float64 - - // Comparer defines a total ordering over the space of []byte keys: a 'less - // than' relationship. The same comparison algorithm must be used for reads - // and writes over the lifetime of the DB. - // - // The default value uses the same ordering as bytes.Compare. - Comparer comparer.Comparer - - // Compression defines the 'sorted table' block compression to use. - // - // The default value (DefaultCompression) uses snappy compression. - Compression Compression - - // DisableBufferPool allows disable use of util.BufferPool functionality. - // - // The default value is false. - DisableBufferPool bool - - // DisableBlockCache allows disable use of cache.Cache functionality on - // 'sorted table' block. - // - // The default value is false. - DisableBlockCache bool - - // DisableCompactionBackoff allows disable compaction retry backoff. - // - // The default value is false. - DisableCompactionBackoff bool - - // DisableLargeBatchTransaction allows disabling switch-to-transaction mode - // on large batch write. If enable batch writes large than WriteBuffer will - // use transaction. - // - // The default is false. - DisableLargeBatchTransaction bool - - // ErrorIfExist defines whether an error should returned if the DB already - // exist. - // - // The default value is false. - ErrorIfExist bool - - // ErrorIfMissing defines whether an error should returned if the DB is - // missing. If false then the database will be created if missing, otherwise - // an error will be returned. - // - // The default value is false. - ErrorIfMissing bool - - // Filter defines an 'effective filter' to use. An 'effective filter' - // if defined will be used to generate per-table filter block. - // The filter name will be stored on disk. - // During reads LevelDB will try to find matching filter from - // 'effective filter' and 'alternative filters'. - // - // Filter can be changed after a DB has been created. It is recommended - // to put old filter to the 'alternative filters' to mitigate lack of - // filter during transition period. - // - // A filter is used to reduce disk reads when looking for a specific key. - // - // The default value is nil. - Filter filter.Filter - - // IteratorSamplingRate defines approximate gap (in bytes) between read - // sampling of an iterator. The samples will be used to determine when - // compaction should be triggered. - // - // The default is 1MiB. - IteratorSamplingRate int - - // NoSync allows completely disable fsync. - // - // The default is false. - NoSync bool - - // NoWriteMerge allows disabling write merge. - // - // The default is false. - NoWriteMerge bool - - // OpenFilesCacher provides cache algorithm for open files caching. - // Specify NoCacher to disable caching algorithm. - // - // The default value is LRUCacher. - OpenFilesCacher Cacher - - // OpenFilesCacheCapacity defines the capacity of the open files caching. - // Use -1 for zero, this has same effect as specifying NoCacher to OpenFilesCacher. - // - // The default value is 500. - OpenFilesCacheCapacity int - - // If true then opens DB in read-only mode. - // - // The default value is false. - ReadOnly bool - - // Strict defines the DB strict level. - Strict Strict - - // WriteBuffer defines maximum size of a 'memdb' before flushed to - // 'sorted table'. 'memdb' is an in-memory DB backed by an on-disk - // unsorted journal. - // - // LevelDB may held up to two 'memdb' at the same time. - // - // The default value is 4MiB. - WriteBuffer int - - // WriteL0StopTrigger defines number of 'sorted table' at level-0 that will - // pause write. - // - // The default value is 12. - WriteL0PauseTrigger int - - // WriteL0SlowdownTrigger defines number of 'sorted table' at level-0 that - // will trigger write slowdown. - // - // The default value is 8. - WriteL0SlowdownTrigger int -} - -func (o *Options) GetAltFilters() []filter.Filter { - if o == nil { - return nil - } - return o.AltFilters -} - -func (o *Options) GetBlockCacher() Cacher { - if o == nil || o.BlockCacher == nil { - return DefaultBlockCacher - } else if o.BlockCacher == NoCacher { - return nil - } - return o.BlockCacher -} - -func (o *Options) GetBlockCacheCapacity() int { - if o == nil || o.BlockCacheCapacity == 0 { - return DefaultBlockCacheCapacity - } else if o.BlockCacheCapacity < 0 { - return 0 - } - return o.BlockCacheCapacity -} - -func (o *Options) GetBlockRestartInterval() int { - if o == nil || o.BlockRestartInterval <= 0 { - return DefaultBlockRestartInterval - } - return o.BlockRestartInterval -} - -func (o *Options) GetBlockSize() int { - if o == nil || o.BlockSize <= 0 { - return DefaultBlockSize - } - return o.BlockSize -} - -func (o *Options) GetCompactionExpandLimit(level int) int { - factor := DefaultCompactionExpandLimitFactor - if o != nil && o.CompactionExpandLimitFactor > 0 { - factor = o.CompactionExpandLimitFactor - } - return o.GetCompactionTableSize(level+1) * factor -} - -func (o *Options) GetCompactionGPOverlaps(level int) int { - factor := DefaultCompactionGPOverlapsFactor - if o != nil && o.CompactionGPOverlapsFactor > 0 { - factor = o.CompactionGPOverlapsFactor - } - return o.GetCompactionTableSize(level+2) * factor -} - -func (o *Options) GetCompactionL0Trigger() int { - if o == nil || o.CompactionL0Trigger == 0 { - return DefaultCompactionL0Trigger - } - return o.CompactionL0Trigger -} - -func (o *Options) GetCompactionSourceLimit(level int) int { - factor := DefaultCompactionSourceLimitFactor - if o != nil && o.CompactionSourceLimitFactor > 0 { - factor = o.CompactionSourceLimitFactor - } - return o.GetCompactionTableSize(level+1) * factor -} - -func (o *Options) GetCompactionTableSize(level int) int { - var ( - base = DefaultCompactionTableSize - mult float64 - ) - if o != nil { - if o.CompactionTableSize > 0 { - base = o.CompactionTableSize - } - if level < len(o.CompactionTableSizeMultiplierPerLevel) && o.CompactionTableSizeMultiplierPerLevel[level] > 0 { - mult = o.CompactionTableSizeMultiplierPerLevel[level] - } else if o.CompactionTableSizeMultiplier > 0 { - mult = math.Pow(o.CompactionTableSizeMultiplier, float64(level)) - } - } - if mult == 0 { - mult = math.Pow(DefaultCompactionTableSizeMultiplier, float64(level)) - } - return int(float64(base) * mult) -} - -func (o *Options) GetCompactionTotalSize(level int) int64 { - var ( - base = DefaultCompactionTotalSize - mult float64 - ) - if o != nil { - if o.CompactionTotalSize > 0 { - base = o.CompactionTotalSize - } - if level < len(o.CompactionTotalSizeMultiplierPerLevel) && o.CompactionTotalSizeMultiplierPerLevel[level] > 0 { - mult = o.CompactionTotalSizeMultiplierPerLevel[level] - } else if o.CompactionTotalSizeMultiplier > 0 { - mult = math.Pow(o.CompactionTotalSizeMultiplier, float64(level)) - } - } - if mult == 0 { - mult = math.Pow(DefaultCompactionTotalSizeMultiplier, float64(level)) - } - return int64(float64(base) * mult) -} - -func (o *Options) GetComparer() comparer.Comparer { - if o == nil || o.Comparer == nil { - return comparer.DefaultComparer - } - return o.Comparer -} - -func (o *Options) GetCompression() Compression { - if o == nil || o.Compression <= DefaultCompression || o.Compression >= nCompression { - return DefaultCompressionType - } - return o.Compression -} - -func (o *Options) GetDisableBufferPool() bool { - if o == nil { - return false - } - return o.DisableBufferPool -} - -func (o *Options) GetDisableBlockCache() bool { - if o == nil { - return false - } - return o.DisableBlockCache -} - -func (o *Options) GetDisableCompactionBackoff() bool { - if o == nil { - return false - } - return o.DisableCompactionBackoff -} - -func (o *Options) GetDisableLargeBatchTransaction() bool { - if o == nil { - return false - } - return o.DisableLargeBatchTransaction -} - -func (o *Options) GetErrorIfExist() bool { - if o == nil { - return false - } - return o.ErrorIfExist -} - -func (o *Options) GetErrorIfMissing() bool { - if o == nil { - return false - } - return o.ErrorIfMissing -} - -func (o *Options) GetFilter() filter.Filter { - if o == nil { - return nil - } - return o.Filter -} - -func (o *Options) GetIteratorSamplingRate() int { - if o == nil || o.IteratorSamplingRate <= 0 { - return DefaultIteratorSamplingRate - } - return o.IteratorSamplingRate -} - -func (o *Options) GetNoSync() bool { - if o == nil { - return false - } - return o.NoSync -} - -func (o *Options) GetNoWriteMerge() bool { - if o == nil { - return false - } - return o.NoWriteMerge -} - -func (o *Options) GetOpenFilesCacher() Cacher { - if o == nil || o.OpenFilesCacher == nil { - return DefaultOpenFilesCacher - } - if o.OpenFilesCacher == NoCacher { - return nil - } - return o.OpenFilesCacher -} - -func (o *Options) GetOpenFilesCacheCapacity() int { - if o == nil || o.OpenFilesCacheCapacity == 0 { - return DefaultOpenFilesCacheCapacity - } else if o.OpenFilesCacheCapacity < 0 { - return 0 - } - return o.OpenFilesCacheCapacity -} - -func (o *Options) GetReadOnly() bool { - if o == nil { - return false - } - return o.ReadOnly -} - -func (o *Options) GetStrict(strict Strict) bool { - if o == nil || o.Strict == 0 { - return DefaultStrict&strict != 0 - } - return o.Strict&strict != 0 -} - -func (o *Options) GetWriteBuffer() int { - if o == nil || o.WriteBuffer <= 0 { - return DefaultWriteBuffer - } - return o.WriteBuffer -} - -func (o *Options) GetWriteL0PauseTrigger() int { - if o == nil || o.WriteL0PauseTrigger == 0 { - return DefaultWriteL0PauseTrigger - } - return o.WriteL0PauseTrigger -} - -func (o *Options) GetWriteL0SlowdownTrigger() int { - if o == nil || o.WriteL0SlowdownTrigger == 0 { - return DefaultWriteL0SlowdownTrigger - } - return o.WriteL0SlowdownTrigger -} - -// ReadOptions holds the optional parameters for 'read operation'. The -// 'read operation' includes Get, Find and NewIterator. -type ReadOptions struct { - // DontFillCache defines whether block reads for this 'read operation' - // should be cached. If false then the block will be cached. This does - // not affects already cached block. - // - // The default value is false. - DontFillCache bool - - // Strict will be OR'ed with global DB 'strict level' unless StrictOverride - // is present. Currently only StrictReader that has effect here. - Strict Strict -} - -func (ro *ReadOptions) GetDontFillCache() bool { - if ro == nil { - return false - } - return ro.DontFillCache -} - -func (ro *ReadOptions) GetStrict(strict Strict) bool { - if ro == nil { - return false - } - return ro.Strict&strict != 0 -} - -// WriteOptions holds the optional parameters for 'write operation'. The -// 'write operation' includes Write, Put and Delete. -type WriteOptions struct { - // NoWriteMerge allows disabling write merge. - // - // The default is false. - NoWriteMerge bool - - // Sync is whether to sync underlying writes from the OS buffer cache - // through to actual disk, if applicable. Setting Sync can result in - // slower writes. - // - // If false, and the machine crashes, then some recent writes may be lost. - // Note that if it is just the process that crashes (and the machine does - // not) then no writes will be lost. - // - // In other words, Sync being false has the same semantics as a write - // system call. Sync being true means write followed by fsync. - // - // The default value is false. - Sync bool -} - -func (wo *WriteOptions) GetNoWriteMerge() bool { - if wo == nil { - return false - } - return wo.NoWriteMerge -} - -func (wo *WriteOptions) GetSync() bool { - if wo == nil { - return false - } - return wo.Sync -} - -func GetStrict(o *Options, ro *ReadOptions, strict Strict) bool { - if ro.GetStrict(StrictOverride) { - return ro.GetStrict(strict) - } else { - return o.GetStrict(strict) || ro.GetStrict(strict) - } -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/options.go b/vendor/github.com/syndtr/goleveldb/leveldb/options.go deleted file mode 100644 index b072b1a..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/options.go +++ /dev/null @@ -1,107 +0,0 @@ -// Copyright (c) 2012, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package leveldb - -import ( - "github.com/syndtr/goleveldb/leveldb/filter" - "github.com/syndtr/goleveldb/leveldb/opt" -) - -func dupOptions(o *opt.Options) *opt.Options { - newo := &opt.Options{} - if o != nil { - *newo = *o - } - if newo.Strict == 0 { - newo.Strict = opt.DefaultStrict - } - return newo -} - -func (s *session) setOptions(o *opt.Options) { - no := dupOptions(o) - // Alternative filters. - if filters := o.GetAltFilters(); len(filters) > 0 { - no.AltFilters = make([]filter.Filter, len(filters)) - for i, filter := range filters { - no.AltFilters[i] = &iFilter{filter} - } - } - // Comparer. - s.icmp = &iComparer{o.GetComparer()} - no.Comparer = s.icmp - // Filter. - if filter := o.GetFilter(); filter != nil { - no.Filter = &iFilter{filter} - } - - s.o = &cachedOptions{Options: no} - s.o.cache() -} - -const optCachedLevel = 7 - -type cachedOptions struct { - *opt.Options - - compactionExpandLimit []int - compactionGPOverlaps []int - compactionSourceLimit []int - compactionTableSize []int - compactionTotalSize []int64 -} - -func (co *cachedOptions) cache() { - co.compactionExpandLimit = make([]int, optCachedLevel) - co.compactionGPOverlaps = make([]int, optCachedLevel) - co.compactionSourceLimit = make([]int, optCachedLevel) - co.compactionTableSize = make([]int, optCachedLevel) - co.compactionTotalSize = make([]int64, optCachedLevel) - - for level := 0; level < optCachedLevel; level++ { - co.compactionExpandLimit[level] = co.Options.GetCompactionExpandLimit(level) - co.compactionGPOverlaps[level] = co.Options.GetCompactionGPOverlaps(level) - co.compactionSourceLimit[level] = co.Options.GetCompactionSourceLimit(level) - co.compactionTableSize[level] = co.Options.GetCompactionTableSize(level) - co.compactionTotalSize[level] = co.Options.GetCompactionTotalSize(level) - } -} - -func (co *cachedOptions) GetCompactionExpandLimit(level int) int { - if level < optCachedLevel { - return co.compactionExpandLimit[level] - } - return co.Options.GetCompactionExpandLimit(level) -} - -func (co *cachedOptions) GetCompactionGPOverlaps(level int) int { - if level < optCachedLevel { - return co.compactionGPOverlaps[level] - } - return co.Options.GetCompactionGPOverlaps(level) -} - -func (co *cachedOptions) GetCompactionSourceLimit(level int) int { - if level < optCachedLevel { - return co.compactionSourceLimit[level] - } - return co.Options.GetCompactionSourceLimit(level) -} - -func (co *cachedOptions) GetCompactionTableSize(level int) int { - if level < optCachedLevel { - return co.compactionTableSize[level] - } - return co.Options.GetCompactionTableSize(level) -} - -func (co *cachedOptions) GetCompactionTotalSize(level int) int64 { - if level < optCachedLevel { - return co.compactionTotalSize[level] - } - return co.Options.GetCompactionTotalSize(level) -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/session.go b/vendor/github.com/syndtr/goleveldb/leveldb/session.go deleted file mode 100644 index ad68a87..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/session.go +++ /dev/null @@ -1,210 +0,0 @@ -// Copyright (c) 2012, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package leveldb - -import ( - "fmt" - "io" - "os" - "sync" - - "github.com/syndtr/goleveldb/leveldb/errors" - "github.com/syndtr/goleveldb/leveldb/journal" - "github.com/syndtr/goleveldb/leveldb/opt" - "github.com/syndtr/goleveldb/leveldb/storage" -) - -// ErrManifestCorrupted records manifest corruption. This error will be -// wrapped with errors.ErrCorrupted. -type ErrManifestCorrupted struct { - Field string - Reason string -} - -func (e *ErrManifestCorrupted) Error() string { - return fmt.Sprintf("leveldb: manifest corrupted (field '%s'): %s", e.Field, e.Reason) -} - -func newErrManifestCorrupted(fd storage.FileDesc, field, reason string) error { - return errors.NewErrCorrupted(fd, &ErrManifestCorrupted{field, reason}) -} - -// session represent a persistent database session. -type session struct { - // Need 64-bit alignment. - stNextFileNum int64 // current unused file number - stJournalNum int64 // current journal file number; need external synchronization - stPrevJournalNum int64 // prev journal file number; no longer used; for compatibility with older version of leveldb - stTempFileNum int64 - stSeqNum uint64 // last mem compacted seq; need external synchronization - - stor storage.Storage - storLock storage.Locker - o *cachedOptions - icmp *iComparer - tops *tOps - fileRef map[int64]int - - manifest *journal.Writer - manifestWriter storage.Writer - manifestFd storage.FileDesc - - stCompPtrs []internalKey // compaction pointers; need external synchronization - stVersion *version // current version - vmu sync.Mutex -} - -// Creates new initialized session instance. -func newSession(stor storage.Storage, o *opt.Options) (s *session, err error) { - if stor == nil { - return nil, os.ErrInvalid - } - storLock, err := stor.Lock() - if err != nil { - return - } - s = &session{ - stor: stor, - storLock: storLock, - fileRef: make(map[int64]int), - } - s.setOptions(o) - s.tops = newTableOps(s) - s.setVersion(newVersion(s)) - s.log("log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed") - return -} - -// Close session. -func (s *session) close() { - s.tops.close() - if s.manifest != nil { - s.manifest.Close() - } - if s.manifestWriter != nil { - s.manifestWriter.Close() - } - s.manifest = nil - s.manifestWriter = nil - s.setVersion(&version{s: s, closing: true}) -} - -// Release session lock. -func (s *session) release() { - s.storLock.Unlock() -} - -// Create a new database session; need external synchronization. -func (s *session) create() error { - // create manifest - return s.newManifest(nil, nil) -} - -// Recover a database session; need external synchronization. -func (s *session) recover() (err error) { - defer func() { - if os.IsNotExist(err) { - // Don't return os.ErrNotExist if the underlying storage contains - // other files that belong to LevelDB. So the DB won't get trashed. - if fds, _ := s.stor.List(storage.TypeAll); len(fds) > 0 { - err = &errors.ErrCorrupted{Fd: storage.FileDesc{Type: storage.TypeManifest}, Err: &errors.ErrMissingFiles{}} - } - } - }() - - fd, err := s.stor.GetMeta() - if err != nil { - return - } - - reader, err := s.stor.Open(fd) - if err != nil { - return - } - defer reader.Close() - - var ( - // Options. - strict = s.o.GetStrict(opt.StrictManifest) - - jr = journal.NewReader(reader, dropper{s, fd}, strict, true) - rec = &sessionRecord{} - staging = s.stVersion.newStaging() - ) - for { - var r io.Reader - r, err = jr.Next() - if err != nil { - if err == io.EOF { - err = nil - break - } - return errors.SetFd(err, fd) - } - - err = rec.decode(r) - if err == nil { - // save compact pointers - for _, r := range rec.compPtrs { - s.setCompPtr(r.level, internalKey(r.ikey)) - } - // commit record to version staging - staging.commit(rec) - } else { - err = errors.SetFd(err, fd) - if strict || !errors.IsCorrupted(err) { - return - } - s.logf("manifest error: %v (skipped)", errors.SetFd(err, fd)) - } - rec.resetCompPtrs() - rec.resetAddedTables() - rec.resetDeletedTables() - } - - switch { - case !rec.has(recComparer): - return newErrManifestCorrupted(fd, "comparer", "missing") - case rec.comparer != s.icmp.uName(): - return newErrManifestCorrupted(fd, "comparer", fmt.Sprintf("mismatch: want '%s', got '%s'", s.icmp.uName(), rec.comparer)) - case !rec.has(recNextFileNum): - return newErrManifestCorrupted(fd, "next-file-num", "missing") - case !rec.has(recJournalNum): - return newErrManifestCorrupted(fd, "journal-file-num", "missing") - case !rec.has(recSeqNum): - return newErrManifestCorrupted(fd, "seq-num", "missing") - } - - s.manifestFd = fd - s.setVersion(staging.finish()) - s.setNextFileNum(rec.nextFileNum) - s.recordCommited(rec) - return nil -} - -// Commit session; need external synchronization. -func (s *session) commit(r *sessionRecord) (err error) { - v := s.version() - defer v.release() - - // spawn new version based on current version - nv := v.spawn(r) - - if s.manifest == nil { - // manifest journal writer not yet created, create one - err = s.newManifest(r, nv) - } else { - err = s.flushManifest(r) - } - - // finally, apply new version if no error rise - if err == nil { - s.setVersion(nv) - } - - return -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/session_compaction.go b/vendor/github.com/syndtr/goleveldb/leveldb/session_compaction.go deleted file mode 100644 index 089cd00..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/session_compaction.go +++ /dev/null @@ -1,302 +0,0 @@ -// Copyright (c) 2012, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package leveldb - -import ( - "sync/atomic" - - "github.com/syndtr/goleveldb/leveldb/iterator" - "github.com/syndtr/goleveldb/leveldb/memdb" - "github.com/syndtr/goleveldb/leveldb/opt" -) - -func (s *session) pickMemdbLevel(umin, umax []byte, maxLevel int) int { - v := s.version() - defer v.release() - return v.pickMemdbLevel(umin, umax, maxLevel) -} - -func (s *session) flushMemdb(rec *sessionRecord, mdb *memdb.DB, maxLevel int) (int, error) { - // Create sorted table. - iter := mdb.NewIterator(nil) - defer iter.Release() - t, n, err := s.tops.createFrom(iter) - if err != nil { - return 0, err - } - - // Pick level other than zero can cause compaction issue with large - // bulk insert and delete on strictly incrementing key-space. The - // problem is that the small deletion markers trapped at lower level, - // while key/value entries keep growing at higher level. Since the - // key-space is strictly incrementing it will not overlaps with - // higher level, thus maximum possible level is always picked, while - // overlapping deletion marker pushed into lower level. - // See: https://github.com/syndtr/goleveldb/issues/127. - flushLevel := s.pickMemdbLevel(t.imin.ukey(), t.imax.ukey(), maxLevel) - rec.addTableFile(flushLevel, t) - - s.logf("memdb@flush created L%d@%d N·%d S·%s %q:%q", flushLevel, t.fd.Num, n, shortenb(int(t.size)), t.imin, t.imax) - return flushLevel, nil -} - -// Pick a compaction based on current state; need external synchronization. -func (s *session) pickCompaction() *compaction { - v := s.version() - - var sourceLevel int - var t0 tFiles - if v.cScore >= 1 { - sourceLevel = v.cLevel - cptr := s.getCompPtr(sourceLevel) - tables := v.levels[sourceLevel] - for _, t := range tables { - if cptr == nil || s.icmp.Compare(t.imax, cptr) > 0 { - t0 = append(t0, t) - break - } - } - if len(t0) == 0 { - t0 = append(t0, tables[0]) - } - } else { - if p := atomic.LoadPointer(&v.cSeek); p != nil { - ts := (*tSet)(p) - sourceLevel = ts.level - t0 = append(t0, ts.table) - } else { - v.release() - return nil - } - } - - return newCompaction(s, v, sourceLevel, t0) -} - -// Create compaction from given level and range; need external synchronization. -func (s *session) getCompactionRange(sourceLevel int, umin, umax []byte, noLimit bool) *compaction { - v := s.version() - - if sourceLevel >= len(v.levels) { - v.release() - return nil - } - - t0 := v.levels[sourceLevel].getOverlaps(nil, s.icmp, umin, umax, sourceLevel == 0) - if len(t0) == 0 { - v.release() - return nil - } - - // Avoid compacting too much in one shot in case the range is large. - // But we cannot do this for level-0 since level-0 files can overlap - // and we must not pick one file and drop another older file if the - // two files overlap. - if !noLimit && sourceLevel > 0 { - limit := int64(v.s.o.GetCompactionSourceLimit(sourceLevel)) - total := int64(0) - for i, t := range t0 { - total += t.size - if total >= limit { - s.logf("table@compaction limiting F·%d -> F·%d", len(t0), i+1) - t0 = t0[:i+1] - break - } - } - } - - return newCompaction(s, v, sourceLevel, t0) -} - -func newCompaction(s *session, v *version, sourceLevel int, t0 tFiles) *compaction { - c := &compaction{ - s: s, - v: v, - sourceLevel: sourceLevel, - levels: [2]tFiles{t0, nil}, - maxGPOverlaps: int64(s.o.GetCompactionGPOverlaps(sourceLevel)), - tPtrs: make([]int, len(v.levels)), - } - c.expand() - c.save() - return c -} - -// compaction represent a compaction state. -type compaction struct { - s *session - v *version - - sourceLevel int - levels [2]tFiles - maxGPOverlaps int64 - - gp tFiles - gpi int - seenKey bool - gpOverlappedBytes int64 - imin, imax internalKey - tPtrs []int - released bool - - snapGPI int - snapSeenKey bool - snapGPOverlappedBytes int64 - snapTPtrs []int -} - -func (c *compaction) save() { - c.snapGPI = c.gpi - c.snapSeenKey = c.seenKey - c.snapGPOverlappedBytes = c.gpOverlappedBytes - c.snapTPtrs = append(c.snapTPtrs[:0], c.tPtrs...) -} - -func (c *compaction) restore() { - c.gpi = c.snapGPI - c.seenKey = c.snapSeenKey - c.gpOverlappedBytes = c.snapGPOverlappedBytes - c.tPtrs = append(c.tPtrs[:0], c.snapTPtrs...) -} - -func (c *compaction) release() { - if !c.released { - c.released = true - c.v.release() - } -} - -// Expand compacted tables; need external synchronization. -func (c *compaction) expand() { - limit := int64(c.s.o.GetCompactionExpandLimit(c.sourceLevel)) - vt0 := c.v.levels[c.sourceLevel] - vt1 := tFiles{} - if level := c.sourceLevel + 1; level < len(c.v.levels) { - vt1 = c.v.levels[level] - } - - t0, t1 := c.levels[0], c.levels[1] - imin, imax := t0.getRange(c.s.icmp) - // We expand t0 here just incase ukey hop across tables. - t0 = vt0.getOverlaps(t0, c.s.icmp, imin.ukey(), imax.ukey(), c.sourceLevel == 0) - if len(t0) != len(c.levels[0]) { - imin, imax = t0.getRange(c.s.icmp) - } - t1 = vt1.getOverlaps(t1, c.s.icmp, imin.ukey(), imax.ukey(), false) - // Get entire range covered by compaction. - amin, amax := append(t0, t1...).getRange(c.s.icmp) - - // See if we can grow the number of inputs in "sourceLevel" without - // changing the number of "sourceLevel+1" files we pick up. - if len(t1) > 0 { - exp0 := vt0.getOverlaps(nil, c.s.icmp, amin.ukey(), amax.ukey(), c.sourceLevel == 0) - if len(exp0) > len(t0) && t1.size()+exp0.size() < limit { - xmin, xmax := exp0.getRange(c.s.icmp) - exp1 := vt1.getOverlaps(nil, c.s.icmp, xmin.ukey(), xmax.ukey(), false) - if len(exp1) == len(t1) { - c.s.logf("table@compaction expanding L%d+L%d (F·%d S·%s)+(F·%d S·%s) -> (F·%d S·%s)+(F·%d S·%s)", - c.sourceLevel, c.sourceLevel+1, len(t0), shortenb(int(t0.size())), len(t1), shortenb(int(t1.size())), - len(exp0), shortenb(int(exp0.size())), len(exp1), shortenb(int(exp1.size()))) - imin, imax = xmin, xmax - t0, t1 = exp0, exp1 - amin, amax = append(t0, t1...).getRange(c.s.icmp) - } - } - } - - // Compute the set of grandparent files that overlap this compaction - // (parent == sourceLevel+1; grandparent == sourceLevel+2) - if level := c.sourceLevel + 2; level < len(c.v.levels) { - c.gp = c.v.levels[level].getOverlaps(c.gp, c.s.icmp, amin.ukey(), amax.ukey(), false) - } - - c.levels[0], c.levels[1] = t0, t1 - c.imin, c.imax = imin, imax -} - -// Check whether compaction is trivial. -func (c *compaction) trivial() bool { - return len(c.levels[0]) == 1 && len(c.levels[1]) == 0 && c.gp.size() <= c.maxGPOverlaps -} - -func (c *compaction) baseLevelForKey(ukey []byte) bool { - for level := c.sourceLevel + 2; level < len(c.v.levels); level++ { - tables := c.v.levels[level] - for c.tPtrs[level] < len(tables) { - t := tables[c.tPtrs[level]] - if c.s.icmp.uCompare(ukey, t.imax.ukey()) <= 0 { - // We've advanced far enough. - if c.s.icmp.uCompare(ukey, t.imin.ukey()) >= 0 { - // Key falls in this file's range, so definitely not base level. - return false - } - break - } - c.tPtrs[level]++ - } - } - return true -} - -func (c *compaction) shouldStopBefore(ikey internalKey) bool { - for ; c.gpi < len(c.gp); c.gpi++ { - gp := c.gp[c.gpi] - if c.s.icmp.Compare(ikey, gp.imax) <= 0 { - break - } - if c.seenKey { - c.gpOverlappedBytes += gp.size - } - } - c.seenKey = true - - if c.gpOverlappedBytes > c.maxGPOverlaps { - // Too much overlap for current output; start new output. - c.gpOverlappedBytes = 0 - return true - } - return false -} - -// Creates an iterator. -func (c *compaction) newIterator() iterator.Iterator { - // Creates iterator slice. - icap := len(c.levels) - if c.sourceLevel == 0 { - // Special case for level-0. - icap = len(c.levels[0]) + 1 - } - its := make([]iterator.Iterator, 0, icap) - - // Options. - ro := &opt.ReadOptions{ - DontFillCache: true, - Strict: opt.StrictOverride, - } - strict := c.s.o.GetStrict(opt.StrictCompaction) - if strict { - ro.Strict |= opt.StrictReader - } - - for i, tables := range c.levels { - if len(tables) == 0 { - continue - } - - // Level-0 is not sorted and may overlaps each other. - if c.sourceLevel+i == 0 { - for _, t := range tables { - its = append(its, c.s.tops.newIterator(t, nil, ro)) - } - } else { - it := iterator.NewIndexedIterator(tables.newIndexIterator(c.s.tops, c.s.icmp, nil, ro), strict) - its = append(its, it) - } - } - - return iterator.NewMergedIterator(its, c.s.icmp, strict) -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/session_record.go b/vendor/github.com/syndtr/goleveldb/leveldb/session_record.go deleted file mode 100644 index 854e1aa..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/session_record.go +++ /dev/null @@ -1,323 +0,0 @@ -// Copyright (c) 2012, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package leveldb - -import ( - "bufio" - "encoding/binary" - "io" - "strings" - - "github.com/syndtr/goleveldb/leveldb/errors" - "github.com/syndtr/goleveldb/leveldb/storage" -) - -type byteReader interface { - io.Reader - io.ByteReader -} - -// These numbers are written to disk and should not be changed. -const ( - recComparer = 1 - recJournalNum = 2 - recNextFileNum = 3 - recSeqNum = 4 - recCompPtr = 5 - recDelTable = 6 - recAddTable = 7 - // 8 was used for large value refs - recPrevJournalNum = 9 -) - -type cpRecord struct { - level int - ikey internalKey -} - -type atRecord struct { - level int - num int64 - size int64 - imin internalKey - imax internalKey -} - -type dtRecord struct { - level int - num int64 -} - -type sessionRecord struct { - hasRec int - comparer string - journalNum int64 - prevJournalNum int64 - nextFileNum int64 - seqNum uint64 - compPtrs []cpRecord - addedTables []atRecord - deletedTables []dtRecord - - scratch [binary.MaxVarintLen64]byte - err error -} - -func (p *sessionRecord) has(rec int) bool { - return p.hasRec&(1< -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package leveldb - -import ( - "bytes" - "testing" -) - -func decodeEncode(v *sessionRecord) (res bool, err error) { - b := new(bytes.Buffer) - err = v.encode(b) - if err != nil { - return - } - v2 := &sessionRecord{} - err = v.decode(b) - if err != nil { - return - } - b2 := new(bytes.Buffer) - err = v2.encode(b2) - if err != nil { - return - } - return bytes.Equal(b.Bytes(), b2.Bytes()), nil -} - -func TestSessionRecord_EncodeDecode(t *testing.T) { - big := int64(1) << 50 - v := &sessionRecord{} - i := int64(0) - test := func() { - res, err := decodeEncode(v) - if err != nil { - t.Fatalf("error when testing encode/decode sessionRecord: %v", err) - } - if !res { - t.Error("encode/decode test failed at iteration:", i) - } - } - - for ; i < 4; i++ { - test() - v.addTable(3, big+300+i, big+400+i, - makeInternalKey(nil, []byte("foo"), uint64(big+500+1), keyTypeVal), - makeInternalKey(nil, []byte("zoo"), uint64(big+600+1), keyTypeDel)) - v.delTable(4, big+700+i) - v.addCompPtr(int(i), makeInternalKey(nil, []byte("x"), uint64(big+900+1), keyTypeVal)) - } - - v.setComparer("foo") - v.setJournalNum(big + 100) - v.setPrevJournalNum(big + 99) - v.setNextFileNum(big + 200) - v.setSeqNum(uint64(big + 1000)) - test() -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/session_util.go b/vendor/github.com/syndtr/goleveldb/leveldb/session_util.go deleted file mode 100644 index 9232893..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/session_util.go +++ /dev/null @@ -1,271 +0,0 @@ -// Copyright (c) 2012, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package leveldb - -import ( - "fmt" - "sync/atomic" - - "github.com/syndtr/goleveldb/leveldb/journal" - "github.com/syndtr/goleveldb/leveldb/storage" -) - -// Logging. - -type dropper struct { - s *session - fd storage.FileDesc -} - -func (d dropper) Drop(err error) { - if e, ok := err.(*journal.ErrCorrupted); ok { - d.s.logf("journal@drop %s-%d S·%s %q", d.fd.Type, d.fd.Num, shortenb(e.Size), e.Reason) - } else { - d.s.logf("journal@drop %s-%d %q", d.fd.Type, d.fd.Num, err) - } -} - -func (s *session) log(v ...interface{}) { s.stor.Log(fmt.Sprint(v...)) } -func (s *session) logf(format string, v ...interface{}) { s.stor.Log(fmt.Sprintf(format, v...)) } - -// File utils. - -func (s *session) newTemp() storage.FileDesc { - num := atomic.AddInt64(&s.stTempFileNum, 1) - 1 - return storage.FileDesc{storage.TypeTemp, num} -} - -func (s *session) addFileRef(fd storage.FileDesc, ref int) int { - ref += s.fileRef[fd.Num] - if ref > 0 { - s.fileRef[fd.Num] = ref - } else if ref == 0 { - delete(s.fileRef, fd.Num) - } else { - panic(fmt.Sprintf("negative ref: %v", fd)) - } - return ref -} - -// Session state. - -// Get current version. This will incr version ref, must call -// version.release (exactly once) after use. -func (s *session) version() *version { - s.vmu.Lock() - defer s.vmu.Unlock() - s.stVersion.incref() - return s.stVersion -} - -func (s *session) tLen(level int) int { - s.vmu.Lock() - defer s.vmu.Unlock() - return s.stVersion.tLen(level) -} - -// Set current version to v. -func (s *session) setVersion(v *version) { - s.vmu.Lock() - defer s.vmu.Unlock() - // Hold by session. It is important to call this first before releasing - // current version, otherwise the still used files might get released. - v.incref() - if s.stVersion != nil { - // Release current version. - s.stVersion.releaseNB() - } - s.stVersion = v -} - -// Get current unused file number. -func (s *session) nextFileNum() int64 { - return atomic.LoadInt64(&s.stNextFileNum) -} - -// Set current unused file number to num. -func (s *session) setNextFileNum(num int64) { - atomic.StoreInt64(&s.stNextFileNum, num) -} - -// Mark file number as used. -func (s *session) markFileNum(num int64) { - nextFileNum := num + 1 - for { - old, x := s.stNextFileNum, nextFileNum - if old > x { - x = old - } - if atomic.CompareAndSwapInt64(&s.stNextFileNum, old, x) { - break - } - } -} - -// Allocate a file number. -func (s *session) allocFileNum() int64 { - return atomic.AddInt64(&s.stNextFileNum, 1) - 1 -} - -// Reuse given file number. -func (s *session) reuseFileNum(num int64) { - for { - old, x := s.stNextFileNum, num - if old != x+1 { - x = old - } - if atomic.CompareAndSwapInt64(&s.stNextFileNum, old, x) { - break - } - } -} - -// Set compaction ptr at given level; need external synchronization. -func (s *session) setCompPtr(level int, ik internalKey) { - if level >= len(s.stCompPtrs) { - newCompPtrs := make([]internalKey, level+1) - copy(newCompPtrs, s.stCompPtrs) - s.stCompPtrs = newCompPtrs - } - s.stCompPtrs[level] = append(internalKey{}, ik...) -} - -// Get compaction ptr at given level; need external synchronization. -func (s *session) getCompPtr(level int) internalKey { - if level >= len(s.stCompPtrs) { - return nil - } - return s.stCompPtrs[level] -} - -// Manifest related utils. - -// Fill given session record obj with current states; need external -// synchronization. -func (s *session) fillRecord(r *sessionRecord, snapshot bool) { - r.setNextFileNum(s.nextFileNum()) - - if snapshot { - if !r.has(recJournalNum) { - r.setJournalNum(s.stJournalNum) - } - - if !r.has(recSeqNum) { - r.setSeqNum(s.stSeqNum) - } - - for level, ik := range s.stCompPtrs { - if ik != nil { - r.addCompPtr(level, ik) - } - } - - r.setComparer(s.icmp.uName()) - } -} - -// Mark if record has been committed, this will update session state; -// need external synchronization. -func (s *session) recordCommited(rec *sessionRecord) { - if rec.has(recJournalNum) { - s.stJournalNum = rec.journalNum - } - - if rec.has(recPrevJournalNum) { - s.stPrevJournalNum = rec.prevJournalNum - } - - if rec.has(recSeqNum) { - s.stSeqNum = rec.seqNum - } - - for _, r := range rec.compPtrs { - s.setCompPtr(r.level, internalKey(r.ikey)) - } -} - -// Create a new manifest file; need external synchronization. -func (s *session) newManifest(rec *sessionRecord, v *version) (err error) { - fd := storage.FileDesc{storage.TypeManifest, s.allocFileNum()} - writer, err := s.stor.Create(fd) - if err != nil { - return - } - jw := journal.NewWriter(writer) - - if v == nil { - v = s.version() - defer v.release() - } - if rec == nil { - rec = &sessionRecord{} - } - s.fillRecord(rec, true) - v.fillRecord(rec) - - defer func() { - if err == nil { - s.recordCommited(rec) - if s.manifest != nil { - s.manifest.Close() - } - if s.manifestWriter != nil { - s.manifestWriter.Close() - } - if !s.manifestFd.Zero() { - s.stor.Remove(s.manifestFd) - } - s.manifestFd = fd - s.manifestWriter = writer - s.manifest = jw - } else { - writer.Close() - s.stor.Remove(fd) - s.reuseFileNum(fd.Num) - } - }() - - w, err := jw.Next() - if err != nil { - return - } - err = rec.encode(w) - if err != nil { - return - } - err = jw.Flush() - if err != nil { - return - } - err = s.stor.SetMeta(fd) - return -} - -// Flush record to disk. -func (s *session) flushManifest(rec *sessionRecord) (err error) { - s.fillRecord(rec, false) - w, err := s.manifest.Next() - if err != nil { - return - } - err = rec.encode(w) - if err != nil { - return - } - err = s.manifest.Flush() - if err != nil { - return - } - if !s.o.GetNoSync() { - err = s.manifestWriter.Sync() - if err != nil { - return - } - } - s.recordCommited(rec) - return -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/storage/file_storage.go b/vendor/github.com/syndtr/goleveldb/leveldb/storage/file_storage.go deleted file mode 100644 index e53434c..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/storage/file_storage.go +++ /dev/null @@ -1,583 +0,0 @@ -// Copyright (c) 2012, Suryandaru Triandana -// All rights reservefs. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package storage - -import ( - "errors" - "fmt" - "io/ioutil" - "os" - "path/filepath" - "runtime" - "strconv" - "strings" - "sync" - "time" -) - -var ( - errFileOpen = errors.New("leveldb/storage: file still open") - errReadOnly = errors.New("leveldb/storage: storage is read-only") -) - -type fileLock interface { - release() error -} - -type fileStorageLock struct { - fs *fileStorage -} - -func (lock *fileStorageLock) Unlock() { - if lock.fs != nil { - lock.fs.mu.Lock() - defer lock.fs.mu.Unlock() - if lock.fs.slock == lock { - lock.fs.slock = nil - } - } -} - -const logSizeThreshold = 1024 * 1024 // 1 MiB - -// fileStorage is a file-system backed storage. -type fileStorage struct { - path string - readOnly bool - - mu sync.Mutex - flock fileLock - slock *fileStorageLock - logw *os.File - logSize int64 - buf []byte - // Opened file counter; if open < 0 means closed. - open int - day int -} - -// OpenFile returns a new filesytem-backed storage implementation with the given -// path. This also acquire a file lock, so any subsequent attempt to open the -// same path will fail. -// -// The storage must be closed after use, by calling Close method. -func OpenFile(path string, readOnly bool) (Storage, error) { - if fi, err := os.Stat(path); err == nil { - if !fi.IsDir() { - return nil, fmt.Errorf("leveldb/storage: open %s: not a directory", path) - } - } else if os.IsNotExist(err) && !readOnly { - if err := os.MkdirAll(path, 0755); err != nil { - return nil, err - } - } else { - return nil, err - } - - flock, err := newFileLock(filepath.Join(path, "LOCK"), readOnly) - if err != nil { - return nil, err - } - - defer func() { - if err != nil { - flock.release() - } - }() - - var ( - logw *os.File - logSize int64 - ) - if !readOnly { - logw, err = os.OpenFile(filepath.Join(path, "LOG"), os.O_WRONLY|os.O_CREATE, 0644) - if err != nil { - return nil, err - } - logSize, err = logw.Seek(0, os.SEEK_END) - if err != nil { - logw.Close() - return nil, err - } - } - - fs := &fileStorage{ - path: path, - readOnly: readOnly, - flock: flock, - logw: logw, - logSize: logSize, - } - runtime.SetFinalizer(fs, (*fileStorage).Close) - return fs, nil -} - -func (fs *fileStorage) Lock() (Locker, error) { - fs.mu.Lock() - defer fs.mu.Unlock() - if fs.open < 0 { - return nil, ErrClosed - } - if fs.readOnly { - return &fileStorageLock{}, nil - } - if fs.slock != nil { - return nil, ErrLocked - } - fs.slock = &fileStorageLock{fs: fs} - return fs.slock, nil -} - -func itoa(buf []byte, i int, wid int) []byte { - u := uint(i) - if u == 0 && wid <= 1 { - return append(buf, '0') - } - - // Assemble decimal in reverse order. - var b [32]byte - bp := len(b) - for ; u > 0 || wid > 0; u /= 10 { - bp-- - wid-- - b[bp] = byte(u%10) + '0' - } - return append(buf, b[bp:]...) -} - -func (fs *fileStorage) printDay(t time.Time) { - if fs.day == t.Day() { - return - } - fs.day = t.Day() - fs.logw.Write([]byte("=============== " + t.Format("Jan 2, 2006 (MST)") + " ===============\n")) -} - -func (fs *fileStorage) doLog(t time.Time, str string) { - if fs.logSize > logSizeThreshold { - // Rotate log file. - fs.logw.Close() - fs.logw = nil - fs.logSize = 0 - rename(filepath.Join(fs.path, "LOG"), filepath.Join(fs.path, "LOG.old")) - } - if fs.logw == nil { - var err error - fs.logw, err = os.OpenFile(filepath.Join(fs.path, "LOG"), os.O_WRONLY|os.O_CREATE, 0644) - if err != nil { - return - } - // Force printDay on new log file. - fs.day = 0 - } - fs.printDay(t) - hour, min, sec := t.Clock() - msec := t.Nanosecond() / 1e3 - // time - fs.buf = itoa(fs.buf[:0], hour, 2) - fs.buf = append(fs.buf, ':') - fs.buf = itoa(fs.buf, min, 2) - fs.buf = append(fs.buf, ':') - fs.buf = itoa(fs.buf, sec, 2) - fs.buf = append(fs.buf, '.') - fs.buf = itoa(fs.buf, msec, 6) - fs.buf = append(fs.buf, ' ') - // write - fs.buf = append(fs.buf, []byte(str)...) - fs.buf = append(fs.buf, '\n') - fs.logw.Write(fs.buf) -} - -func (fs *fileStorage) Log(str string) { - if !fs.readOnly { - t := time.Now() - fs.mu.Lock() - defer fs.mu.Unlock() - if fs.open < 0 { - return - } - fs.doLog(t, str) - } -} - -func (fs *fileStorage) log(str string) { - if !fs.readOnly { - fs.doLog(time.Now(), str) - } -} - -func (fs *fileStorage) SetMeta(fd FileDesc) (err error) { - if !FileDescOk(fd) { - return ErrInvalidFile - } - if fs.readOnly { - return errReadOnly - } - - fs.mu.Lock() - defer fs.mu.Unlock() - if fs.open < 0 { - return ErrClosed - } - defer func() { - if err != nil { - fs.log(fmt.Sprintf("CURRENT: %v", err)) - } - }() - path := fmt.Sprintf("%s.%d", filepath.Join(fs.path, "CURRENT"), fd.Num) - w, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) - if err != nil { - return - } - _, err = fmt.Fprintln(w, fsGenName(fd)) - // Close the file first. - if cerr := w.Close(); cerr != nil { - fs.log(fmt.Sprintf("close CURRENT.%d: %v", fd.Num, cerr)) - } - if err != nil { - return - } - return rename(path, filepath.Join(fs.path, "CURRENT")) -} - -func (fs *fileStorage) GetMeta() (fd FileDesc, err error) { - fs.mu.Lock() - defer fs.mu.Unlock() - if fs.open < 0 { - return FileDesc{}, ErrClosed - } - dir, err := os.Open(fs.path) - if err != nil { - return - } - names, err := dir.Readdirnames(0) - // Close the dir first before checking for Readdirnames error. - if ce := dir.Close(); ce != nil { - fs.log(fmt.Sprintf("close dir: %v", ce)) - } - if err != nil { - return - } - // Find latest CURRENT file. - var rem []string - var pend bool - var cerr error - for _, name := range names { - if strings.HasPrefix(name, "CURRENT") { - pend1 := len(name) > 7 - var pendNum int64 - // Make sure it is valid name for a CURRENT file, otherwise skip it. - if pend1 { - if name[7] != '.' || len(name) < 9 { - fs.log(fmt.Sprintf("skipping %s: invalid file name", name)) - continue - } - var e1 error - if pendNum, e1 = strconv.ParseInt(name[8:], 10, 0); e1 != nil { - fs.log(fmt.Sprintf("skipping %s: invalid file num: %v", name, e1)) - continue - } - } - path := filepath.Join(fs.path, name) - r, e1 := os.OpenFile(path, os.O_RDONLY, 0) - if e1 != nil { - return FileDesc{}, e1 - } - b, e1 := ioutil.ReadAll(r) - if e1 != nil { - r.Close() - return FileDesc{}, e1 - } - var fd1 FileDesc - if len(b) < 1 || b[len(b)-1] != '\n' || !fsParseNamePtr(string(b[:len(b)-1]), &fd1) { - fs.log(fmt.Sprintf("skipping %s: corrupted or incomplete", name)) - if pend1 { - rem = append(rem, name) - } - if !pend1 || cerr == nil { - metaFd, _ := fsParseName(name) - cerr = &ErrCorrupted{ - Fd: metaFd, - Err: errors.New("leveldb/storage: corrupted or incomplete meta file"), - } - } - } else if pend1 && pendNum != fd1.Num { - fs.log(fmt.Sprintf("skipping %s: inconsistent pending-file num: %d vs %d", name, pendNum, fd1.Num)) - rem = append(rem, name) - } else if fd1.Num < fd.Num { - fs.log(fmt.Sprintf("skipping %s: obsolete", name)) - if pend1 { - rem = append(rem, name) - } - } else { - fd = fd1 - pend = pend1 - } - if err := r.Close(); err != nil { - fs.log(fmt.Sprintf("close %s: %v", name, err)) - } - } - } - // Don't remove any files if there is no valid CURRENT file. - if fd.Zero() { - if cerr != nil { - err = cerr - } else { - err = os.ErrNotExist - } - return - } - if !fs.readOnly { - // Rename pending CURRENT file to an effective CURRENT. - if pend { - path := fmt.Sprintf("%s.%d", filepath.Join(fs.path, "CURRENT"), fd.Num) - if err := rename(path, filepath.Join(fs.path, "CURRENT")); err != nil { - fs.log(fmt.Sprintf("CURRENT.%d -> CURRENT: %v", fd.Num, err)) - } - } - // Remove obsolete or incomplete pending CURRENT files. - for _, name := range rem { - path := filepath.Join(fs.path, name) - if err := os.Remove(path); err != nil { - fs.log(fmt.Sprintf("remove %s: %v", name, err)) - } - } - } - return -} - -func (fs *fileStorage) List(ft FileType) (fds []FileDesc, err error) { - fs.mu.Lock() - defer fs.mu.Unlock() - if fs.open < 0 { - return nil, ErrClosed - } - dir, err := os.Open(fs.path) - if err != nil { - return - } - names, err := dir.Readdirnames(0) - // Close the dir first before checking for Readdirnames error. - if cerr := dir.Close(); cerr != nil { - fs.log(fmt.Sprintf("close dir: %v", cerr)) - } - if err == nil { - for _, name := range names { - if fd, ok := fsParseName(name); ok && fd.Type&ft != 0 { - fds = append(fds, fd) - } - } - } - return -} - -func (fs *fileStorage) Open(fd FileDesc) (Reader, error) { - if !FileDescOk(fd) { - return nil, ErrInvalidFile - } - - fs.mu.Lock() - defer fs.mu.Unlock() - if fs.open < 0 { - return nil, ErrClosed - } - of, err := os.OpenFile(filepath.Join(fs.path, fsGenName(fd)), os.O_RDONLY, 0) - if err != nil { - if fsHasOldName(fd) && os.IsNotExist(err) { - of, err = os.OpenFile(filepath.Join(fs.path, fsGenOldName(fd)), os.O_RDONLY, 0) - if err == nil { - goto ok - } - } - return nil, err - } -ok: - fs.open++ - return &fileWrap{File: of, fs: fs, fd: fd}, nil -} - -func (fs *fileStorage) Create(fd FileDesc) (Writer, error) { - if !FileDescOk(fd) { - return nil, ErrInvalidFile - } - if fs.readOnly { - return nil, errReadOnly - } - - fs.mu.Lock() - defer fs.mu.Unlock() - if fs.open < 0 { - return nil, ErrClosed - } - of, err := os.OpenFile(filepath.Join(fs.path, fsGenName(fd)), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644) - if err != nil { - return nil, err - } - fs.open++ - return &fileWrap{File: of, fs: fs, fd: fd}, nil -} - -func (fs *fileStorage) Remove(fd FileDesc) error { - if !FileDescOk(fd) { - return ErrInvalidFile - } - if fs.readOnly { - return errReadOnly - } - - fs.mu.Lock() - defer fs.mu.Unlock() - if fs.open < 0 { - return ErrClosed - } - err := os.Remove(filepath.Join(fs.path, fsGenName(fd))) - if err != nil { - if fsHasOldName(fd) && os.IsNotExist(err) { - if e1 := os.Remove(filepath.Join(fs.path, fsGenOldName(fd))); !os.IsNotExist(e1) { - fs.log(fmt.Sprintf("remove %s: %v (old name)", fd, err)) - err = e1 - } - } else { - fs.log(fmt.Sprintf("remove %s: %v", fd, err)) - } - } - return err -} - -func (fs *fileStorage) Rename(oldfd, newfd FileDesc) error { - if !FileDescOk(oldfd) || !FileDescOk(newfd) { - return ErrInvalidFile - } - if oldfd == newfd { - return nil - } - if fs.readOnly { - return errReadOnly - } - - fs.mu.Lock() - defer fs.mu.Unlock() - if fs.open < 0 { - return ErrClosed - } - return rename(filepath.Join(fs.path, fsGenName(oldfd)), filepath.Join(fs.path, fsGenName(newfd))) -} - -func (fs *fileStorage) Close() error { - fs.mu.Lock() - defer fs.mu.Unlock() - if fs.open < 0 { - return ErrClosed - } - // Clear the finalizer. - runtime.SetFinalizer(fs, nil) - - if fs.open > 0 { - fs.log(fmt.Sprintf("close: warning, %d files still open", fs.open)) - } - fs.open = -1 - if fs.logw != nil { - fs.logw.Close() - } - return fs.flock.release() -} - -type fileWrap struct { - *os.File - fs *fileStorage - fd FileDesc - closed bool -} - -func (fw *fileWrap) Sync() error { - if err := fw.File.Sync(); err != nil { - return err - } - if fw.fd.Type == TypeManifest { - // Also sync parent directory if file type is manifest. - // See: https://code.google.com/p/leveldb/issues/detail?id=190. - if err := syncDir(fw.fs.path); err != nil { - fw.fs.log(fmt.Sprintf("syncDir: %v", err)) - return err - } - } - return nil -} - -func (fw *fileWrap) Close() error { - fw.fs.mu.Lock() - defer fw.fs.mu.Unlock() - if fw.closed { - return ErrClosed - } - fw.closed = true - fw.fs.open-- - err := fw.File.Close() - if err != nil { - fw.fs.log(fmt.Sprintf("close %s: %v", fw.fd, err)) - } - return err -} - -func fsGenName(fd FileDesc) string { - switch fd.Type { - case TypeManifest: - return fmt.Sprintf("MANIFEST-%06d", fd.Num) - case TypeJournal: - return fmt.Sprintf("%06d.log", fd.Num) - case TypeTable: - return fmt.Sprintf("%06d.ldb", fd.Num) - case TypeTemp: - return fmt.Sprintf("%06d.tmp", fd.Num) - default: - panic("invalid file type") - } -} - -func fsHasOldName(fd FileDesc) bool { - return fd.Type == TypeTable -} - -func fsGenOldName(fd FileDesc) string { - switch fd.Type { - case TypeTable: - return fmt.Sprintf("%06d.sst", fd.Num) - } - return fsGenName(fd) -} - -func fsParseName(name string) (fd FileDesc, ok bool) { - var tail string - _, err := fmt.Sscanf(name, "%d.%s", &fd.Num, &tail) - if err == nil { - switch tail { - case "log": - fd.Type = TypeJournal - case "ldb", "sst": - fd.Type = TypeTable - case "tmp": - fd.Type = TypeTemp - default: - return - } - return fd, true - } - n, _ := fmt.Sscanf(name, "MANIFEST-%d%s", &fd.Num, &tail) - if n == 1 { - fd.Type = TypeManifest - return fd, true - } - return -} - -func fsParseNamePtr(name string, fd *FileDesc) bool { - _fd, ok := fsParseName(name) - if fd != nil { - *fd = _fd - } - return ok -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/storage/file_storage_nacl.go b/vendor/github.com/syndtr/goleveldb/leveldb/storage/file_storage_nacl.go deleted file mode 100644 index 5545aee..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/storage/file_storage_nacl.go +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright (c) 2012, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -// +build nacl - -package storage - -import ( - "os" - "syscall" -) - -func newFileLock(path string, readOnly bool) (fl fileLock, err error) { - return nil, syscall.ENOTSUP -} - -func setFileLock(f *os.File, readOnly, lock bool) error { - return syscall.ENOTSUP -} - -func rename(oldpath, newpath string) error { - return syscall.ENOTSUP -} - -func isErrInvalid(err error) bool { - return false -} - -func syncDir(name string) error { - return syscall.ENOTSUP -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/storage/file_storage_plan9.go b/vendor/github.com/syndtr/goleveldb/leveldb/storage/file_storage_plan9.go deleted file mode 100644 index bab62bf..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/storage/file_storage_plan9.go +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright (c) 2012, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package storage - -import ( - "os" - "path/filepath" -) - -type plan9FileLock struct { - f *os.File -} - -func (fl *plan9FileLock) release() error { - return fl.f.Close() -} - -func newFileLock(path string, readOnly bool) (fl fileLock, err error) { - var ( - flag int - perm os.FileMode - ) - if readOnly { - flag = os.O_RDONLY - } else { - flag = os.O_RDWR - perm = os.ModeExclusive - } - f, err := os.OpenFile(path, flag, perm) - if os.IsNotExist(err) { - f, err = os.OpenFile(path, flag|os.O_CREATE, perm|0644) - } - if err != nil { - return - } - fl = &plan9FileLock{f: f} - return -} - -func rename(oldpath, newpath string) error { - if _, err := os.Stat(newpath); err == nil { - if err := os.Remove(newpath); err != nil { - return err - } - } - - _, fname := filepath.Split(newpath) - return os.Rename(oldpath, fname) -} - -func syncDir(name string) error { - f, err := os.Open(name) - if err != nil { - return err - } - defer f.Close() - if err := f.Sync(); err != nil { - return err - } - return nil -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/storage/file_storage_solaris.go b/vendor/github.com/syndtr/goleveldb/leveldb/storage/file_storage_solaris.go deleted file mode 100644 index 79901ee..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/storage/file_storage_solaris.go +++ /dev/null @@ -1,81 +0,0 @@ -// Copyright (c) 2012, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -// +build solaris - -package storage - -import ( - "os" - "syscall" -) - -type unixFileLock struct { - f *os.File -} - -func (fl *unixFileLock) release() error { - if err := setFileLock(fl.f, false, false); err != nil { - return err - } - return fl.f.Close() -} - -func newFileLock(path string, readOnly bool) (fl fileLock, err error) { - var flag int - if readOnly { - flag = os.O_RDONLY - } else { - flag = os.O_RDWR - } - f, err := os.OpenFile(path, flag, 0) - if os.IsNotExist(err) { - f, err = os.OpenFile(path, flag|os.O_CREATE, 0644) - } - if err != nil { - return - } - err = setFileLock(f, readOnly, true) - if err != nil { - f.Close() - return - } - fl = &unixFileLock{f: f} - return -} - -func setFileLock(f *os.File, readOnly, lock bool) error { - flock := syscall.Flock_t{ - Type: syscall.F_UNLCK, - Start: 0, - Len: 0, - Whence: 1, - } - if lock { - if readOnly { - flock.Type = syscall.F_RDLCK - } else { - flock.Type = syscall.F_WRLCK - } - } - return syscall.FcntlFlock(f.Fd(), syscall.F_SETLK, &flock) -} - -func rename(oldpath, newpath string) error { - return os.Rename(oldpath, newpath) -} - -func syncDir(name string) error { - f, err := os.Open(name) - if err != nil { - return err - } - defer f.Close() - if err := f.Sync(); err != nil { - return err - } - return nil -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/storage/file_storage_test.go b/vendor/github.com/syndtr/goleveldb/leveldb/storage/file_storage_test.go deleted file mode 100644 index 28a59ec..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/storage/file_storage_test.go +++ /dev/null @@ -1,176 +0,0 @@ -// Copyright (c) 2012, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package storage - -import ( - "fmt" - "os" - "path/filepath" - "testing" -) - -var cases = []struct { - oldName []string - name string - ftype FileType - num int64 -}{ - {nil, "000100.log", TypeJournal, 100}, - {nil, "000000.log", TypeJournal, 0}, - {[]string{"000000.sst"}, "000000.ldb", TypeTable, 0}, - {nil, "MANIFEST-000002", TypeManifest, 2}, - {nil, "MANIFEST-000007", TypeManifest, 7}, - {nil, "9223372036854775807.log", TypeJournal, 9223372036854775807}, - {nil, "000100.tmp", TypeTemp, 100}, -} - -var invalidCases = []string{ - "", - "foo", - "foo-dx-100.log", - ".log", - "", - "manifest", - "CURREN", - "CURRENTX", - "MANIFES", - "MANIFEST", - "MANIFEST-", - "XMANIFEST-3", - "MANIFEST-3x", - "LOC", - "LOCKx", - "LO", - "LOGx", - "18446744073709551616.log", - "184467440737095516150.log", - "100", - "100.", - "100.lop", -} - -func TestFileStorage_CreateFileName(t *testing.T) { - for _, c := range cases { - if name := fsGenName(FileDesc{c.ftype, c.num}); name != c.name { - t.Errorf("invalid filename got '%s', want '%s'", name, c.name) - } - } -} - -func TestFileStorage_ParseFileName(t *testing.T) { - for _, c := range cases { - for _, name := range append([]string{c.name}, c.oldName...) { - fd, ok := fsParseName(name) - if !ok { - t.Errorf("cannot parse filename '%s'", name) - continue - } - if fd.Type != c.ftype { - t.Errorf("filename '%s' invalid type got '%d', want '%d'", name, fd.Type, c.ftype) - } - if fd.Num != c.num { - t.Errorf("filename '%s' invalid number got '%d', want '%d'", name, fd.Num, c.num) - } - } - } -} - -func TestFileStorage_InvalidFileName(t *testing.T) { - for _, name := range invalidCases { - if fsParseNamePtr(name, nil) { - t.Errorf("filename '%s' should be invalid", name) - } - } -} - -func TestFileStorage_Locking(t *testing.T) { - path := filepath.Join(os.TempDir(), fmt.Sprintf("goleveldb-testrwlock-%d", os.Getuid())) - if err := os.RemoveAll(path); err != nil && !os.IsNotExist(err) { - t.Fatal("RemoveAll: got error: ", err) - } - defer os.RemoveAll(path) - - p1, err := OpenFile(path, false) - if err != nil { - t.Fatal("OpenFile(1): got error: ", err) - } - - p2, err := OpenFile(path, false) - if err != nil { - t.Logf("OpenFile(2): got error: %s (expected)", err) - } else { - p2.Close() - p1.Close() - t.Fatal("OpenFile(2): expect error") - } - - p1.Close() - - p3, err := OpenFile(path, false) - if err != nil { - t.Fatal("OpenFile(3): got error: ", err) - } - defer p3.Close() - - l, err := p3.Lock() - if err != nil { - t.Fatal("storage lock failed(1): ", err) - } - _, err = p3.Lock() - if err == nil { - t.Fatal("expect error for second storage lock attempt") - } else { - t.Logf("storage lock got error: %s (expected)", err) - } - l.Unlock() - _, err = p3.Lock() - if err != nil { - t.Fatal("storage lock failed(2): ", err) - } -} - -func TestFileStorage_ReadOnlyLocking(t *testing.T) { - path := filepath.Join(os.TempDir(), fmt.Sprintf("goleveldb-testrolock-%d", os.Getuid())) - if err := os.RemoveAll(path); err != nil && !os.IsNotExist(err) { - t.Fatal("RemoveAll: got error: ", err) - } - defer os.RemoveAll(path) - - p1, err := OpenFile(path, false) - if err != nil { - t.Fatal("OpenFile(1): got error: ", err) - } - - _, err = OpenFile(path, true) - if err != nil { - t.Logf("OpenFile(2): got error: %s (expected)", err) - } else { - t.Fatal("OpenFile(2): expect error") - } - - p1.Close() - - p3, err := OpenFile(path, true) - if err != nil { - t.Fatal("OpenFile(3): got error: ", err) - } - - p4, err := OpenFile(path, true) - if err != nil { - t.Fatal("OpenFile(4): got error: ", err) - } - - _, err = OpenFile(path, false) - if err != nil { - t.Logf("OpenFile(5): got error: %s (expected)", err) - } else { - t.Fatal("OpenFile(2): expect error") - } - - p3.Close() - p4.Close() -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/storage/file_storage_unix.go b/vendor/github.com/syndtr/goleveldb/leveldb/storage/file_storage_unix.go deleted file mode 100644 index 7e29915..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/storage/file_storage_unix.go +++ /dev/null @@ -1,86 +0,0 @@ -// Copyright (c) 2012, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -// +build darwin dragonfly freebsd linux netbsd openbsd - -package storage - -import ( - "os" - "syscall" -) - -type unixFileLock struct { - f *os.File -} - -func (fl *unixFileLock) release() error { - if err := setFileLock(fl.f, false, false); err != nil { - return err - } - return fl.f.Close() -} - -func newFileLock(path string, readOnly bool) (fl fileLock, err error) { - var flag int - if readOnly { - flag = os.O_RDONLY - } else { - flag = os.O_RDWR - } - f, err := os.OpenFile(path, flag, 0) - if os.IsNotExist(err) { - f, err = os.OpenFile(path, flag|os.O_CREATE, 0644) - } - if err != nil { - return - } - err = setFileLock(f, readOnly, true) - if err != nil { - f.Close() - return - } - fl = &unixFileLock{f: f} - return -} - -func setFileLock(f *os.File, readOnly, lock bool) error { - how := syscall.LOCK_UN - if lock { - if readOnly { - how = syscall.LOCK_SH - } else { - how = syscall.LOCK_EX - } - } - return syscall.Flock(int(f.Fd()), how|syscall.LOCK_NB) -} - -func rename(oldpath, newpath string) error { - return os.Rename(oldpath, newpath) -} - -func isErrInvalid(err error) bool { - if err == os.ErrInvalid { - return true - } - if syserr, ok := err.(*os.SyscallError); ok && syserr.Err == syscall.EINVAL { - return true - } - return false -} - -func syncDir(name string) error { - f, err := os.Open(name) - if err != nil { - return err - } - defer f.Close() - if err := f.Sync(); err != nil && !isErrInvalid(err) { - return err - } - return nil -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/storage/file_storage_windows.go b/vendor/github.com/syndtr/goleveldb/leveldb/storage/file_storage_windows.go deleted file mode 100644 index 899335f..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/storage/file_storage_windows.go +++ /dev/null @@ -1,78 +0,0 @@ -// Copyright (c) 2013, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package storage - -import ( - "syscall" - "unsafe" -) - -var ( - modkernel32 = syscall.NewLazyDLL("kernel32.dll") - - procMoveFileExW = modkernel32.NewProc("MoveFileExW") -) - -const ( - _MOVEFILE_REPLACE_EXISTING = 1 -) - -type windowsFileLock struct { - fd syscall.Handle -} - -func (fl *windowsFileLock) release() error { - return syscall.Close(fl.fd) -} - -func newFileLock(path string, readOnly bool) (fl fileLock, err error) { - pathp, err := syscall.UTF16PtrFromString(path) - if err != nil { - return - } - var access, shareMode uint32 - if readOnly { - access = syscall.GENERIC_READ - shareMode = syscall.FILE_SHARE_READ - } else { - access = syscall.GENERIC_READ | syscall.GENERIC_WRITE - } - fd, err := syscall.CreateFile(pathp, access, shareMode, nil, syscall.OPEN_EXISTING, syscall.FILE_ATTRIBUTE_NORMAL, 0) - if err == syscall.ERROR_FILE_NOT_FOUND { - fd, err = syscall.CreateFile(pathp, access, shareMode, nil, syscall.OPEN_ALWAYS, syscall.FILE_ATTRIBUTE_NORMAL, 0) - } - if err != nil { - return - } - fl = &windowsFileLock{fd: fd} - return -} - -func moveFileEx(from *uint16, to *uint16, flags uint32) error { - r1, _, e1 := syscall.Syscall(procMoveFileExW.Addr(), 3, uintptr(unsafe.Pointer(from)), uintptr(unsafe.Pointer(to)), uintptr(flags)) - if r1 == 0 { - if e1 != 0 { - return error(e1) - } - return syscall.EINVAL - } - return nil -} - -func rename(oldpath, newpath string) error { - from, err := syscall.UTF16PtrFromString(oldpath) - if err != nil { - return err - } - to, err := syscall.UTF16PtrFromString(newpath) - if err != nil { - return err - } - return moveFileEx(from, to, _MOVEFILE_REPLACE_EXISTING) -} - -func syncDir(name string) error { return nil } diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/storage/mem_storage.go b/vendor/github.com/syndtr/goleveldb/leveldb/storage/mem_storage.go deleted file mode 100644 index 9b0421f..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/storage/mem_storage.go +++ /dev/null @@ -1,218 +0,0 @@ -// Copyright (c) 2013, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package storage - -import ( - "bytes" - "os" - "sync" -) - -const typeShift = 3 - -type memStorageLock struct { - ms *memStorage -} - -func (lock *memStorageLock) Unlock() { - ms := lock.ms - ms.mu.Lock() - defer ms.mu.Unlock() - if ms.slock == lock { - ms.slock = nil - } - return -} - -// memStorage is a memory-backed storage. -type memStorage struct { - mu sync.Mutex - slock *memStorageLock - files map[uint64]*memFile - meta FileDesc -} - -// NewMemStorage returns a new memory-backed storage implementation. -func NewMemStorage() Storage { - return &memStorage{ - files: make(map[uint64]*memFile), - } -} - -func (ms *memStorage) Lock() (Locker, error) { - ms.mu.Lock() - defer ms.mu.Unlock() - if ms.slock != nil { - return nil, ErrLocked - } - ms.slock = &memStorageLock{ms: ms} - return ms.slock, nil -} - -func (*memStorage) Log(str string) {} - -func (ms *memStorage) SetMeta(fd FileDesc) error { - if !FileDescOk(fd) { - return ErrInvalidFile - } - - ms.mu.Lock() - ms.meta = fd - ms.mu.Unlock() - return nil -} - -func (ms *memStorage) GetMeta() (FileDesc, error) { - ms.mu.Lock() - defer ms.mu.Unlock() - if ms.meta.Zero() { - return FileDesc{}, os.ErrNotExist - } - return ms.meta, nil -} - -func (ms *memStorage) List(ft FileType) ([]FileDesc, error) { - ms.mu.Lock() - var fds []FileDesc - for x := range ms.files { - fd := unpackFile(x) - if fd.Type&ft != 0 { - fds = append(fds, fd) - } - } - ms.mu.Unlock() - return fds, nil -} - -func (ms *memStorage) Open(fd FileDesc) (Reader, error) { - if !FileDescOk(fd) { - return nil, ErrInvalidFile - } - - ms.mu.Lock() - defer ms.mu.Unlock() - if m, exist := ms.files[packFile(fd)]; exist { - if m.open { - return nil, errFileOpen - } - m.open = true - return &memReader{Reader: bytes.NewReader(m.Bytes()), ms: ms, m: m}, nil - } - return nil, os.ErrNotExist -} - -func (ms *memStorage) Create(fd FileDesc) (Writer, error) { - if !FileDescOk(fd) { - return nil, ErrInvalidFile - } - - x := packFile(fd) - ms.mu.Lock() - defer ms.mu.Unlock() - m, exist := ms.files[x] - if exist { - if m.open { - return nil, errFileOpen - } - m.Reset() - } else { - m = &memFile{} - ms.files[x] = m - } - m.open = true - return &memWriter{memFile: m, ms: ms}, nil -} - -func (ms *memStorage) Remove(fd FileDesc) error { - if !FileDescOk(fd) { - return ErrInvalidFile - } - - x := packFile(fd) - ms.mu.Lock() - defer ms.mu.Unlock() - if _, exist := ms.files[x]; exist { - delete(ms.files, x) - return nil - } - return os.ErrNotExist -} - -func (ms *memStorage) Rename(oldfd, newfd FileDesc) error { - if FileDescOk(oldfd) || FileDescOk(newfd) { - return ErrInvalidFile - } - if oldfd == newfd { - return nil - } - - oldx := packFile(oldfd) - newx := packFile(newfd) - ms.mu.Lock() - defer ms.mu.Unlock() - oldm, exist := ms.files[oldx] - if !exist { - return os.ErrNotExist - } - newm, exist := ms.files[newx] - if (exist && newm.open) || oldm.open { - return errFileOpen - } - delete(ms.files, oldx) - ms.files[newx] = oldm - return nil -} - -func (*memStorage) Close() error { return nil } - -type memFile struct { - bytes.Buffer - open bool -} - -type memReader struct { - *bytes.Reader - ms *memStorage - m *memFile - closed bool -} - -func (mr *memReader) Close() error { - mr.ms.mu.Lock() - defer mr.ms.mu.Unlock() - if mr.closed { - return ErrClosed - } - mr.m.open = false - return nil -} - -type memWriter struct { - *memFile - ms *memStorage - closed bool -} - -func (*memWriter) Sync() error { return nil } - -func (mw *memWriter) Close() error { - mw.ms.mu.Lock() - defer mw.ms.mu.Unlock() - if mw.closed { - return ErrClosed - } - mw.memFile.open = false - return nil -} - -func packFile(fd FileDesc) uint64 { - return uint64(fd.Num)<> typeShift)} -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/storage/mem_storage_test.go b/vendor/github.com/syndtr/goleveldb/leveldb/storage/mem_storage_test.go deleted file mode 100644 index fb03d30..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/storage/mem_storage_test.go +++ /dev/null @@ -1,65 +0,0 @@ -// Copyright (c) 2013, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package storage - -import ( - "bytes" - "testing" -) - -func TestMemStorage(t *testing.T) { - m := NewMemStorage() - - l, err := m.Lock() - if err != nil { - t.Fatal("storage lock failed(1): ", err) - } - _, err = m.Lock() - if err == nil { - t.Fatal("expect error for second storage lock attempt") - } else { - t.Logf("storage lock got error: %s (expected)", err) - } - l.Unlock() - _, err = m.Lock() - if err != nil { - t.Fatal("storage lock failed(2): ", err) - } - - w, err := m.Create(FileDesc{TypeTable, 1}) - if err != nil { - t.Fatal("Storage.Create: ", err) - } - w.Write([]byte("abc")) - w.Close() - if fds, _ := m.List(TypeAll); len(fds) != 1 { - t.Fatal("invalid GetFiles len") - } - buf := new(bytes.Buffer) - r, err := m.Open(FileDesc{TypeTable, 1}) - if err != nil { - t.Fatal("Open: got error: ", err) - } - buf.ReadFrom(r) - r.Close() - if got := buf.String(); got != "abc" { - t.Fatalf("Read: invalid value, want=abc got=%s", got) - } - if _, err := m.Open(FileDesc{TypeTable, 1}); err != nil { - t.Fatal("Open: got error: ", err) - } - if _, err := m.Open(FileDesc{TypeTable, 1}); err == nil { - t.Fatal("expecting error") - } - m.Remove(FileDesc{TypeTable, 1}) - if fds, _ := m.List(TypeAll); len(fds) != 0 { - t.Fatal("invalid GetFiles len", len(fds)) - } - if _, err := m.Open(FileDesc{TypeTable, 1}); err == nil { - t.Fatal("expecting error") - } -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/storage/storage.go b/vendor/github.com/syndtr/goleveldb/leveldb/storage/storage.go deleted file mode 100644 index c16bce6..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/storage/storage.go +++ /dev/null @@ -1,179 +0,0 @@ -// Copyright (c) 2012, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -// Package storage provides storage abstraction for LevelDB. -package storage - -import ( - "errors" - "fmt" - "io" -) - -// FileType represent a file type. -type FileType int - -// File types. -const ( - TypeManifest FileType = 1 << iota - TypeJournal - TypeTable - TypeTemp - - TypeAll = TypeManifest | TypeJournal | TypeTable | TypeTemp -) - -func (t FileType) String() string { - switch t { - case TypeManifest: - return "manifest" - case TypeJournal: - return "journal" - case TypeTable: - return "table" - case TypeTemp: - return "temp" - } - return fmt.Sprintf("", t) -} - -// Common error. -var ( - ErrInvalidFile = errors.New("leveldb/storage: invalid file for argument") - ErrLocked = errors.New("leveldb/storage: already locked") - ErrClosed = errors.New("leveldb/storage: closed") -) - -// ErrCorrupted is the type that wraps errors that indicate corruption of -// a file. Package storage has its own type instead of using -// errors.ErrCorrupted to prevent circular import. -type ErrCorrupted struct { - Fd FileDesc - Err error -} - -func (e *ErrCorrupted) Error() string { - if !e.Fd.Zero() { - return fmt.Sprintf("%v [file=%v]", e.Err, e.Fd) - } - return e.Err.Error() -} - -// Syncer is the interface that wraps basic Sync method. -type Syncer interface { - // Sync commits the current contents of the file to stable storage. - Sync() error -} - -// Reader is the interface that groups the basic Read, Seek, ReadAt and Close -// methods. -type Reader interface { - io.ReadSeeker - io.ReaderAt - io.Closer -} - -// Writer is the interface that groups the basic Write, Sync and Close -// methods. -type Writer interface { - io.WriteCloser - Syncer -} - -// Locker is the interface that wraps Unlock method. -type Locker interface { - Unlock() -} - -// FileDesc is a 'file descriptor'. -type FileDesc struct { - Type FileType - Num int64 -} - -func (fd FileDesc) String() string { - switch fd.Type { - case TypeManifest: - return fmt.Sprintf("MANIFEST-%06d", fd.Num) - case TypeJournal: - return fmt.Sprintf("%06d.log", fd.Num) - case TypeTable: - return fmt.Sprintf("%06d.ldb", fd.Num) - case TypeTemp: - return fmt.Sprintf("%06d.tmp", fd.Num) - default: - return fmt.Sprintf("%#x-%d", fd.Type, fd.Num) - } -} - -// Zero returns true if fd == (FileDesc{}). -func (fd FileDesc) Zero() bool { - return fd == (FileDesc{}) -} - -// FileDescOk returns true if fd is a valid 'file descriptor'. -func FileDescOk(fd FileDesc) bool { - switch fd.Type { - case TypeManifest: - case TypeJournal: - case TypeTable: - case TypeTemp: - default: - return false - } - return fd.Num >= 0 -} - -// Storage is the storage. A storage instance must be safe for concurrent use. -type Storage interface { - // Lock locks the storage. Any subsequent attempt to call Lock will fail - // until the last lock released. - // Caller should call Unlock method after use. - Lock() (Locker, error) - - // Log logs a string. This is used for logging. - // An implementation may write to a file, stdout or simply do nothing. - Log(str string) - - // SetMeta store 'file descriptor' that can later be acquired using GetMeta - // method. The 'file descriptor' should point to a valid file. - // SetMeta should be implemented in such way that changes should happen - // atomically. - SetMeta(fd FileDesc) error - - // GetMeta returns 'file descriptor' stored in meta. The 'file descriptor' - // can be updated using SetMeta method. - // Returns os.ErrNotExist if meta doesn't store any 'file descriptor', or - // 'file descriptor' point to nonexistent file. - GetMeta() (FileDesc, error) - - // List returns file descriptors that match the given file types. - // The file types may be OR'ed together. - List(ft FileType) ([]FileDesc, error) - - // Open opens file with the given 'file descriptor' read-only. - // Returns os.ErrNotExist error if the file does not exist. - // Returns ErrClosed if the underlying storage is closed. - Open(fd FileDesc) (Reader, error) - - // Create creates file with the given 'file descriptor', truncate if already - // exist and opens write-only. - // Returns ErrClosed if the underlying storage is closed. - Create(fd FileDesc) (Writer, error) - - // Remove removes file with the given 'file descriptor'. - // Returns ErrClosed if the underlying storage is closed. - Remove(fd FileDesc) error - - // Rename renames file from oldfd to newfd. - // Returns ErrClosed if the underlying storage is closed. - Rename(oldfd, newfd FileDesc) error - - // Close closes the storage. - // It is valid to call Close multiple times. Other methods should not be - // called after the storage has been closed. - Close() error -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/table.go b/vendor/github.com/syndtr/goleveldb/leveldb/table.go deleted file mode 100644 index 81d18a5..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/table.go +++ /dev/null @@ -1,529 +0,0 @@ -// Copyright (c) 2012, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package leveldb - -import ( - "fmt" - "sort" - "sync/atomic" - - "github.com/syndtr/goleveldb/leveldb/cache" - "github.com/syndtr/goleveldb/leveldb/iterator" - "github.com/syndtr/goleveldb/leveldb/opt" - "github.com/syndtr/goleveldb/leveldb/storage" - "github.com/syndtr/goleveldb/leveldb/table" - "github.com/syndtr/goleveldb/leveldb/util" -) - -// tFile holds basic information about a table. -type tFile struct { - fd storage.FileDesc - seekLeft int32 - size int64 - imin, imax internalKey -} - -// Returns true if given key is after largest key of this table. -func (t *tFile) after(icmp *iComparer, ukey []byte) bool { - return ukey != nil && icmp.uCompare(ukey, t.imax.ukey()) > 0 -} - -// Returns true if given key is before smallest key of this table. -func (t *tFile) before(icmp *iComparer, ukey []byte) bool { - return ukey != nil && icmp.uCompare(ukey, t.imin.ukey()) < 0 -} - -// Returns true if given key range overlaps with this table key range. -func (t *tFile) overlaps(icmp *iComparer, umin, umax []byte) bool { - return !t.after(icmp, umin) && !t.before(icmp, umax) -} - -// Cosumes one seek and return current seeks left. -func (t *tFile) consumeSeek() int32 { - return atomic.AddInt32(&t.seekLeft, -1) -} - -// Creates new tFile. -func newTableFile(fd storage.FileDesc, size int64, imin, imax internalKey) *tFile { - f := &tFile{ - fd: fd, - size: size, - imin: imin, - imax: imax, - } - - // We arrange to automatically compact this file after - // a certain number of seeks. Let's assume: - // (1) One seek costs 10ms - // (2) Writing or reading 1MB costs 10ms (100MB/s) - // (3) A compaction of 1MB does 25MB of IO: - // 1MB read from this level - // 10-12MB read from next level (boundaries may be misaligned) - // 10-12MB written to next level - // This implies that 25 seeks cost the same as the compaction - // of 1MB of data. I.e., one seek costs approximately the - // same as the compaction of 40KB of data. We are a little - // conservative and allow approximately one seek for every 16KB - // of data before triggering a compaction. - f.seekLeft = int32(size / 16384) - if f.seekLeft < 100 { - f.seekLeft = 100 - } - - return f -} - -func tableFileFromRecord(r atRecord) *tFile { - return newTableFile(storage.FileDesc{storage.TypeTable, r.num}, r.size, r.imin, r.imax) -} - -// tFiles hold multiple tFile. -type tFiles []*tFile - -func (tf tFiles) Len() int { return len(tf) } -func (tf tFiles) Swap(i, j int) { tf[i], tf[j] = tf[j], tf[i] } - -func (tf tFiles) nums() string { - x := "[ " - for i, f := range tf { - if i != 0 { - x += ", " - } - x += fmt.Sprint(f.fd.Num) - } - x += " ]" - return x -} - -// Returns true if i smallest key is less than j. -// This used for sort by key in ascending order. -func (tf tFiles) lessByKey(icmp *iComparer, i, j int) bool { - a, b := tf[i], tf[j] - n := icmp.Compare(a.imin, b.imin) - if n == 0 { - return a.fd.Num < b.fd.Num - } - return n < 0 -} - -// Returns true if i file number is greater than j. -// This used for sort by file number in descending order. -func (tf tFiles) lessByNum(i, j int) bool { - return tf[i].fd.Num > tf[j].fd.Num -} - -// Sorts tables by key in ascending order. -func (tf tFiles) sortByKey(icmp *iComparer) { - sort.Sort(&tFilesSortByKey{tFiles: tf, icmp: icmp}) -} - -// Sorts tables by file number in descending order. -func (tf tFiles) sortByNum() { - sort.Sort(&tFilesSortByNum{tFiles: tf}) -} - -// Returns sum of all tables size. -func (tf tFiles) size() (sum int64) { - for _, t := range tf { - sum += t.size - } - return sum -} - -// Searches smallest index of tables whose its smallest -// key is after or equal with given key. -func (tf tFiles) searchMin(icmp *iComparer, ikey internalKey) int { - return sort.Search(len(tf), func(i int) bool { - return icmp.Compare(tf[i].imin, ikey) >= 0 - }) -} - -// Searches smallest index of tables whose its largest -// key is after or equal with given key. -func (tf tFiles) searchMax(icmp *iComparer, ikey internalKey) int { - return sort.Search(len(tf), func(i int) bool { - return icmp.Compare(tf[i].imax, ikey) >= 0 - }) -} - -// Returns true if given key range overlaps with one or more -// tables key range. If unsorted is true then binary search will not be used. -func (tf tFiles) overlaps(icmp *iComparer, umin, umax []byte, unsorted bool) bool { - if unsorted { - // Check against all files. - for _, t := range tf { - if t.overlaps(icmp, umin, umax) { - return true - } - } - return false - } - - i := 0 - if len(umin) > 0 { - // Find the earliest possible internal key for min. - i = tf.searchMax(icmp, makeInternalKey(nil, umin, keyMaxSeq, keyTypeSeek)) - } - if i >= len(tf) { - // Beginning of range is after all files, so no overlap. - return false - } - return !tf[i].before(icmp, umax) -} - -// Returns tables whose its key range overlaps with given key range. -// Range will be expanded if ukey found hop across tables. -// If overlapped is true then the search will be restarted if umax -// expanded. -// The dst content will be overwritten. -func (tf tFiles) getOverlaps(dst tFiles, icmp *iComparer, umin, umax []byte, overlapped bool) tFiles { - dst = dst[:0] - for i := 0; i < len(tf); { - t := tf[i] - if t.overlaps(icmp, umin, umax) { - if umin != nil && icmp.uCompare(t.imin.ukey(), umin) < 0 { - umin = t.imin.ukey() - dst = dst[:0] - i = 0 - continue - } else if umax != nil && icmp.uCompare(t.imax.ukey(), umax) > 0 { - umax = t.imax.ukey() - // Restart search if it is overlapped. - if overlapped { - dst = dst[:0] - i = 0 - continue - } - } - - dst = append(dst, t) - } - i++ - } - - return dst -} - -// Returns tables key range. -func (tf tFiles) getRange(icmp *iComparer) (imin, imax internalKey) { - for i, t := range tf { - if i == 0 { - imin, imax = t.imin, t.imax - continue - } - if icmp.Compare(t.imin, imin) < 0 { - imin = t.imin - } - if icmp.Compare(t.imax, imax) > 0 { - imax = t.imax - } - } - - return -} - -// Creates iterator index from tables. -func (tf tFiles) newIndexIterator(tops *tOps, icmp *iComparer, slice *util.Range, ro *opt.ReadOptions) iterator.IteratorIndexer { - if slice != nil { - var start, limit int - if slice.Start != nil { - start = tf.searchMax(icmp, internalKey(slice.Start)) - } - if slice.Limit != nil { - limit = tf.searchMin(icmp, internalKey(slice.Limit)) - } else { - limit = tf.Len() - } - tf = tf[start:limit] - } - return iterator.NewArrayIndexer(&tFilesArrayIndexer{ - tFiles: tf, - tops: tops, - icmp: icmp, - slice: slice, - ro: ro, - }) -} - -// Tables iterator index. -type tFilesArrayIndexer struct { - tFiles - tops *tOps - icmp *iComparer - slice *util.Range - ro *opt.ReadOptions -} - -func (a *tFilesArrayIndexer) Search(key []byte) int { - return a.searchMax(a.icmp, internalKey(key)) -} - -func (a *tFilesArrayIndexer) Get(i int) iterator.Iterator { - if i == 0 || i == a.Len()-1 { - return a.tops.newIterator(a.tFiles[i], a.slice, a.ro) - } - return a.tops.newIterator(a.tFiles[i], nil, a.ro) -} - -// Helper type for sortByKey. -type tFilesSortByKey struct { - tFiles - icmp *iComparer -} - -func (x *tFilesSortByKey) Less(i, j int) bool { - return x.lessByKey(x.icmp, i, j) -} - -// Helper type for sortByNum. -type tFilesSortByNum struct { - tFiles -} - -func (x *tFilesSortByNum) Less(i, j int) bool { - return x.lessByNum(i, j) -} - -// Table operations. -type tOps struct { - s *session - noSync bool - cache *cache.Cache - bcache *cache.Cache - bpool *util.BufferPool -} - -// Creates an empty table and returns table writer. -func (t *tOps) create() (*tWriter, error) { - fd := storage.FileDesc{storage.TypeTable, t.s.allocFileNum()} - fw, err := t.s.stor.Create(fd) - if err != nil { - return nil, err - } - return &tWriter{ - t: t, - fd: fd, - w: fw, - tw: table.NewWriter(fw, t.s.o.Options), - }, nil -} - -// Builds table from src iterator. -func (t *tOps) createFrom(src iterator.Iterator) (f *tFile, n int, err error) { - w, err := t.create() - if err != nil { - return - } - - defer func() { - if err != nil { - w.drop() - } - }() - - for src.Next() { - err = w.append(src.Key(), src.Value()) - if err != nil { - return - } - } - err = src.Error() - if err != nil { - return - } - - n = w.tw.EntriesLen() - f, err = w.finish() - return -} - -// Opens table. It returns a cache handle, which should -// be released after use. -func (t *tOps) open(f *tFile) (ch *cache.Handle, err error) { - ch = t.cache.Get(0, uint64(f.fd.Num), func() (size int, value cache.Value) { - var r storage.Reader - r, err = t.s.stor.Open(f.fd) - if err != nil { - return 0, nil - } - - var bcache *cache.NamespaceGetter - if t.bcache != nil { - bcache = &cache.NamespaceGetter{Cache: t.bcache, NS: uint64(f.fd.Num)} - } - - var tr *table.Reader - tr, err = table.NewReader(r, f.size, f.fd, bcache, t.bpool, t.s.o.Options) - if err != nil { - r.Close() - return 0, nil - } - return 1, tr - - }) - if ch == nil && err == nil { - err = ErrClosed - } - return -} - -// Finds key/value pair whose key is greater than or equal to the -// given key. -func (t *tOps) find(f *tFile, key []byte, ro *opt.ReadOptions) (rkey, rvalue []byte, err error) { - ch, err := t.open(f) - if err != nil { - return nil, nil, err - } - defer ch.Release() - return ch.Value().(*table.Reader).Find(key, true, ro) -} - -// Finds key that is greater than or equal to the given key. -func (t *tOps) findKey(f *tFile, key []byte, ro *opt.ReadOptions) (rkey []byte, err error) { - ch, err := t.open(f) - if err != nil { - return nil, err - } - defer ch.Release() - return ch.Value().(*table.Reader).FindKey(key, true, ro) -} - -// Returns approximate offset of the given key. -func (t *tOps) offsetOf(f *tFile, key []byte) (offset int64, err error) { - ch, err := t.open(f) - if err != nil { - return - } - defer ch.Release() - return ch.Value().(*table.Reader).OffsetOf(key) -} - -// Creates an iterator from the given table. -func (t *tOps) newIterator(f *tFile, slice *util.Range, ro *opt.ReadOptions) iterator.Iterator { - ch, err := t.open(f) - if err != nil { - return iterator.NewEmptyIterator(err) - } - iter := ch.Value().(*table.Reader).NewIterator(slice, ro) - iter.SetReleaser(ch) - return iter -} - -// Removes table from persistent storage. It waits until -// no one use the the table. -func (t *tOps) remove(f *tFile) { - t.cache.Delete(0, uint64(f.fd.Num), func() { - if err := t.s.stor.Remove(f.fd); err != nil { - t.s.logf("table@remove removing @%d %q", f.fd.Num, err) - } else { - t.s.logf("table@remove removed @%d", f.fd.Num) - } - if t.bcache != nil { - t.bcache.EvictNS(uint64(f.fd.Num)) - } - }) -} - -// Closes the table ops instance. It will close all tables, -// regadless still used or not. -func (t *tOps) close() { - t.bpool.Close() - t.cache.Close() - if t.bcache != nil { - t.bcache.CloseWeak() - } -} - -// Creates new initialized table ops instance. -func newTableOps(s *session) *tOps { - var ( - cacher cache.Cacher - bcache *cache.Cache - bpool *util.BufferPool - ) - if s.o.GetOpenFilesCacheCapacity() > 0 { - cacher = cache.NewLRU(s.o.GetOpenFilesCacheCapacity()) - } - if !s.o.GetDisableBlockCache() { - var bcacher cache.Cacher - if s.o.GetBlockCacheCapacity() > 0 { - bcacher = cache.NewLRU(s.o.GetBlockCacheCapacity()) - } - bcache = cache.NewCache(bcacher) - } - if !s.o.GetDisableBufferPool() { - bpool = util.NewBufferPool(s.o.GetBlockSize() + 5) - } - return &tOps{ - s: s, - noSync: s.o.GetNoSync(), - cache: cache.NewCache(cacher), - bcache: bcache, - bpool: bpool, - } -} - -// tWriter wraps the table writer. It keep track of file descriptor -// and added key range. -type tWriter struct { - t *tOps - - fd storage.FileDesc - w storage.Writer - tw *table.Writer - - first, last []byte -} - -// Append key/value pair to the table. -func (w *tWriter) append(key, value []byte) error { - if w.first == nil { - w.first = append([]byte{}, key...) - } - w.last = append(w.last[:0], key...) - return w.tw.Append(key, value) -} - -// Returns true if the table is empty. -func (w *tWriter) empty() bool { - return w.first == nil -} - -// Closes the storage.Writer. -func (w *tWriter) close() { - if w.w != nil { - w.w.Close() - w.w = nil - } -} - -// Finalizes the table and returns table file. -func (w *tWriter) finish() (f *tFile, err error) { - defer w.close() - err = w.tw.Close() - if err != nil { - return - } - if !w.t.noSync { - err = w.w.Sync() - if err != nil { - return - } - } - f = newTableFile(w.fd, int64(w.tw.BytesLen()), internalKey(w.first), internalKey(w.last)) - return -} - -// Drops the table. -func (w *tWriter) drop() { - w.close() - w.t.s.stor.Remove(w.fd) - w.t.s.reuseFileNum(w.fd.Num) - w.tw = nil - w.first = nil - w.last = nil -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/table/block_test.go b/vendor/github.com/syndtr/goleveldb/leveldb/table/block_test.go deleted file mode 100644 index 00e6f9e..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/table/block_test.go +++ /dev/null @@ -1,139 +0,0 @@ -// Copyright (c) 2014, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package table - -import ( - "encoding/binary" - "fmt" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - - "github.com/syndtr/goleveldb/leveldb/comparer" - "github.com/syndtr/goleveldb/leveldb/iterator" - "github.com/syndtr/goleveldb/leveldb/testutil" - "github.com/syndtr/goleveldb/leveldb/util" -) - -type blockTesting struct { - tr *Reader - b *block -} - -func (t *blockTesting) TestNewIterator(slice *util.Range) iterator.Iterator { - return t.tr.newBlockIter(t.b, nil, slice, false) -} - -var _ = testutil.Defer(func() { - Describe("Block", func() { - Build := func(kv *testutil.KeyValue, restartInterval int) *blockTesting { - // Building the block. - bw := &blockWriter{ - restartInterval: restartInterval, - scratch: make([]byte, 30), - } - kv.Iterate(func(i int, key, value []byte) { - bw.append(key, value) - }) - bw.finish() - - // Opening the block. - data := bw.buf.Bytes() - restartsLen := int(binary.LittleEndian.Uint32(data[len(data)-4:])) - return &blockTesting{ - tr: &Reader{cmp: comparer.DefaultComparer}, - b: &block{ - data: data, - restartsLen: restartsLen, - restartsOffset: len(data) - (restartsLen+1)*4, - }, - } - } - - Describe("read test", func() { - for restartInterval := 1; restartInterval <= 5; restartInterval++ { - Describe(fmt.Sprintf("with restart interval of %d", restartInterval), func() { - kv := &testutil.KeyValue{} - Text := func() string { - return fmt.Sprintf("and %d keys", kv.Len()) - } - - Test := func() { - // Make block. - br := Build(kv, restartInterval) - // Do testing. - testutil.KeyValueTesting(nil, kv.Clone(), br, nil, nil) - } - - Describe(Text(), Test) - - kv.PutString("", "empty") - Describe(Text(), Test) - - kv.PutString("a1", "foo") - Describe(Text(), Test) - - kv.PutString("a2", "v") - Describe(Text(), Test) - - kv.PutString("a3qqwrkks", "hello") - Describe(Text(), Test) - - kv.PutString("a4", "bar") - Describe(Text(), Test) - - kv.PutString("a5111111", "v5") - kv.PutString("a6", "") - kv.PutString("a7", "v7") - kv.PutString("a8", "vvvvvvvvvvvvvvvvvvvvvv8") - kv.PutString("b", "v9") - kv.PutString("c9", "v9") - kv.PutString("c91", "v9") - kv.PutString("d0", "v9") - Describe(Text(), Test) - }) - } - }) - - Describe("out-of-bound slice test", func() { - kv := &testutil.KeyValue{} - kv.PutString("k1", "v1") - kv.PutString("k2", "v2") - kv.PutString("k3abcdefgg", "v3") - kv.PutString("k4", "v4") - kv.PutString("k5", "v5") - for restartInterval := 1; restartInterval <= 5; restartInterval++ { - Describe(fmt.Sprintf("with restart interval of %d", restartInterval), func() { - // Make block. - bt := Build(kv, restartInterval) - - Test := func(r *util.Range) func(done Done) { - return func(done Done) { - iter := bt.TestNewIterator(r) - Expect(iter.Error()).ShouldNot(HaveOccurred()) - - t := testutil.IteratorTesting{ - KeyValue: kv.Clone(), - Iter: iter, - } - - testutil.DoIteratorTesting(&t) - iter.Release() - done <- true - } - } - - It("Should do iterations and seeks correctly #0", - Test(&util.Range{Start: []byte("k0"), Limit: []byte("k6")}), 2.0) - - It("Should do iterations and seeks correctly #1", - Test(&util.Range{Start: []byte(""), Limit: []byte("zzzzzzz")}), 2.0) - }) - } - }) - }) -}) diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/table/reader.go b/vendor/github.com/syndtr/goleveldb/leveldb/table/reader.go deleted file mode 100644 index 16cfbaa..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/table/reader.go +++ /dev/null @@ -1,1135 +0,0 @@ -// Copyright (c) 2012, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package table - -import ( - "encoding/binary" - "fmt" - "io" - "sort" - "strings" - "sync" - - "github.com/golang/snappy" - - "github.com/syndtr/goleveldb/leveldb/cache" - "github.com/syndtr/goleveldb/leveldb/comparer" - "github.com/syndtr/goleveldb/leveldb/errors" - "github.com/syndtr/goleveldb/leveldb/filter" - "github.com/syndtr/goleveldb/leveldb/iterator" - "github.com/syndtr/goleveldb/leveldb/opt" - "github.com/syndtr/goleveldb/leveldb/storage" - "github.com/syndtr/goleveldb/leveldb/util" -) - -// Reader errors. -var ( - ErrNotFound = errors.ErrNotFound - ErrReaderReleased = errors.New("leveldb/table: reader released") - ErrIterReleased = errors.New("leveldb/table: iterator released") -) - -// ErrCorrupted describes error due to corruption. This error will be wrapped -// with errors.ErrCorrupted. -type ErrCorrupted struct { - Pos int64 - Size int64 - Kind string - Reason string -} - -func (e *ErrCorrupted) Error() string { - return fmt.Sprintf("leveldb/table: corruption on %s (pos=%d): %s", e.Kind, e.Pos, e.Reason) -} - -func max(x, y int) int { - if x > y { - return x - } - return y -} - -type block struct { - bpool *util.BufferPool - bh blockHandle - data []byte - restartsLen int - restartsOffset int -} - -func (b *block) seek(cmp comparer.Comparer, rstart, rlimit int, key []byte) (index, offset int, err error) { - index = sort.Search(b.restartsLen-rstart-(b.restartsLen-rlimit), func(i int) bool { - offset := int(binary.LittleEndian.Uint32(b.data[b.restartsOffset+4*(rstart+i):])) - offset++ // shared always zero, since this is a restart point - v1, n1 := binary.Uvarint(b.data[offset:]) // key length - _, n2 := binary.Uvarint(b.data[offset+n1:]) // value length - m := offset + n1 + n2 - return cmp.Compare(b.data[m:m+int(v1)], key) > 0 - }) + rstart - 1 - if index < rstart { - // The smallest key is greater-than key sought. - index = rstart - } - offset = int(binary.LittleEndian.Uint32(b.data[b.restartsOffset+4*index:])) - return -} - -func (b *block) restartIndex(rstart, rlimit, offset int) int { - return sort.Search(b.restartsLen-rstart-(b.restartsLen-rlimit), func(i int) bool { - return int(binary.LittleEndian.Uint32(b.data[b.restartsOffset+4*(rstart+i):])) > offset - }) + rstart - 1 -} - -func (b *block) restartOffset(index int) int { - return int(binary.LittleEndian.Uint32(b.data[b.restartsOffset+4*index:])) -} - -func (b *block) entry(offset int) (key, value []byte, nShared, n int, err error) { - if offset >= b.restartsOffset { - if offset != b.restartsOffset { - err = &ErrCorrupted{Reason: "entries offset not aligned"} - } - return - } - v0, n0 := binary.Uvarint(b.data[offset:]) // Shared prefix length - v1, n1 := binary.Uvarint(b.data[offset+n0:]) // Key length - v2, n2 := binary.Uvarint(b.data[offset+n0+n1:]) // Value length - m := n0 + n1 + n2 - n = m + int(v1) + int(v2) - if n0 <= 0 || n1 <= 0 || n2 <= 0 || offset+n > b.restartsOffset { - err = &ErrCorrupted{Reason: "entries corrupted"} - return - } - key = b.data[offset+m : offset+m+int(v1)] - value = b.data[offset+m+int(v1) : offset+n] - nShared = int(v0) - return -} - -func (b *block) Release() { - b.bpool.Put(b.data) - b.bpool = nil - b.data = nil -} - -type dir int - -const ( - dirReleased dir = iota - 1 - dirSOI - dirEOI - dirBackward - dirForward -) - -type blockIter struct { - tr *Reader - block *block - blockReleaser util.Releaser - releaser util.Releaser - key, value []byte - offset int - // Previous offset, only filled by Next. - prevOffset int - prevNode []int - prevKeys []byte - restartIndex int - // Iterator direction. - dir dir - // Restart index slice range. - riStart int - riLimit int - // Offset slice range. - offsetStart int - offsetRealStart int - offsetLimit int - // Error. - err error -} - -func (i *blockIter) sErr(err error) { - i.err = err - i.key = nil - i.value = nil - i.prevNode = nil - i.prevKeys = nil -} - -func (i *blockIter) reset() { - if i.dir == dirBackward { - i.prevNode = i.prevNode[:0] - i.prevKeys = i.prevKeys[:0] - } - i.restartIndex = i.riStart - i.offset = i.offsetStart - i.dir = dirSOI - i.key = i.key[:0] - i.value = nil -} - -func (i *blockIter) isFirst() bool { - switch i.dir { - case dirForward: - return i.prevOffset == i.offsetRealStart - case dirBackward: - return len(i.prevNode) == 1 && i.restartIndex == i.riStart - } - return false -} - -func (i *blockIter) isLast() bool { - switch i.dir { - case dirForward, dirBackward: - return i.offset == i.offsetLimit - } - return false -} - -func (i *blockIter) First() bool { - if i.err != nil { - return false - } else if i.dir == dirReleased { - i.err = ErrIterReleased - return false - } - - if i.dir == dirBackward { - i.prevNode = i.prevNode[:0] - i.prevKeys = i.prevKeys[:0] - } - i.dir = dirSOI - return i.Next() -} - -func (i *blockIter) Last() bool { - if i.err != nil { - return false - } else if i.dir == dirReleased { - i.err = ErrIterReleased - return false - } - - if i.dir == dirBackward { - i.prevNode = i.prevNode[:0] - i.prevKeys = i.prevKeys[:0] - } - i.dir = dirEOI - return i.Prev() -} - -func (i *blockIter) Seek(key []byte) bool { - if i.err != nil { - return false - } else if i.dir == dirReleased { - i.err = ErrIterReleased - return false - } - - ri, offset, err := i.block.seek(i.tr.cmp, i.riStart, i.riLimit, key) - if err != nil { - i.sErr(err) - return false - } - i.restartIndex = ri - i.offset = max(i.offsetStart, offset) - if i.dir == dirSOI || i.dir == dirEOI { - i.dir = dirForward - } - for i.Next() { - if i.tr.cmp.Compare(i.key, key) >= 0 { - return true - } - } - return false -} - -func (i *blockIter) Next() bool { - if i.dir == dirEOI || i.err != nil { - return false - } else if i.dir == dirReleased { - i.err = ErrIterReleased - return false - } - - if i.dir == dirSOI { - i.restartIndex = i.riStart - i.offset = i.offsetStart - } else if i.dir == dirBackward { - i.prevNode = i.prevNode[:0] - i.prevKeys = i.prevKeys[:0] - } - for i.offset < i.offsetRealStart { - key, value, nShared, n, err := i.block.entry(i.offset) - if err != nil { - i.sErr(i.tr.fixErrCorruptedBH(i.block.bh, err)) - return false - } - if n == 0 { - i.dir = dirEOI - return false - } - i.key = append(i.key[:nShared], key...) - i.value = value - i.offset += n - } - if i.offset >= i.offsetLimit { - i.dir = dirEOI - if i.offset != i.offsetLimit { - i.sErr(i.tr.newErrCorruptedBH(i.block.bh, "entries offset not aligned")) - } - return false - } - key, value, nShared, n, err := i.block.entry(i.offset) - if err != nil { - i.sErr(i.tr.fixErrCorruptedBH(i.block.bh, err)) - return false - } - if n == 0 { - i.dir = dirEOI - return false - } - i.key = append(i.key[:nShared], key...) - i.value = value - i.prevOffset = i.offset - i.offset += n - i.dir = dirForward - return true -} - -func (i *blockIter) Prev() bool { - if i.dir == dirSOI || i.err != nil { - return false - } else if i.dir == dirReleased { - i.err = ErrIterReleased - return false - } - - var ri int - if i.dir == dirForward { - // Change direction. - i.offset = i.prevOffset - if i.offset == i.offsetRealStart { - i.dir = dirSOI - return false - } - ri = i.block.restartIndex(i.restartIndex, i.riLimit, i.offset) - i.dir = dirBackward - } else if i.dir == dirEOI { - // At the end of iterator. - i.restartIndex = i.riLimit - i.offset = i.offsetLimit - if i.offset == i.offsetRealStart { - i.dir = dirSOI - return false - } - ri = i.riLimit - 1 - i.dir = dirBackward - } else if len(i.prevNode) == 1 { - // This is the end of a restart range. - i.offset = i.prevNode[0] - i.prevNode = i.prevNode[:0] - if i.restartIndex == i.riStart { - i.dir = dirSOI - return false - } - i.restartIndex-- - ri = i.restartIndex - } else { - // In the middle of restart range, get from cache. - n := len(i.prevNode) - 3 - node := i.prevNode[n:] - i.prevNode = i.prevNode[:n] - // Get the key. - ko := node[0] - i.key = append(i.key[:0], i.prevKeys[ko:]...) - i.prevKeys = i.prevKeys[:ko] - // Get the value. - vo := node[1] - vl := vo + node[2] - i.value = i.block.data[vo:vl] - i.offset = vl - return true - } - // Build entries cache. - i.key = i.key[:0] - i.value = nil - offset := i.block.restartOffset(ri) - if offset == i.offset { - ri-- - if ri < 0 { - i.dir = dirSOI - return false - } - offset = i.block.restartOffset(ri) - } - i.prevNode = append(i.prevNode, offset) - for { - key, value, nShared, n, err := i.block.entry(offset) - if err != nil { - i.sErr(i.tr.fixErrCorruptedBH(i.block.bh, err)) - return false - } - if offset >= i.offsetRealStart { - if i.value != nil { - // Appends 3 variables: - // 1. Previous keys offset - // 2. Value offset in the data block - // 3. Value length - i.prevNode = append(i.prevNode, len(i.prevKeys), offset-len(i.value), len(i.value)) - i.prevKeys = append(i.prevKeys, i.key...) - } - i.value = value - } - i.key = append(i.key[:nShared], key...) - offset += n - // Stop if target offset reached. - if offset >= i.offset { - if offset != i.offset { - i.sErr(i.tr.newErrCorruptedBH(i.block.bh, "entries offset not aligned")) - return false - } - - break - } - } - i.restartIndex = ri - i.offset = offset - return true -} - -func (i *blockIter) Key() []byte { - if i.err != nil || i.dir <= dirEOI { - return nil - } - return i.key -} - -func (i *blockIter) Value() []byte { - if i.err != nil || i.dir <= dirEOI { - return nil - } - return i.value -} - -func (i *blockIter) Release() { - if i.dir != dirReleased { - i.tr = nil - i.block = nil - i.prevNode = nil - i.prevKeys = nil - i.key = nil - i.value = nil - i.dir = dirReleased - if i.blockReleaser != nil { - i.blockReleaser.Release() - i.blockReleaser = nil - } - if i.releaser != nil { - i.releaser.Release() - i.releaser = nil - } - } -} - -func (i *blockIter) SetReleaser(releaser util.Releaser) { - if i.dir == dirReleased { - panic(util.ErrReleased) - } - if i.releaser != nil && releaser != nil { - panic(util.ErrHasReleaser) - } - i.releaser = releaser -} - -func (i *blockIter) Valid() bool { - return i.err == nil && (i.dir == dirBackward || i.dir == dirForward) -} - -func (i *blockIter) Error() error { - return i.err -} - -type filterBlock struct { - bpool *util.BufferPool - data []byte - oOffset int - baseLg uint - filtersNum int -} - -func (b *filterBlock) contains(filter filter.Filter, offset uint64, key []byte) bool { - i := int(offset >> b.baseLg) - if i < b.filtersNum { - o := b.data[b.oOffset+i*4:] - n := int(binary.LittleEndian.Uint32(o)) - m := int(binary.LittleEndian.Uint32(o[4:])) - if n < m && m <= b.oOffset { - return filter.Contains(b.data[n:m], key) - } else if n == m { - return false - } - } - return true -} - -func (b *filterBlock) Release() { - b.bpool.Put(b.data) - b.bpool = nil - b.data = nil -} - -type indexIter struct { - *blockIter - tr *Reader - slice *util.Range - // Options - fillCache bool -} - -func (i *indexIter) Get() iterator.Iterator { - value := i.Value() - if value == nil { - return nil - } - dataBH, n := decodeBlockHandle(value) - if n == 0 { - return iterator.NewEmptyIterator(i.tr.newErrCorruptedBH(i.tr.indexBH, "bad data block handle")) - } - - var slice *util.Range - if i.slice != nil && (i.blockIter.isFirst() || i.blockIter.isLast()) { - slice = i.slice - } - return i.tr.getDataIterErr(dataBH, slice, i.tr.verifyChecksum, i.fillCache) -} - -// Reader is a table reader. -type Reader struct { - mu sync.RWMutex - fd storage.FileDesc - reader io.ReaderAt - cache *cache.NamespaceGetter - err error - bpool *util.BufferPool - // Options - o *opt.Options - cmp comparer.Comparer - filter filter.Filter - verifyChecksum bool - - dataEnd int64 - metaBH, indexBH, filterBH blockHandle - indexBlock *block - filterBlock *filterBlock -} - -func (r *Reader) blockKind(bh blockHandle) string { - switch bh.offset { - case r.metaBH.offset: - return "meta-block" - case r.indexBH.offset: - return "index-block" - case r.filterBH.offset: - if r.filterBH.length > 0 { - return "filter-block" - } - } - return "data-block" -} - -func (r *Reader) newErrCorrupted(pos, size int64, kind, reason string) error { - return &errors.ErrCorrupted{Fd: r.fd, Err: &ErrCorrupted{Pos: pos, Size: size, Kind: kind, Reason: reason}} -} - -func (r *Reader) newErrCorruptedBH(bh blockHandle, reason string) error { - return r.newErrCorrupted(int64(bh.offset), int64(bh.length), r.blockKind(bh), reason) -} - -func (r *Reader) fixErrCorruptedBH(bh blockHandle, err error) error { - if cerr, ok := err.(*ErrCorrupted); ok { - cerr.Pos = int64(bh.offset) - cerr.Size = int64(bh.length) - cerr.Kind = r.blockKind(bh) - return &errors.ErrCorrupted{Fd: r.fd, Err: cerr} - } - return err -} - -func (r *Reader) readRawBlock(bh blockHandle, verifyChecksum bool) ([]byte, error) { - data := r.bpool.Get(int(bh.length + blockTrailerLen)) - if _, err := r.reader.ReadAt(data, int64(bh.offset)); err != nil && err != io.EOF { - return nil, err - } - - if verifyChecksum { - n := bh.length + 1 - checksum0 := binary.LittleEndian.Uint32(data[n:]) - checksum1 := util.NewCRC(data[:n]).Value() - if checksum0 != checksum1 { - r.bpool.Put(data) - return nil, r.newErrCorruptedBH(bh, fmt.Sprintf("checksum mismatch, want=%#x got=%#x", checksum0, checksum1)) - } - } - - switch data[bh.length] { - case blockTypeNoCompression: - data = data[:bh.length] - case blockTypeSnappyCompression: - decLen, err := snappy.DecodedLen(data[:bh.length]) - if err != nil { - r.bpool.Put(data) - return nil, r.newErrCorruptedBH(bh, err.Error()) - } - decData := r.bpool.Get(decLen) - decData, err = snappy.Decode(decData, data[:bh.length]) - r.bpool.Put(data) - if err != nil { - r.bpool.Put(decData) - return nil, r.newErrCorruptedBH(bh, err.Error()) - } - data = decData - default: - r.bpool.Put(data) - return nil, r.newErrCorruptedBH(bh, fmt.Sprintf("unknown compression type %#x", data[bh.length])) - } - return data, nil -} - -func (r *Reader) readBlock(bh blockHandle, verifyChecksum bool) (*block, error) { - data, err := r.readRawBlock(bh, verifyChecksum) - if err != nil { - return nil, err - } - restartsLen := int(binary.LittleEndian.Uint32(data[len(data)-4:])) - b := &block{ - bpool: r.bpool, - bh: bh, - data: data, - restartsLen: restartsLen, - restartsOffset: len(data) - (restartsLen+1)*4, - } - return b, nil -} - -func (r *Reader) readBlockCached(bh blockHandle, verifyChecksum, fillCache bool) (*block, util.Releaser, error) { - if r.cache != nil { - var ( - err error - ch *cache.Handle - ) - if fillCache { - ch = r.cache.Get(bh.offset, func() (size int, value cache.Value) { - var b *block - b, err = r.readBlock(bh, verifyChecksum) - if err != nil { - return 0, nil - } - return cap(b.data), b - }) - } else { - ch = r.cache.Get(bh.offset, nil) - } - if ch != nil { - b, ok := ch.Value().(*block) - if !ok { - ch.Release() - return nil, nil, errors.New("leveldb/table: inconsistent block type") - } - return b, ch, err - } else if err != nil { - return nil, nil, err - } - } - - b, err := r.readBlock(bh, verifyChecksum) - return b, b, err -} - -func (r *Reader) readFilterBlock(bh blockHandle) (*filterBlock, error) { - data, err := r.readRawBlock(bh, true) - if err != nil { - return nil, err - } - n := len(data) - if n < 5 { - return nil, r.newErrCorruptedBH(bh, "too short") - } - m := n - 5 - oOffset := int(binary.LittleEndian.Uint32(data[m:])) - if oOffset > m { - return nil, r.newErrCorruptedBH(bh, "invalid data-offsets offset") - } - b := &filterBlock{ - bpool: r.bpool, - data: data, - oOffset: oOffset, - baseLg: uint(data[n-1]), - filtersNum: (m - oOffset) / 4, - } - return b, nil -} - -func (r *Reader) readFilterBlockCached(bh blockHandle, fillCache bool) (*filterBlock, util.Releaser, error) { - if r.cache != nil { - var ( - err error - ch *cache.Handle - ) - if fillCache { - ch = r.cache.Get(bh.offset, func() (size int, value cache.Value) { - var b *filterBlock - b, err = r.readFilterBlock(bh) - if err != nil { - return 0, nil - } - return cap(b.data), b - }) - } else { - ch = r.cache.Get(bh.offset, nil) - } - if ch != nil { - b, ok := ch.Value().(*filterBlock) - if !ok { - ch.Release() - return nil, nil, errors.New("leveldb/table: inconsistent block type") - } - return b, ch, err - } else if err != nil { - return nil, nil, err - } - } - - b, err := r.readFilterBlock(bh) - return b, b, err -} - -func (r *Reader) getIndexBlock(fillCache bool) (b *block, rel util.Releaser, err error) { - if r.indexBlock == nil { - return r.readBlockCached(r.indexBH, true, fillCache) - } - return r.indexBlock, util.NoopReleaser{}, nil -} - -func (r *Reader) getFilterBlock(fillCache bool) (*filterBlock, util.Releaser, error) { - if r.filterBlock == nil { - return r.readFilterBlockCached(r.filterBH, fillCache) - } - return r.filterBlock, util.NoopReleaser{}, nil -} - -func (r *Reader) newBlockIter(b *block, bReleaser util.Releaser, slice *util.Range, inclLimit bool) *blockIter { - bi := &blockIter{ - tr: r, - block: b, - blockReleaser: bReleaser, - // Valid key should never be nil. - key: make([]byte, 0), - dir: dirSOI, - riStart: 0, - riLimit: b.restartsLen, - offsetStart: 0, - offsetRealStart: 0, - offsetLimit: b.restartsOffset, - } - if slice != nil { - if slice.Start != nil { - if bi.Seek(slice.Start) { - bi.riStart = b.restartIndex(bi.restartIndex, b.restartsLen, bi.prevOffset) - bi.offsetStart = b.restartOffset(bi.riStart) - bi.offsetRealStart = bi.prevOffset - } else { - bi.riStart = b.restartsLen - bi.offsetStart = b.restartsOffset - bi.offsetRealStart = b.restartsOffset - } - } - if slice.Limit != nil { - if bi.Seek(slice.Limit) && (!inclLimit || bi.Next()) { - bi.offsetLimit = bi.prevOffset - bi.riLimit = bi.restartIndex + 1 - } - } - bi.reset() - if bi.offsetStart > bi.offsetLimit { - bi.sErr(errors.New("leveldb/table: invalid slice range")) - } - } - return bi -} - -func (r *Reader) getDataIter(dataBH blockHandle, slice *util.Range, verifyChecksum, fillCache bool) iterator.Iterator { - b, rel, err := r.readBlockCached(dataBH, verifyChecksum, fillCache) - if err != nil { - return iterator.NewEmptyIterator(err) - } - return r.newBlockIter(b, rel, slice, false) -} - -func (r *Reader) getDataIterErr(dataBH blockHandle, slice *util.Range, verifyChecksum, fillCache bool) iterator.Iterator { - r.mu.RLock() - defer r.mu.RUnlock() - - if r.err != nil { - return iterator.NewEmptyIterator(r.err) - } - - return r.getDataIter(dataBH, slice, verifyChecksum, fillCache) -} - -// NewIterator creates an iterator from the table. -// -// Slice allows slicing the iterator to only contains keys in the given -// range. A nil Range.Start is treated as a key before all keys in the -// table. And a nil Range.Limit is treated as a key after all keys in -// the table. -// -// The returned iterator is not safe for concurrent use and should be released -// after use. -// -// Also read Iterator documentation of the leveldb/iterator package. -func (r *Reader) NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator { - r.mu.RLock() - defer r.mu.RUnlock() - - if r.err != nil { - return iterator.NewEmptyIterator(r.err) - } - - fillCache := !ro.GetDontFillCache() - indexBlock, rel, err := r.getIndexBlock(fillCache) - if err != nil { - return iterator.NewEmptyIterator(err) - } - index := &indexIter{ - blockIter: r.newBlockIter(indexBlock, rel, slice, true), - tr: r, - slice: slice, - fillCache: !ro.GetDontFillCache(), - } - return iterator.NewIndexedIterator(index, opt.GetStrict(r.o, ro, opt.StrictReader)) -} - -func (r *Reader) find(key []byte, filtered bool, ro *opt.ReadOptions, noValue bool) (rkey, value []byte, err error) { - r.mu.RLock() - defer r.mu.RUnlock() - - if r.err != nil { - err = r.err - return - } - - indexBlock, rel, err := r.getIndexBlock(true) - if err != nil { - return - } - defer rel.Release() - - index := r.newBlockIter(indexBlock, nil, nil, true) - defer index.Release() - - if !index.Seek(key) { - if err = index.Error(); err == nil { - err = ErrNotFound - } - return - } - - dataBH, n := decodeBlockHandle(index.Value()) - if n == 0 { - r.err = r.newErrCorruptedBH(r.indexBH, "bad data block handle") - return nil, nil, r.err - } - - // The filter should only used for exact match. - if filtered && r.filter != nil { - filterBlock, frel, ferr := r.getFilterBlock(true) - if ferr == nil { - if !filterBlock.contains(r.filter, dataBH.offset, key) { - frel.Release() - return nil, nil, ErrNotFound - } - frel.Release() - } else if !errors.IsCorrupted(ferr) { - return nil, nil, ferr - } - } - - data := r.getDataIter(dataBH, nil, r.verifyChecksum, !ro.GetDontFillCache()) - if !data.Seek(key) { - data.Release() - if err = data.Error(); err != nil { - return - } - - // The nearest greater-than key is the first key of the next block. - if !index.Next() { - if err = index.Error(); err == nil { - err = ErrNotFound - } - return - } - - dataBH, n = decodeBlockHandle(index.Value()) - if n == 0 { - r.err = r.newErrCorruptedBH(r.indexBH, "bad data block handle") - return nil, nil, r.err - } - - data = r.getDataIter(dataBH, nil, r.verifyChecksum, !ro.GetDontFillCache()) - if !data.Next() { - data.Release() - if err = data.Error(); err == nil { - err = ErrNotFound - } - return - } - } - - // Key doesn't use block buffer, no need to copy the buffer. - rkey = data.Key() - if !noValue { - if r.bpool == nil { - value = data.Value() - } else { - // Value does use block buffer, and since the buffer will be - // recycled, it need to be copied. - value = append([]byte{}, data.Value()...) - } - } - data.Release() - return -} - -// Find finds key/value pair whose key is greater than or equal to the -// given key. It returns ErrNotFound if the table doesn't contain -// such pair. -// If filtered is true then the nearest 'block' will be checked against -// 'filter data' (if present) and will immediately return ErrNotFound if -// 'filter data' indicates that such pair doesn't exist. -// -// The caller may modify the contents of the returned slice as it is its -// own copy. -// It is safe to modify the contents of the argument after Find returns. -func (r *Reader) Find(key []byte, filtered bool, ro *opt.ReadOptions) (rkey, value []byte, err error) { - return r.find(key, filtered, ro, false) -} - -// FindKey finds key that is greater than or equal to the given key. -// It returns ErrNotFound if the table doesn't contain such key. -// If filtered is true then the nearest 'block' will be checked against -// 'filter data' (if present) and will immediately return ErrNotFound if -// 'filter data' indicates that such key doesn't exist. -// -// The caller may modify the contents of the returned slice as it is its -// own copy. -// It is safe to modify the contents of the argument after Find returns. -func (r *Reader) FindKey(key []byte, filtered bool, ro *opt.ReadOptions) (rkey []byte, err error) { - rkey, _, err = r.find(key, filtered, ro, true) - return -} - -// Get gets the value for the given key. It returns errors.ErrNotFound -// if the table does not contain the key. -// -// The caller may modify the contents of the returned slice as it is its -// own copy. -// It is safe to modify the contents of the argument after Find returns. -func (r *Reader) Get(key []byte, ro *opt.ReadOptions) (value []byte, err error) { - r.mu.RLock() - defer r.mu.RUnlock() - - if r.err != nil { - err = r.err - return - } - - rkey, value, err := r.find(key, false, ro, false) - if err == nil && r.cmp.Compare(rkey, key) != 0 { - value = nil - err = ErrNotFound - } - return -} - -// OffsetOf returns approximate offset for the given key. -// -// It is safe to modify the contents of the argument after Get returns. -func (r *Reader) OffsetOf(key []byte) (offset int64, err error) { - r.mu.RLock() - defer r.mu.RUnlock() - - if r.err != nil { - err = r.err - return - } - - indexBlock, rel, err := r.readBlockCached(r.indexBH, true, true) - if err != nil { - return - } - defer rel.Release() - - index := r.newBlockIter(indexBlock, nil, nil, true) - defer index.Release() - if index.Seek(key) { - dataBH, n := decodeBlockHandle(index.Value()) - if n == 0 { - r.err = r.newErrCorruptedBH(r.indexBH, "bad data block handle") - return - } - offset = int64(dataBH.offset) - return - } - err = index.Error() - if err == nil { - offset = r.dataEnd - } - return -} - -// Release implements util.Releaser. -// It also close the file if it is an io.Closer. -func (r *Reader) Release() { - r.mu.Lock() - defer r.mu.Unlock() - - if closer, ok := r.reader.(io.Closer); ok { - closer.Close() - } - if r.indexBlock != nil { - r.indexBlock.Release() - r.indexBlock = nil - } - if r.filterBlock != nil { - r.filterBlock.Release() - r.filterBlock = nil - } - r.reader = nil - r.cache = nil - r.bpool = nil - r.err = ErrReaderReleased -} - -// NewReader creates a new initialized table reader for the file. -// The fi, cache and bpool is optional and can be nil. -// -// The returned table reader instance is safe for concurrent use. -func NewReader(f io.ReaderAt, size int64, fd storage.FileDesc, cache *cache.NamespaceGetter, bpool *util.BufferPool, o *opt.Options) (*Reader, error) { - if f == nil { - return nil, errors.New("leveldb/table: nil file") - } - - r := &Reader{ - fd: fd, - reader: f, - cache: cache, - bpool: bpool, - o: o, - cmp: o.GetComparer(), - verifyChecksum: o.GetStrict(opt.StrictBlockChecksum), - } - - if size < footerLen { - r.err = r.newErrCorrupted(0, size, "table", "too small") - return r, nil - } - - footerPos := size - footerLen - var footer [footerLen]byte - if _, err := r.reader.ReadAt(footer[:], footerPos); err != nil && err != io.EOF { - return nil, err - } - if string(footer[footerLen-len(magic):footerLen]) != magic { - r.err = r.newErrCorrupted(footerPos, footerLen, "table-footer", "bad magic number") - return r, nil - } - - var n int - // Decode the metaindex block handle. - r.metaBH, n = decodeBlockHandle(footer[:]) - if n == 0 { - r.err = r.newErrCorrupted(footerPos, footerLen, "table-footer", "bad metaindex block handle") - return r, nil - } - - // Decode the index block handle. - r.indexBH, n = decodeBlockHandle(footer[n:]) - if n == 0 { - r.err = r.newErrCorrupted(footerPos, footerLen, "table-footer", "bad index block handle") - return r, nil - } - - // Read metaindex block. - metaBlock, err := r.readBlock(r.metaBH, true) - if err != nil { - if errors.IsCorrupted(err) { - r.err = err - return r, nil - } - return nil, err - } - - // Set data end. - r.dataEnd = int64(r.metaBH.offset) - - // Read metaindex. - metaIter := r.newBlockIter(metaBlock, nil, nil, true) - for metaIter.Next() { - key := string(metaIter.Key()) - if !strings.HasPrefix(key, "filter.") { - continue - } - fn := key[7:] - if f0 := o.GetFilter(); f0 != nil && f0.Name() == fn { - r.filter = f0 - } else { - for _, f0 := range o.GetAltFilters() { - if f0.Name() == fn { - r.filter = f0 - break - } - } - } - if r.filter != nil { - filterBH, n := decodeBlockHandle(metaIter.Value()) - if n == 0 { - continue - } - r.filterBH = filterBH - // Update data end. - r.dataEnd = int64(filterBH.offset) - break - } - } - metaIter.Release() - metaBlock.Release() - - // Cache index and filter block locally, since we don't have global cache. - if cache == nil { - r.indexBlock, err = r.readBlock(r.indexBH, true) - if err != nil { - if errors.IsCorrupted(err) { - r.err = err - return r, nil - } - return nil, err - } - if r.filter != nil { - r.filterBlock, err = r.readFilterBlock(r.filterBH) - if err != nil { - if !errors.IsCorrupted(err) { - return nil, err - } - - // Don't use filter then. - r.filter = nil - } - } - } - - return r, nil -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/table/table.go b/vendor/github.com/syndtr/goleveldb/leveldb/table/table.go deleted file mode 100644 index beacdc1..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/table/table.go +++ /dev/null @@ -1,177 +0,0 @@ -// Copyright (c) 2012, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -// Package table allows read and write sorted key/value. -package table - -import ( - "encoding/binary" -) - -/* -Table: - -Table is consist of one or more data blocks, an optional filter block -a metaindex block, an index block and a table footer. Metaindex block -is a special block used to keep parameters of the table, such as filter -block name and its block handle. Index block is a special block used to -keep record of data blocks offset and length, index block use one as -restart interval. The key used by index block are the last key of preceding -block, shorter separator of adjacent blocks or shorter successor of the -last key of the last block. Filter block is an optional block contains -sequence of filter data generated by a filter generator. - -Table data structure: - + optional - / - +--------------+--------------+--------------+------+-------+-----------------+-------------+--------+ - | data block 1 | ... | data block n | filter block | metaindex block | index block | footer | - +--------------+--------------+--------------+--------------+-----------------+-------------+--------+ - - Each block followed by a 5-bytes trailer contains compression type and checksum. - -Table block trailer: - - +---------------------------+-------------------+ - | compression type (1-byte) | checksum (4-byte) | - +---------------------------+-------------------+ - - The checksum is a CRC-32 computed using Castagnoli's polynomial. Compression - type also included in the checksum. - -Table footer: - - +------------------- 40-bytes -------------------+ - / \ - +------------------------+--------------------+------+-----------------+ - | metaindex block handle / index block handle / ---- | magic (8-bytes) | - +------------------------+--------------------+------+-----------------+ - - The magic are first 64-bit of SHA-1 sum of "http://code.google.com/p/leveldb/". - -NOTE: All fixed-length integer are little-endian. -*/ - -/* -Block: - -Block is consist of one or more key/value entries and a block trailer. -Block entry shares key prefix with its preceding key until a restart -point reached. A block should contains at least one restart point. -First restart point are always zero. - -Block data structure: - - + restart point + restart point (depends on restart interval) - / / - +---------------+---------------+---------------+---------------+---------+ - | block entry 1 | block entry 2 | ... | block entry n | trailer | - +---------------+---------------+---------------+---------------+---------+ - -Key/value entry: - - +---- key len ----+ - / \ - +-------+---------+-----------+---------+--------------------+--------------+----------------+ - | shared (varint) | not shared (varint) | value len (varint) | key (varlen) | value (varlen) | - +-----------------+---------------------+--------------------+--------------+----------------+ - - Block entry shares key prefix with its preceding key: - Conditions: - restart_interval=2 - entry one : key=deck,value=v1 - entry two : key=dock,value=v2 - entry three: key=duck,value=v3 - The entries will be encoded as follow: - - + restart point (offset=0) + restart point (offset=16) - / / - +-----+-----+-----+----------+--------+-----+-----+-----+---------+--------+-----+-----+-----+----------+--------+ - | 0 | 4 | 2 | "deck" | "v1" | 1 | 3 | 2 | "ock" | "v2" | 0 | 4 | 2 | "duck" | "v3" | - +-----+-----+-----+----------+--------+-----+-----+-----+---------+--------+-----+-----+-----+----------+--------+ - \ / \ / \ / - +----------- entry one -----------+ +----------- entry two ----------+ +---------- entry three ----------+ - - The block trailer will contains two restart points: - - +------------+-----------+--------+ - | 0 | 16 | 2 | - +------------+-----------+---+----+ - \ / \ - +-- restart points --+ + restart points length - -Block trailer: - - +-- 4-bytes --+ - / \ - +-----------------+-----------------+-----------------+------------------------------+ - | restart point 1 | .... | restart point n | restart points len (4-bytes) | - +-----------------+-----------------+-----------------+------------------------------+ - - -NOTE: All fixed-length integer are little-endian. -*/ - -/* -Filter block: - -Filter block consist of one or more filter data and a filter block trailer. -The trailer contains filter data offsets, a trailer offset and a 1-byte base Lg. - -Filter block data structure: - - + offset 1 + offset 2 + offset n + trailer offset - / / / / - +---------------+---------------+---------------+---------+ - | filter data 1 | ... | filter data n | trailer | - +---------------+---------------+---------------+---------+ - -Filter block trailer: - - +- 4-bytes -+ - / \ - +---------------+---------------+---------------+-------------------------------+------------------+ - | data 1 offset | .... | data n offset | data-offsets offset (4-bytes) | base Lg (1-byte) | - +-------------- +---------------+---------------+-------------------------------+------------------+ - - -NOTE: All fixed-length integer are little-endian. -*/ - -const ( - blockTrailerLen = 5 - footerLen = 48 - - magic = "\x57\xfb\x80\x8b\x24\x75\x47\xdb" - - // The block type gives the per-block compression format. - // These constants are part of the file format and should not be changed. - blockTypeNoCompression = 0 - blockTypeSnappyCompression = 1 - - // Generate new filter every 2KB of data - filterBaseLg = 11 - filterBase = 1 << filterBaseLg -) - -type blockHandle struct { - offset, length uint64 -} - -func decodeBlockHandle(src []byte) (blockHandle, int) { - offset, n := binary.Uvarint(src) - length, m := binary.Uvarint(src[n:]) - if n == 0 || m == 0 { - return blockHandle{}, 0 - } - return blockHandle{offset, length}, n + m -} - -func encodeBlockHandle(dst []byte, b blockHandle) int { - n := binary.PutUvarint(dst, b.offset) - m := binary.PutUvarint(dst[n:], b.length) - return n + m -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/table/table_suite_test.go b/vendor/github.com/syndtr/goleveldb/leveldb/table/table_suite_test.go deleted file mode 100644 index 6465da6..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/table/table_suite_test.go +++ /dev/null @@ -1,11 +0,0 @@ -package table - -import ( - "testing" - - "github.com/syndtr/goleveldb/leveldb/testutil" -) - -func TestTable(t *testing.T) { - testutil.RunSuite(t, "Table Suite") -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/table/table_test.go b/vendor/github.com/syndtr/goleveldb/leveldb/table/table_test.go deleted file mode 100644 index 232efcd..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/table/table_test.go +++ /dev/null @@ -1,123 +0,0 @@ -// Copyright (c) 2014, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package table - -import ( - "bytes" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - - "github.com/syndtr/goleveldb/leveldb/iterator" - "github.com/syndtr/goleveldb/leveldb/opt" - "github.com/syndtr/goleveldb/leveldb/storage" - "github.com/syndtr/goleveldb/leveldb/testutil" - "github.com/syndtr/goleveldb/leveldb/util" -) - -type tableWrapper struct { - *Reader -} - -func (t tableWrapper) TestFind(key []byte) (rkey, rvalue []byte, err error) { - return t.Reader.Find(key, false, nil) -} - -func (t tableWrapper) TestGet(key []byte) (value []byte, err error) { - return t.Reader.Get(key, nil) -} - -func (t tableWrapper) TestNewIterator(slice *util.Range) iterator.Iterator { - return t.Reader.NewIterator(slice, nil) -} - -var _ = testutil.Defer(func() { - Describe("Table", func() { - Describe("approximate offset test", func() { - var ( - buf = &bytes.Buffer{} - o = &opt.Options{ - BlockSize: 1024, - Compression: opt.NoCompression, - } - ) - - // Building the table. - tw := NewWriter(buf, o) - tw.Append([]byte("k01"), []byte("hello")) - tw.Append([]byte("k02"), []byte("hello2")) - tw.Append([]byte("k03"), bytes.Repeat([]byte{'x'}, 10000)) - tw.Append([]byte("k04"), bytes.Repeat([]byte{'x'}, 200000)) - tw.Append([]byte("k05"), bytes.Repeat([]byte{'x'}, 300000)) - tw.Append([]byte("k06"), []byte("hello3")) - tw.Append([]byte("k07"), bytes.Repeat([]byte{'x'}, 100000)) - err := tw.Close() - - It("Should be able to approximate offset of a key correctly", func() { - Expect(err).ShouldNot(HaveOccurred()) - - tr, err := NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len()), storage.FileDesc{}, nil, nil, o) - Expect(err).ShouldNot(HaveOccurred()) - CheckOffset := func(key string, expect, threshold int) { - offset, err := tr.OffsetOf([]byte(key)) - Expect(err).ShouldNot(HaveOccurred()) - Expect(offset).Should(BeNumerically("~", expect, threshold), "Offset of key %q", key) - } - - CheckOffset("k0", 0, 0) - CheckOffset("k01a", 0, 0) - CheckOffset("k02", 0, 0) - CheckOffset("k03", 0, 0) - CheckOffset("k04", 10000, 1000) - CheckOffset("k04a", 210000, 1000) - CheckOffset("k05", 210000, 1000) - CheckOffset("k06", 510000, 1000) - CheckOffset("k07", 510000, 1000) - CheckOffset("xyz", 610000, 2000) - }) - }) - - Describe("read test", func() { - Build := func(kv testutil.KeyValue) testutil.DB { - o := &opt.Options{ - BlockSize: 512, - BlockRestartInterval: 3, - } - buf := &bytes.Buffer{} - - // Building the table. - tw := NewWriter(buf, o) - kv.Iterate(func(i int, key, value []byte) { - tw.Append(key, value) - }) - tw.Close() - - // Opening the table. - tr, _ := NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len()), storage.FileDesc{}, nil, nil, o) - return tableWrapper{tr} - } - Test := func(kv *testutil.KeyValue, body func(r *Reader)) func() { - return func() { - db := Build(*kv) - if body != nil { - body(db.(tableWrapper).Reader) - } - testutil.KeyValueTesting(nil, *kv, db, nil, nil) - } - } - - testutil.AllKeyValueTesting(nil, Build, nil, nil) - Describe("with one key per block", Test(testutil.KeyValue_Generate(nil, 9, 1, 1, 10, 512, 512), func(r *Reader) { - It("should have correct blocks number", func() { - indexBlock, err := r.readBlock(r.indexBH, true) - Expect(err).To(BeNil()) - Expect(indexBlock.restartsLen).Should(Equal(9)) - }) - })) - }) - }) -}) diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/table/writer.go b/vendor/github.com/syndtr/goleveldb/leveldb/table/writer.go deleted file mode 100644 index b96b271..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/table/writer.go +++ /dev/null @@ -1,375 +0,0 @@ -// Copyright (c) 2012, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package table - -import ( - "encoding/binary" - "errors" - "fmt" - "io" - - "github.com/golang/snappy" - - "github.com/syndtr/goleveldb/leveldb/comparer" - "github.com/syndtr/goleveldb/leveldb/filter" - "github.com/syndtr/goleveldb/leveldb/opt" - "github.com/syndtr/goleveldb/leveldb/util" -) - -func sharedPrefixLen(a, b []byte) int { - i, n := 0, len(a) - if n > len(b) { - n = len(b) - } - for i < n && a[i] == b[i] { - i++ - } - return i -} - -type blockWriter struct { - restartInterval int - buf util.Buffer - nEntries int - prevKey []byte - restarts []uint32 - scratch []byte -} - -func (w *blockWriter) append(key, value []byte) { - nShared := 0 - if w.nEntries%w.restartInterval == 0 { - w.restarts = append(w.restarts, uint32(w.buf.Len())) - } else { - nShared = sharedPrefixLen(w.prevKey, key) - } - n := binary.PutUvarint(w.scratch[0:], uint64(nShared)) - n += binary.PutUvarint(w.scratch[n:], uint64(len(key)-nShared)) - n += binary.PutUvarint(w.scratch[n:], uint64(len(value))) - w.buf.Write(w.scratch[:n]) - w.buf.Write(key[nShared:]) - w.buf.Write(value) - w.prevKey = append(w.prevKey[:0], key...) - w.nEntries++ -} - -func (w *blockWriter) finish() { - // Write restarts entry. - if w.nEntries == 0 { - // Must have at least one restart entry. - w.restarts = append(w.restarts, 0) - } - w.restarts = append(w.restarts, uint32(len(w.restarts))) - for _, x := range w.restarts { - buf4 := w.buf.Alloc(4) - binary.LittleEndian.PutUint32(buf4, x) - } -} - -func (w *blockWriter) reset() { - w.buf.Reset() - w.nEntries = 0 - w.restarts = w.restarts[:0] -} - -func (w *blockWriter) bytesLen() int { - restartsLen := len(w.restarts) - if restartsLen == 0 { - restartsLen = 1 - } - return w.buf.Len() + 4*restartsLen + 4 -} - -type filterWriter struct { - generator filter.FilterGenerator - buf util.Buffer - nKeys int - offsets []uint32 -} - -func (w *filterWriter) add(key []byte) { - if w.generator == nil { - return - } - w.generator.Add(key) - w.nKeys++ -} - -func (w *filterWriter) flush(offset uint64) { - if w.generator == nil { - return - } - for x := int(offset / filterBase); x > len(w.offsets); { - w.generate() - } -} - -func (w *filterWriter) finish() { - if w.generator == nil { - return - } - // Generate last keys. - - if w.nKeys > 0 { - w.generate() - } - w.offsets = append(w.offsets, uint32(w.buf.Len())) - for _, x := range w.offsets { - buf4 := w.buf.Alloc(4) - binary.LittleEndian.PutUint32(buf4, x) - } - w.buf.WriteByte(filterBaseLg) -} - -func (w *filterWriter) generate() { - // Record offset. - w.offsets = append(w.offsets, uint32(w.buf.Len())) - // Generate filters. - if w.nKeys > 0 { - w.generator.Generate(&w.buf) - w.nKeys = 0 - } -} - -// Writer is a table writer. -type Writer struct { - writer io.Writer - err error - // Options - cmp comparer.Comparer - filter filter.Filter - compression opt.Compression - blockSize int - - dataBlock blockWriter - indexBlock blockWriter - filterBlock filterWriter - pendingBH blockHandle - offset uint64 - nEntries int - // Scratch allocated enough for 5 uvarint. Block writer should not use - // first 20-bytes since it will be used to encode block handle, which - // then passed to the block writer itself. - scratch [50]byte - comparerScratch []byte - compressionScratch []byte -} - -func (w *Writer) writeBlock(buf *util.Buffer, compression opt.Compression) (bh blockHandle, err error) { - // Compress the buffer if necessary. - var b []byte - if compression == opt.SnappyCompression { - // Allocate scratch enough for compression and block trailer. - if n := snappy.MaxEncodedLen(buf.Len()) + blockTrailerLen; len(w.compressionScratch) < n { - w.compressionScratch = make([]byte, n) - } - compressed := snappy.Encode(w.compressionScratch, buf.Bytes()) - n := len(compressed) - b = compressed[:n+blockTrailerLen] - b[n] = blockTypeSnappyCompression - } else { - tmp := buf.Alloc(blockTrailerLen) - tmp[0] = blockTypeNoCompression - b = buf.Bytes() - } - - // Calculate the checksum. - n := len(b) - 4 - checksum := util.NewCRC(b[:n]).Value() - binary.LittleEndian.PutUint32(b[n:], checksum) - - // Write the buffer to the file. - _, err = w.writer.Write(b) - if err != nil { - return - } - bh = blockHandle{w.offset, uint64(len(b) - blockTrailerLen)} - w.offset += uint64(len(b)) - return -} - -func (w *Writer) flushPendingBH(key []byte) { - if w.pendingBH.length == 0 { - return - } - var separator []byte - if len(key) == 0 { - separator = w.cmp.Successor(w.comparerScratch[:0], w.dataBlock.prevKey) - } else { - separator = w.cmp.Separator(w.comparerScratch[:0], w.dataBlock.prevKey, key) - } - if separator == nil { - separator = w.dataBlock.prevKey - } else { - w.comparerScratch = separator - } - n := encodeBlockHandle(w.scratch[:20], w.pendingBH) - // Append the block handle to the index block. - w.indexBlock.append(separator, w.scratch[:n]) - // Reset prev key of the data block. - w.dataBlock.prevKey = w.dataBlock.prevKey[:0] - // Clear pending block handle. - w.pendingBH = blockHandle{} -} - -func (w *Writer) finishBlock() error { - w.dataBlock.finish() - bh, err := w.writeBlock(&w.dataBlock.buf, w.compression) - if err != nil { - return err - } - w.pendingBH = bh - // Reset the data block. - w.dataBlock.reset() - // Flush the filter block. - w.filterBlock.flush(w.offset) - return nil -} - -// Append appends key/value pair to the table. The keys passed must -// be in increasing order. -// -// It is safe to modify the contents of the arguments after Append returns. -func (w *Writer) Append(key, value []byte) error { - if w.err != nil { - return w.err - } - if w.nEntries > 0 && w.cmp.Compare(w.dataBlock.prevKey, key) >= 0 { - w.err = fmt.Errorf("leveldb/table: Writer: keys are not in increasing order: %q, %q", w.dataBlock.prevKey, key) - return w.err - } - - w.flushPendingBH(key) - // Append key/value pair to the data block. - w.dataBlock.append(key, value) - // Add key to the filter block. - w.filterBlock.add(key) - - // Finish the data block if block size target reached. - if w.dataBlock.bytesLen() >= w.blockSize { - if err := w.finishBlock(); err != nil { - w.err = err - return w.err - } - } - w.nEntries++ - return nil -} - -// BlocksLen returns number of blocks written so far. -func (w *Writer) BlocksLen() int { - n := w.indexBlock.nEntries - if w.pendingBH.length > 0 { - // Includes the pending block. - n++ - } - return n -} - -// EntriesLen returns number of entries added so far. -func (w *Writer) EntriesLen() int { - return w.nEntries -} - -// BytesLen returns number of bytes written so far. -func (w *Writer) BytesLen() int { - return int(w.offset) -} - -// Close will finalize the table. Calling Append is not possible -// after Close, but calling BlocksLen, EntriesLen and BytesLen -// is still possible. -func (w *Writer) Close() error { - if w.err != nil { - return w.err - } - - // Write the last data block. Or empty data block if there - // aren't any data blocks at all. - if w.dataBlock.nEntries > 0 || w.nEntries == 0 { - if err := w.finishBlock(); err != nil { - w.err = err - return w.err - } - } - w.flushPendingBH(nil) - - // Write the filter block. - var filterBH blockHandle - w.filterBlock.finish() - if buf := &w.filterBlock.buf; buf.Len() > 0 { - filterBH, w.err = w.writeBlock(buf, opt.NoCompression) - if w.err != nil { - return w.err - } - } - - // Write the metaindex block. - if filterBH.length > 0 { - key := []byte("filter." + w.filter.Name()) - n := encodeBlockHandle(w.scratch[:20], filterBH) - w.dataBlock.append(key, w.scratch[:n]) - } - w.dataBlock.finish() - metaindexBH, err := w.writeBlock(&w.dataBlock.buf, w.compression) - if err != nil { - w.err = err - return w.err - } - - // Write the index block. - w.indexBlock.finish() - indexBH, err := w.writeBlock(&w.indexBlock.buf, w.compression) - if err != nil { - w.err = err - return w.err - } - - // Write the table footer. - footer := w.scratch[:footerLen] - for i := range footer { - footer[i] = 0 - } - n := encodeBlockHandle(footer, metaindexBH) - encodeBlockHandle(footer[n:], indexBH) - copy(footer[footerLen-len(magic):], magic) - if _, err := w.writer.Write(footer); err != nil { - w.err = err - return w.err - } - w.offset += footerLen - - w.err = errors.New("leveldb/table: writer is closed") - return nil -} - -// NewWriter creates a new initialized table writer for the file. -// -// Table writer is not safe for concurrent use. -func NewWriter(f io.Writer, o *opt.Options) *Writer { - w := &Writer{ - writer: f, - cmp: o.GetComparer(), - filter: o.GetFilter(), - compression: o.GetCompression(), - blockSize: o.GetBlockSize(), - comparerScratch: make([]byte, 0), - } - // data block - w.dataBlock.restartInterval = o.GetBlockRestartInterval() - // The first 20-bytes are used for encoding block handle. - w.dataBlock.scratch = w.scratch[20:] - // index block - w.indexBlock.restartInterval = 1 - w.indexBlock.scratch = w.scratch[20:] - // filter block - if w.filter != nil { - w.filterBlock.generator = w.filter.NewGenerator() - w.filterBlock.flush(0) - } - return w -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/testutil/db.go b/vendor/github.com/syndtr/goleveldb/leveldb/testutil/db.go deleted file mode 100644 index ec3f177..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/testutil/db.go +++ /dev/null @@ -1,222 +0,0 @@ -// Copyright (c) 2014, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package testutil - -import ( - "fmt" - "math/rand" - - . "github.com/onsi/gomega" - - "github.com/syndtr/goleveldb/leveldb/errors" - "github.com/syndtr/goleveldb/leveldb/iterator" - "github.com/syndtr/goleveldb/leveldb/util" -) - -type DB interface{} - -type Put interface { - TestPut(key []byte, value []byte) error -} - -type Delete interface { - TestDelete(key []byte) error -} - -type Find interface { - TestFind(key []byte) (rkey, rvalue []byte, err error) -} - -type Get interface { - TestGet(key []byte) (value []byte, err error) -} - -type Has interface { - TestHas(key []byte) (ret bool, err error) -} - -type NewIterator interface { - TestNewIterator(slice *util.Range) iterator.Iterator -} - -type DBAct int - -func (a DBAct) String() string { - switch a { - case DBNone: - return "none" - case DBPut: - return "put" - case DBOverwrite: - return "overwrite" - case DBDelete: - return "delete" - case DBDeleteNA: - return "delete_na" - } - return "unknown" -} - -const ( - DBNone DBAct = iota - DBPut - DBOverwrite - DBDelete - DBDeleteNA -) - -type DBTesting struct { - Rand *rand.Rand - DB interface { - Get - Put - Delete - } - PostFn func(t *DBTesting) - Deleted, Present KeyValue - Act, LastAct DBAct - ActKey, LastActKey []byte -} - -func (t *DBTesting) post() { - if t.PostFn != nil { - t.PostFn(t) - } -} - -func (t *DBTesting) setAct(act DBAct, key []byte) { - t.LastAct, t.Act = t.Act, act - t.LastActKey, t.ActKey = t.ActKey, key -} - -func (t *DBTesting) text() string { - return fmt.Sprintf("last action was <%v> %q, <%v> %q", t.LastAct, t.LastActKey, t.Act, t.ActKey) -} - -func (t *DBTesting) Text() string { - return "DBTesting " + t.text() -} - -func (t *DBTesting) TestPresentKV(key, value []byte) { - rvalue, err := t.DB.TestGet(key) - Expect(err).ShouldNot(HaveOccurred(), "Get on key %q, %s", key, t.text()) - Expect(rvalue).Should(Equal(value), "Value for key %q, %s", key, t.text()) -} - -func (t *DBTesting) TestAllPresent() { - t.Present.IterateShuffled(t.Rand, func(i int, key, value []byte) { - t.TestPresentKV(key, value) - }) -} - -func (t *DBTesting) TestDeletedKey(key []byte) { - _, err := t.DB.TestGet(key) - Expect(err).Should(Equal(errors.ErrNotFound), "Get on deleted key %q, %s", key, t.text()) -} - -func (t *DBTesting) TestAllDeleted() { - t.Deleted.IterateShuffled(t.Rand, func(i int, key, value []byte) { - t.TestDeletedKey(key) - }) -} - -func (t *DBTesting) TestAll() { - dn := t.Deleted.Len() - pn := t.Present.Len() - ShuffledIndex(t.Rand, dn+pn, 1, func(i int) { - if i >= dn { - key, value := t.Present.Index(i - dn) - t.TestPresentKV(key, value) - } else { - t.TestDeletedKey(t.Deleted.KeyAt(i)) - } - }) -} - -func (t *DBTesting) Put(key, value []byte) { - if new := t.Present.PutU(key, value); new { - t.setAct(DBPut, key) - } else { - t.setAct(DBOverwrite, key) - } - t.Deleted.Delete(key) - err := t.DB.TestPut(key, value) - Expect(err).ShouldNot(HaveOccurred(), t.Text()) - t.TestPresentKV(key, value) - t.post() -} - -func (t *DBTesting) PutRandom() bool { - if t.Deleted.Len() > 0 { - i := t.Rand.Intn(t.Deleted.Len()) - key, value := t.Deleted.Index(i) - t.Put(key, value) - return true - } - return false -} - -func (t *DBTesting) Delete(key []byte) { - if exist, value := t.Present.Delete(key); exist { - t.setAct(DBDelete, key) - t.Deleted.PutU(key, value) - } else { - t.setAct(DBDeleteNA, key) - } - err := t.DB.TestDelete(key) - Expect(err).ShouldNot(HaveOccurred(), t.Text()) - t.TestDeletedKey(key) - t.post() -} - -func (t *DBTesting) DeleteRandom() bool { - if t.Present.Len() > 0 { - i := t.Rand.Intn(t.Present.Len()) - t.Delete(t.Present.KeyAt(i)) - return true - } - return false -} - -func (t *DBTesting) RandomAct(round int) { - for i := 0; i < round; i++ { - if t.Rand.Int()%2 == 0 { - t.PutRandom() - } else { - t.DeleteRandom() - } - } -} - -func DoDBTesting(t *DBTesting) { - if t.Rand == nil { - t.Rand = NewRand() - } - - t.DeleteRandom() - t.PutRandom() - t.DeleteRandom() - t.DeleteRandom() - for i := t.Deleted.Len() / 2; i >= 0; i-- { - t.PutRandom() - } - t.RandomAct((t.Deleted.Len() + t.Present.Len()) * 10) - - // Additional iterator testing - if db, ok := t.DB.(NewIterator); ok { - iter := db.TestNewIterator(nil) - Expect(iter.Error()).NotTo(HaveOccurred()) - - it := IteratorTesting{ - KeyValue: t.Present, - Iter: iter, - } - - DoIteratorTesting(&it) - iter.Release() - } -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/testutil/ginkgo.go b/vendor/github.com/syndtr/goleveldb/leveldb/testutil/ginkgo.go deleted file mode 100644 index 82f3d0e..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/testutil/ginkgo.go +++ /dev/null @@ -1,21 +0,0 @@ -package testutil - -import ( - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" -) - -func RunSuite(t GinkgoTestingT, name string) { - RunDefer() - - SynchronizedBeforeSuite(func() []byte { - RunDefer("setup") - return nil - }, func(data []byte) {}) - SynchronizedAfterSuite(func() { - RunDefer("teardown") - }, func() {}) - - RegisterFailHandler(Fail) - RunSpecs(t, name) -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/testutil/iter.go b/vendor/github.com/syndtr/goleveldb/leveldb/testutil/iter.go deleted file mode 100644 index df6d9db..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/testutil/iter.go +++ /dev/null @@ -1,327 +0,0 @@ -// Copyright (c) 2014, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package testutil - -import ( - "fmt" - "math/rand" - - . "github.com/onsi/gomega" - - "github.com/syndtr/goleveldb/leveldb/iterator" -) - -type IterAct int - -func (a IterAct) String() string { - switch a { - case IterNone: - return "none" - case IterFirst: - return "first" - case IterLast: - return "last" - case IterPrev: - return "prev" - case IterNext: - return "next" - case IterSeek: - return "seek" - case IterSOI: - return "soi" - case IterEOI: - return "eoi" - } - return "unknown" -} - -const ( - IterNone IterAct = iota - IterFirst - IterLast - IterPrev - IterNext - IterSeek - IterSOI - IterEOI -) - -type IteratorTesting struct { - KeyValue - Iter iterator.Iterator - Rand *rand.Rand - PostFn func(t *IteratorTesting) - Pos int - Act, LastAct IterAct - - once bool -} - -func (t *IteratorTesting) init() { - if !t.once { - t.Pos = -1 - t.once = true - } -} - -func (t *IteratorTesting) post() { - if t.PostFn != nil { - t.PostFn(t) - } -} - -func (t *IteratorTesting) setAct(act IterAct) { - t.LastAct, t.Act = t.Act, act -} - -func (t *IteratorTesting) text() string { - return fmt.Sprintf("at pos %d and last action was <%v> -> <%v>", t.Pos, t.LastAct, t.Act) -} - -func (t *IteratorTesting) Text() string { - return "IteratorTesting is " + t.text() -} - -func (t *IteratorTesting) IsFirst() bool { - t.init() - return t.Len() > 0 && t.Pos == 0 -} - -func (t *IteratorTesting) IsLast() bool { - t.init() - return t.Len() > 0 && t.Pos == t.Len()-1 -} - -func (t *IteratorTesting) TestKV() { - t.init() - key, value := t.Index(t.Pos) - Expect(t.Iter.Key()).NotTo(BeNil()) - Expect(t.Iter.Key()).Should(Equal(key), "Key is invalid, %s", t.text()) - Expect(t.Iter.Value()).Should(Equal(value), "Value for key %q, %s", key, t.text()) -} - -func (t *IteratorTesting) First() { - t.init() - t.setAct(IterFirst) - - ok := t.Iter.First() - Expect(t.Iter.Error()).ShouldNot(HaveOccurred()) - if t.Len() > 0 { - t.Pos = 0 - Expect(ok).Should(BeTrue(), t.Text()) - t.TestKV() - } else { - t.Pos = -1 - Expect(ok).ShouldNot(BeTrue(), t.Text()) - } - t.post() -} - -func (t *IteratorTesting) Last() { - t.init() - t.setAct(IterLast) - - ok := t.Iter.Last() - Expect(t.Iter.Error()).ShouldNot(HaveOccurred()) - if t.Len() > 0 { - t.Pos = t.Len() - 1 - Expect(ok).Should(BeTrue(), t.Text()) - t.TestKV() - } else { - t.Pos = 0 - Expect(ok).ShouldNot(BeTrue(), t.Text()) - } - t.post() -} - -func (t *IteratorTesting) Next() { - t.init() - t.setAct(IterNext) - - ok := t.Iter.Next() - Expect(t.Iter.Error()).ShouldNot(HaveOccurred()) - if t.Pos < t.Len()-1 { - t.Pos++ - Expect(ok).Should(BeTrue(), t.Text()) - t.TestKV() - } else { - t.Pos = t.Len() - Expect(ok).ShouldNot(BeTrue(), t.Text()) - } - t.post() -} - -func (t *IteratorTesting) Prev() { - t.init() - t.setAct(IterPrev) - - ok := t.Iter.Prev() - Expect(t.Iter.Error()).ShouldNot(HaveOccurred()) - if t.Pos > 0 { - t.Pos-- - Expect(ok).Should(BeTrue(), t.Text()) - t.TestKV() - } else { - t.Pos = -1 - Expect(ok).ShouldNot(BeTrue(), t.Text()) - } - t.post() -} - -func (t *IteratorTesting) Seek(i int) { - t.init() - t.setAct(IterSeek) - - key, _ := t.Index(i) - oldKey, _ := t.IndexOrNil(t.Pos) - - ok := t.Iter.Seek(key) - Expect(t.Iter.Error()).ShouldNot(HaveOccurred()) - Expect(ok).Should(BeTrue(), fmt.Sprintf("Seek from key %q to %q, to pos %d, %s", oldKey, key, i, t.text())) - - t.Pos = i - t.TestKV() - t.post() -} - -func (t *IteratorTesting) SeekInexact(i int) { - t.init() - t.setAct(IterSeek) - var key0 []byte - key1, _ := t.Index(i) - if i > 0 { - key0, _ = t.Index(i - 1) - } - key := BytesSeparator(key0, key1) - oldKey, _ := t.IndexOrNil(t.Pos) - - ok := t.Iter.Seek(key) - Expect(t.Iter.Error()).ShouldNot(HaveOccurred()) - Expect(ok).Should(BeTrue(), fmt.Sprintf("Seek from key %q to %q (%q), to pos %d, %s", oldKey, key, key1, i, t.text())) - - t.Pos = i - t.TestKV() - t.post() -} - -func (t *IteratorTesting) SeekKey(key []byte) { - t.init() - t.setAct(IterSeek) - oldKey, _ := t.IndexOrNil(t.Pos) - i := t.Search(key) - - ok := t.Iter.Seek(key) - Expect(t.Iter.Error()).ShouldNot(HaveOccurred()) - if i < t.Len() { - key_, _ := t.Index(i) - Expect(ok).Should(BeTrue(), fmt.Sprintf("Seek from key %q to %q (%q), to pos %d, %s", oldKey, key, key_, i, t.text())) - t.Pos = i - t.TestKV() - } else { - Expect(ok).ShouldNot(BeTrue(), fmt.Sprintf("Seek from key %q to %q, %s", oldKey, key, t.text())) - } - - t.Pos = i - t.post() -} - -func (t *IteratorTesting) SOI() { - t.init() - t.setAct(IterSOI) - Expect(t.Pos).Should(BeNumerically("<=", 0), t.Text()) - for i := 0; i < 3; i++ { - t.Prev() - } - t.post() -} - -func (t *IteratorTesting) EOI() { - t.init() - t.setAct(IterEOI) - Expect(t.Pos).Should(BeNumerically(">=", t.Len()-1), t.Text()) - for i := 0; i < 3; i++ { - t.Next() - } - t.post() -} - -func (t *IteratorTesting) WalkPrev(fn func(t *IteratorTesting)) { - t.init() - for old := t.Pos; t.Pos > 0; old = t.Pos { - fn(t) - Expect(t.Pos).Should(BeNumerically("<", old), t.Text()) - } -} - -func (t *IteratorTesting) WalkNext(fn func(t *IteratorTesting)) { - t.init() - for old := t.Pos; t.Pos < t.Len()-1; old = t.Pos { - fn(t) - Expect(t.Pos).Should(BeNumerically(">", old), t.Text()) - } -} - -func (t *IteratorTesting) PrevAll() { - t.WalkPrev(func(t *IteratorTesting) { - t.Prev() - }) -} - -func (t *IteratorTesting) NextAll() { - t.WalkNext(func(t *IteratorTesting) { - t.Next() - }) -} - -func DoIteratorTesting(t *IteratorTesting) { - if t.Rand == nil { - t.Rand = NewRand() - } - t.SOI() - t.NextAll() - t.First() - t.SOI() - t.NextAll() - t.EOI() - t.PrevAll() - t.Last() - t.EOI() - t.PrevAll() - t.SOI() - - t.NextAll() - t.PrevAll() - t.NextAll() - t.Last() - t.PrevAll() - t.First() - t.NextAll() - t.EOI() - - ShuffledIndex(t.Rand, t.Len(), 1, func(i int) { - t.Seek(i) - }) - - ShuffledIndex(t.Rand, t.Len(), 1, func(i int) { - t.SeekInexact(i) - }) - - ShuffledIndex(t.Rand, t.Len(), 1, func(i int) { - t.Seek(i) - if i%2 != 0 { - t.PrevAll() - t.SOI() - } else { - t.NextAll() - t.EOI() - } - }) - - for _, key := range []string{"", "foo", "bar", "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"} { - t.SeekKey([]byte(key)) - } -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/testutil/kv.go b/vendor/github.com/syndtr/goleveldb/leveldb/testutil/kv.go deleted file mode 100644 index 608cbf3..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/testutil/kv.go +++ /dev/null @@ -1,352 +0,0 @@ -// Copyright (c) 2014, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package testutil - -import ( - "fmt" - "math/rand" - "sort" - "strings" - - "github.com/syndtr/goleveldb/leveldb/util" -) - -type KeyValueEntry struct { - key, value []byte -} - -type KeyValue struct { - entries []KeyValueEntry - nbytes int -} - -func (kv *KeyValue) Put(key, value []byte) { - if n := len(kv.entries); n > 0 && cmp.Compare(kv.entries[n-1].key, key) >= 0 { - panic(fmt.Sprintf("Put: keys are not in increasing order: %q, %q", kv.entries[n-1].key, key)) - } - kv.entries = append(kv.entries, KeyValueEntry{key, value}) - kv.nbytes += len(key) + len(value) -} - -func (kv *KeyValue) PutString(key, value string) { - kv.Put([]byte(key), []byte(value)) -} - -func (kv *KeyValue) PutU(key, value []byte) bool { - if i, exist := kv.Get(key); !exist { - if i < kv.Len() { - kv.entries = append(kv.entries[:i+1], kv.entries[i:]...) - kv.entries[i] = KeyValueEntry{key, value} - } else { - kv.entries = append(kv.entries, KeyValueEntry{key, value}) - } - kv.nbytes += len(key) + len(value) - return true - } else { - kv.nbytes += len(value) - len(kv.ValueAt(i)) - kv.entries[i].value = value - } - return false -} - -func (kv *KeyValue) PutUString(key, value string) bool { - return kv.PutU([]byte(key), []byte(value)) -} - -func (kv *KeyValue) Delete(key []byte) (exist bool, value []byte) { - i, exist := kv.Get(key) - if exist { - value = kv.entries[i].value - kv.DeleteIndex(i) - } - return -} - -func (kv *KeyValue) DeleteIndex(i int) bool { - if i < kv.Len() { - kv.nbytes -= len(kv.KeyAt(i)) + len(kv.ValueAt(i)) - kv.entries = append(kv.entries[:i], kv.entries[i+1:]...) - return true - } - return false -} - -func (kv KeyValue) Len() int { - return len(kv.entries) -} - -func (kv *KeyValue) Size() int { - return kv.nbytes -} - -func (kv KeyValue) KeyAt(i int) []byte { - return kv.entries[i].key -} - -func (kv KeyValue) ValueAt(i int) []byte { - return kv.entries[i].value -} - -func (kv KeyValue) Index(i int) (key, value []byte) { - if i < 0 || i >= len(kv.entries) { - panic(fmt.Sprintf("Index #%d: out of range", i)) - } - return kv.entries[i].key, kv.entries[i].value -} - -func (kv KeyValue) IndexInexact(i int) (key_, key, value []byte) { - key, value = kv.Index(i) - var key0 []byte - var key1 = kv.KeyAt(i) - if i > 0 { - key0 = kv.KeyAt(i - 1) - } - key_ = BytesSeparator(key0, key1) - return -} - -func (kv KeyValue) IndexOrNil(i int) (key, value []byte) { - if i >= 0 && i < len(kv.entries) { - return kv.entries[i].key, kv.entries[i].value - } - return nil, nil -} - -func (kv KeyValue) IndexString(i int) (key, value string) { - key_, _value := kv.Index(i) - return string(key_), string(_value) -} - -func (kv KeyValue) Search(key []byte) int { - return sort.Search(kv.Len(), func(i int) bool { - return cmp.Compare(kv.KeyAt(i), key) >= 0 - }) -} - -func (kv KeyValue) SearchString(key string) int { - return kv.Search([]byte(key)) -} - -func (kv KeyValue) Get(key []byte) (i int, exist bool) { - i = kv.Search(key) - if i < kv.Len() && cmp.Compare(kv.KeyAt(i), key) == 0 { - exist = true - } - return -} - -func (kv KeyValue) GetString(key string) (i int, exist bool) { - return kv.Get([]byte(key)) -} - -func (kv KeyValue) Iterate(fn func(i int, key, value []byte)) { - for i, x := range kv.entries { - fn(i, x.key, x.value) - } -} - -func (kv KeyValue) IterateString(fn func(i int, key, value string)) { - kv.Iterate(func(i int, key, value []byte) { - fn(i, string(key), string(value)) - }) -} - -func (kv KeyValue) IterateShuffled(rnd *rand.Rand, fn func(i int, key, value []byte)) { - ShuffledIndex(rnd, kv.Len(), 1, func(i int) { - fn(i, kv.entries[i].key, kv.entries[i].value) - }) -} - -func (kv KeyValue) IterateShuffledString(rnd *rand.Rand, fn func(i int, key, value string)) { - kv.IterateShuffled(rnd, func(i int, key, value []byte) { - fn(i, string(key), string(value)) - }) -} - -func (kv KeyValue) IterateInexact(fn func(i int, key_, key, value []byte)) { - for i := range kv.entries { - key_, key, value := kv.IndexInexact(i) - fn(i, key_, key, value) - } -} - -func (kv KeyValue) IterateInexactString(fn func(i int, key_, key, value string)) { - kv.IterateInexact(func(i int, key_, key, value []byte) { - fn(i, string(key_), string(key), string(value)) - }) -} - -func (kv KeyValue) Clone() KeyValue { - return KeyValue{append([]KeyValueEntry{}, kv.entries...), kv.nbytes} -} - -func (kv KeyValue) Slice(start, limit int) KeyValue { - if start < 0 || limit > kv.Len() { - panic(fmt.Sprintf("Slice %d .. %d: out of range", start, limit)) - } else if limit < start { - panic(fmt.Sprintf("Slice %d .. %d: invalid range", start, limit)) - } - return KeyValue{append([]KeyValueEntry{}, kv.entries[start:limit]...), kv.nbytes} -} - -func (kv KeyValue) SliceKey(start, limit []byte) KeyValue { - start_ := 0 - limit_ := kv.Len() - if start != nil { - start_ = kv.Search(start) - } - if limit != nil { - limit_ = kv.Search(limit) - } - return kv.Slice(start_, limit_) -} - -func (kv KeyValue) SliceKeyString(start, limit string) KeyValue { - return kv.SliceKey([]byte(start), []byte(limit)) -} - -func (kv KeyValue) SliceRange(r *util.Range) KeyValue { - if r != nil { - return kv.SliceKey(r.Start, r.Limit) - } - return kv.Clone() -} - -func (kv KeyValue) Range(start, limit int) (r util.Range) { - if kv.Len() > 0 { - if start == kv.Len() { - r.Start = BytesAfter(kv.KeyAt(start - 1)) - } else { - r.Start = kv.KeyAt(start) - } - } - if limit < kv.Len() { - r.Limit = kv.KeyAt(limit) - } - return -} - -func KeyValue_EmptyKey() *KeyValue { - kv := &KeyValue{} - kv.PutString("", "v") - return kv -} - -func KeyValue_EmptyValue() *KeyValue { - kv := &KeyValue{} - kv.PutString("abc", "") - kv.PutString("abcd", "") - return kv -} - -func KeyValue_OneKeyValue() *KeyValue { - kv := &KeyValue{} - kv.PutString("abc", "v") - return kv -} - -func KeyValue_BigValue() *KeyValue { - kv := &KeyValue{} - kv.PutString("big1", strings.Repeat("1", 200000)) - return kv -} - -func KeyValue_SpecialKey() *KeyValue { - kv := &KeyValue{} - kv.PutString("\xff\xff", "v3") - return kv -} - -func KeyValue_MultipleKeyValue() *KeyValue { - kv := &KeyValue{} - kv.PutString("a", "v") - kv.PutString("aa", "v1") - kv.PutString("aaa", "v2") - kv.PutString("aaacccccccccc", "v2") - kv.PutString("aaaccccccccccd", "v3") - kv.PutString("aaaccccccccccf", "v4") - kv.PutString("aaaccccccccccfg", "v5") - kv.PutString("ab", "v6") - kv.PutString("abc", "v7") - kv.PutString("abcd", "v8") - kv.PutString("accccccccccccccc", "v9") - kv.PutString("b", "v10") - kv.PutString("bb", "v11") - kv.PutString("bc", "v12") - kv.PutString("c", "v13") - kv.PutString("c1", "v13") - kv.PutString("czzzzzzzzzzzzzz", "v14") - kv.PutString("fffffffffffffff", "v15") - kv.PutString("g11", "v15") - kv.PutString("g111", "v15") - kv.PutString("g111\xff", "v15") - kv.PutString("zz", "v16") - kv.PutString("zzzzzzz", "v16") - kv.PutString("zzzzzzzzzzzzzzzz", "v16") - return kv -} - -var keymap = []byte("012345678ABCDEFGHIJKLMNOPQRSTUVWXYabcdefghijklmnopqrstuvwxy") - -func KeyValue_Generate(rnd *rand.Rand, n, incr, minlen, maxlen, vminlen, vmaxlen int) *KeyValue { - if rnd == nil { - rnd = NewRand() - } - if maxlen < minlen { - panic("max len should >= min len") - } - - rrand := func(min, max int) int { - if min == max { - return max - } - return rnd.Intn(max-min) + min - } - - kv := &KeyValue{} - endC := byte(len(keymap) - incr) - gen := make([]byte, 0, maxlen) - for i := 0; i < n; i++ { - m := rrand(minlen, maxlen) - last := gen - retry: - gen = last[:m] - if k := len(last); m > k { - for j := k; j < m; j++ { - gen[j] = 0 - } - } else { - for j := m - 1; j >= 0; j-- { - c := last[j] - if c == endC { - continue - } - gen[j] = c + byte(incr) - for j++; j < m; j++ { - gen[j] = 0 - } - goto ok - } - if m < maxlen { - m++ - goto retry - } - panic(fmt.Sprintf("only able to generate %d keys out of %d keys, try increasing max len", kv.Len(), n)) - ok: - } - key := make([]byte, m) - for j := 0; j < m; j++ { - key[j] = keymap[gen[j]] - } - value := make([]byte, rrand(vminlen, vmaxlen)) - for n := copy(value, []byte(fmt.Sprintf("v%d", i))); n < len(value); n++ { - value[n] = 'x' - } - kv.Put(key, value) - } - return kv -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/testutil/kvtest.go b/vendor/github.com/syndtr/goleveldb/leveldb/testutil/kvtest.go deleted file mode 100644 index f7563dc..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/testutil/kvtest.go +++ /dev/null @@ -1,212 +0,0 @@ -// Copyright (c) 2014, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package testutil - -import ( - "fmt" - "math/rand" - - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - - "github.com/syndtr/goleveldb/leveldb/errors" - "github.com/syndtr/goleveldb/leveldb/util" -) - -func TestFind(db Find, kv KeyValue) { - ShuffledIndex(nil, kv.Len(), 1, func(i int) { - key_, key, value := kv.IndexInexact(i) - - // Using exact key. - rkey, rvalue, err := db.TestFind(key) - Expect(err).ShouldNot(HaveOccurred(), "Error for exact key %q", key) - Expect(rkey).Should(Equal(key), "Key") - Expect(rvalue).Should(Equal(value), "Value for exact key %q", key) - - // Using inexact key. - rkey, rvalue, err = db.TestFind(key_) - Expect(err).ShouldNot(HaveOccurred(), "Error for inexact key %q (%q)", key_, key) - Expect(rkey).Should(Equal(key), "Key for inexact key %q (%q)", key_, key) - Expect(rvalue).Should(Equal(value), "Value for inexact key %q (%q)", key_, key) - }) -} - -func TestFindAfterLast(db Find, kv KeyValue) { - var key []byte - if kv.Len() > 0 { - key_, _ := kv.Index(kv.Len() - 1) - key = BytesAfter(key_) - } - rkey, _, err := db.TestFind(key) - Expect(err).Should(HaveOccurred(), "Find for key %q yield key %q", key, rkey) - Expect(err).Should(Equal(errors.ErrNotFound)) -} - -func TestGet(db Get, kv KeyValue) { - ShuffledIndex(nil, kv.Len(), 1, func(i int) { - key_, key, value := kv.IndexInexact(i) - - // Using exact key. - rvalue, err := db.TestGet(key) - Expect(err).ShouldNot(HaveOccurred(), "Error for key %q", key) - Expect(rvalue).Should(Equal(value), "Value for key %q", key) - - // Using inexact key. - if len(key_) > 0 { - _, err = db.TestGet(key_) - Expect(err).Should(HaveOccurred(), "Error for key %q", key_) - Expect(err).Should(Equal(errors.ErrNotFound)) - } - }) -} - -func TestHas(db Has, kv KeyValue) { - ShuffledIndex(nil, kv.Len(), 1, func(i int) { - key_, key, _ := kv.IndexInexact(i) - - // Using exact key. - ret, err := db.TestHas(key) - Expect(err).ShouldNot(HaveOccurred(), "Error for key %q", key) - Expect(ret).Should(BeTrue(), "False for key %q", key) - - // Using inexact key. - if len(key_) > 0 { - ret, err = db.TestHas(key_) - Expect(err).ShouldNot(HaveOccurred(), "Error for key %q", key_) - Expect(ret).ShouldNot(BeTrue(), "True for key %q", key) - } - }) -} - -func TestIter(db NewIterator, r *util.Range, kv KeyValue) { - iter := db.TestNewIterator(r) - Expect(iter.Error()).ShouldNot(HaveOccurred()) - - t := IteratorTesting{ - KeyValue: kv, - Iter: iter, - } - - DoIteratorTesting(&t) - iter.Release() -} - -func KeyValueTesting(rnd *rand.Rand, kv KeyValue, p DB, setup func(KeyValue) DB, teardown func(DB)) { - if rnd == nil { - rnd = NewRand() - } - - if p == nil { - BeforeEach(func() { - p = setup(kv) - }) - if teardown != nil { - AfterEach(func() { - teardown(p) - }) - } - } - - It("Should find all keys with Find", func() { - if db, ok := p.(Find); ok { - TestFind(db, kv) - } - }) - - It("Should return error if Find on key after the last", func() { - if db, ok := p.(Find); ok { - TestFindAfterLast(db, kv) - } - }) - - It("Should only find exact key with Get", func() { - if db, ok := p.(Get); ok { - TestGet(db, kv) - } - }) - - It("Should only find present key with Has", func() { - if db, ok := p.(Has); ok { - TestHas(db, kv) - } - }) - - It("Should iterates and seeks correctly", func(done Done) { - if db, ok := p.(NewIterator); ok { - TestIter(db, nil, kv.Clone()) - } - done <- true - }, 30.0) - - It("Should iterates and seeks slice correctly", func(done Done) { - if db, ok := p.(NewIterator); ok { - RandomIndex(rnd, kv.Len(), Min(kv.Len(), 50), func(i int) { - type slice struct { - r *util.Range - start, limit int - } - - key_, _, _ := kv.IndexInexact(i) - for _, x := range []slice{ - {&util.Range{Start: key_, Limit: nil}, i, kv.Len()}, - {&util.Range{Start: nil, Limit: key_}, 0, i}, - } { - By(fmt.Sprintf("Random index of %d .. %d", x.start, x.limit), func() { - TestIter(db, x.r, kv.Slice(x.start, x.limit)) - }) - } - }) - } - done <- true - }, 200.0) - - It("Should iterates and seeks slice correctly", func(done Done) { - if db, ok := p.(NewIterator); ok { - RandomRange(rnd, kv.Len(), Min(kv.Len(), 50), func(start, limit int) { - By(fmt.Sprintf("Random range of %d .. %d", start, limit), func() { - r := kv.Range(start, limit) - TestIter(db, &r, kv.Slice(start, limit)) - }) - }) - } - done <- true - }, 200.0) -} - -func AllKeyValueTesting(rnd *rand.Rand, body, setup func(KeyValue) DB, teardown func(DB)) { - Test := func(kv *KeyValue) func() { - return func() { - var p DB - if setup != nil { - Defer("setup", func() { - p = setup(*kv) - }) - } - if teardown != nil { - Defer("teardown", func() { - teardown(p) - }) - } - if body != nil { - p = body(*kv) - } - KeyValueTesting(rnd, *kv, p, func(KeyValue) DB { - return p - }, nil) - } - } - - Describe("with no key/value (empty)", Test(&KeyValue{})) - Describe("with empty key", Test(KeyValue_EmptyKey())) - Describe("with empty value", Test(KeyValue_EmptyValue())) - Describe("with one key/value", Test(KeyValue_OneKeyValue())) - Describe("with big value", Test(KeyValue_BigValue())) - Describe("with special key", Test(KeyValue_SpecialKey())) - Describe("with multiple key/value", Test(KeyValue_MultipleKeyValue())) - Describe("with generated key/value 2-incr", Test(KeyValue_Generate(nil, 120, 2, 1, 50, 10, 120))) - Describe("with generated key/value 3-incr", Test(KeyValue_Generate(nil, 120, 3, 1, 50, 10, 120))) -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/testutil/storage.go b/vendor/github.com/syndtr/goleveldb/leveldb/testutil/storage.go deleted file mode 100644 index 581daf3..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/testutil/storage.go +++ /dev/null @@ -1,693 +0,0 @@ -// Copyright (c) 2014, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package testutil - -import ( - "bytes" - "fmt" - "io" - "math/rand" - "os" - "path/filepath" - "runtime" - "strings" - "sync" - - . "github.com/onsi/gomega" - - "github.com/syndtr/goleveldb/leveldb/storage" -) - -var ( - storageMu sync.Mutex - storageUseFS = true - storageKeepFS = false - storageNum int -) - -type StorageMode int - -const ( - ModeOpen StorageMode = 1 << iota - ModeCreate - ModeRemove - ModeRename - ModeRead - ModeWrite - ModeSync - ModeClose -) - -const ( - modeOpen = iota - modeCreate - modeRemove - modeRename - modeRead - modeWrite - modeSync - modeClose - - modeCount -) - -const ( - typeManifest = iota - typeJournal - typeTable - typeTemp - - typeCount -) - -const flattenCount = modeCount * typeCount - -func flattenType(m StorageMode, t storage.FileType) int { - var x int - switch m { - case ModeOpen: - x = modeOpen - case ModeCreate: - x = modeCreate - case ModeRemove: - x = modeRemove - case ModeRename: - x = modeRename - case ModeRead: - x = modeRead - case ModeWrite: - x = modeWrite - case ModeSync: - x = modeSync - case ModeClose: - x = modeClose - default: - panic("invalid storage mode") - } - x *= typeCount - switch t { - case storage.TypeManifest: - return x + typeManifest - case storage.TypeJournal: - return x + typeJournal - case storage.TypeTable: - return x + typeTable - case storage.TypeTemp: - return x + typeTemp - default: - panic("invalid file type") - } -} - -func listFlattenType(m StorageMode, t storage.FileType) []int { - ret := make([]int, 0, flattenCount) - add := func(x int) { - x *= typeCount - switch { - case t&storage.TypeManifest != 0: - ret = append(ret, x+typeManifest) - case t&storage.TypeJournal != 0: - ret = append(ret, x+typeJournal) - case t&storage.TypeTable != 0: - ret = append(ret, x+typeTable) - case t&storage.TypeTemp != 0: - ret = append(ret, x+typeTemp) - } - } - switch { - case m&ModeOpen != 0: - add(modeOpen) - case m&ModeCreate != 0: - add(modeCreate) - case m&ModeRemove != 0: - add(modeRemove) - case m&ModeRename != 0: - add(modeRename) - case m&ModeRead != 0: - add(modeRead) - case m&ModeWrite != 0: - add(modeWrite) - case m&ModeSync != 0: - add(modeSync) - case m&ModeClose != 0: - add(modeClose) - } - return ret -} - -func packFile(fd storage.FileDesc) uint64 { - if fd.Num>>(63-typeCount) != 0 { - panic("overflow") - } - return uint64(fd.Num<> typeCount)} -} - -type emulatedError struct { - err error -} - -func (err emulatedError) Error() string { - return fmt.Sprintf("emulated storage error: %v", err.err) -} - -type storageLock struct { - s *Storage - l storage.Locker -} - -func (l storageLock) Unlock() { - l.l.Unlock() - l.s.logI("storage lock released") -} - -type reader struct { - s *Storage - fd storage.FileDesc - storage.Reader -} - -func (r *reader) Read(p []byte) (n int, err error) { - err = r.s.emulateError(ModeRead, r.fd.Type) - if err == nil { - r.s.stall(ModeRead, r.fd.Type) - n, err = r.Reader.Read(p) - } - r.s.count(ModeRead, r.fd.Type, n) - if err != nil && err != io.EOF { - r.s.logI("read error, fd=%s n=%d err=%v", r.fd, n, err) - } - return -} - -func (r *reader) ReadAt(p []byte, off int64) (n int, err error) { - err = r.s.emulateError(ModeRead, r.fd.Type) - if err == nil { - r.s.stall(ModeRead, r.fd.Type) - n, err = r.Reader.ReadAt(p, off) - } - r.s.count(ModeRead, r.fd.Type, n) - if err != nil && err != io.EOF { - r.s.logI("readAt error, fd=%s offset=%d n=%d err=%v", r.fd, off, n, err) - } - return -} - -func (r *reader) Close() (err error) { - return r.s.fileClose(r.fd, r.Reader) -} - -type writer struct { - s *Storage - fd storage.FileDesc - storage.Writer -} - -func (w *writer) Write(p []byte) (n int, err error) { - err = w.s.emulateError(ModeWrite, w.fd.Type) - if err == nil { - w.s.stall(ModeWrite, w.fd.Type) - n, err = w.Writer.Write(p) - } - w.s.count(ModeWrite, w.fd.Type, n) - if err != nil && err != io.EOF { - w.s.logI("write error, fd=%s n=%d err=%v", w.fd, n, err) - } - return -} - -func (w *writer) Sync() (err error) { - err = w.s.emulateError(ModeSync, w.fd.Type) - if err == nil { - w.s.stall(ModeSync, w.fd.Type) - err = w.Writer.Sync() - } - w.s.count(ModeSync, w.fd.Type, 0) - if err != nil { - w.s.logI("sync error, fd=%s err=%v", w.fd, err) - } - return -} - -func (w *writer) Close() (err error) { - return w.s.fileClose(w.fd, w.Writer) -} - -type Storage struct { - storage.Storage - path string - onClose func() (preserve bool, err error) - onLog func(str string) - - lmu sync.Mutex - lb bytes.Buffer - - mu sync.Mutex - rand *rand.Rand - // Open files, true=writer, false=reader - opens map[uint64]bool - counters [flattenCount]int - bytesCounter [flattenCount]int64 - emulatedError [flattenCount]error - emulatedErrorOnce [flattenCount]bool - emulatedRandomError [flattenCount]error - emulatedRandomErrorProb [flattenCount]float64 - stallCond sync.Cond - stalled [flattenCount]bool -} - -func (s *Storage) log(skip int, str string) { - s.lmu.Lock() - defer s.lmu.Unlock() - _, file, line, ok := runtime.Caller(skip + 2) - if ok { - // Truncate file name at last file name separator. - if index := strings.LastIndex(file, "/"); index >= 0 { - file = file[index+1:] - } else if index = strings.LastIndex(file, "\\"); index >= 0 { - file = file[index+1:] - } - } else { - file = "???" - line = 1 - } - fmt.Fprintf(&s.lb, "%s:%d: ", file, line) - lines := strings.Split(str, "\n") - if l := len(lines); l > 1 && lines[l-1] == "" { - lines = lines[:l-1] - } - for i, line := range lines { - if i > 0 { - s.lb.WriteString("\n\t") - } - s.lb.WriteString(line) - } - if s.onLog != nil { - s.onLog(s.lb.String()) - s.lb.Reset() - } else { - s.lb.WriteByte('\n') - } -} - -func (s *Storage) logISkip(skip int, format string, args ...interface{}) { - pc, _, _, ok := runtime.Caller(skip + 1) - if ok { - if f := runtime.FuncForPC(pc); f != nil { - fname := f.Name() - if index := strings.LastIndex(fname, "."); index >= 0 { - fname = fname[index+1:] - } - format = fname + ": " + format - } - } - s.log(skip+1, fmt.Sprintf(format, args...)) -} - -func (s *Storage) logI(format string, args ...interface{}) { - s.logISkip(1, format, args...) -} - -func (s *Storage) OnLog(onLog func(log string)) { - s.lmu.Lock() - s.onLog = onLog - if s.lb.Len() != 0 { - log := s.lb.String() - s.onLog(log[:len(log)-1]) - s.lb.Reset() - } - s.lmu.Unlock() -} - -func (s *Storage) Log(str string) { - s.log(1, "Log: "+str) - s.Storage.Log(str) -} - -func (s *Storage) Lock() (l storage.Locker, err error) { - l, err = s.Storage.Lock() - if err != nil { - s.logI("storage locking failed, err=%v", err) - } else { - s.logI("storage locked") - l = storageLock{s, l} - } - return -} - -func (s *Storage) List(t storage.FileType) (fds []storage.FileDesc, err error) { - fds, err = s.Storage.List(t) - if err != nil { - s.logI("list failed, err=%v", err) - return - } - s.logI("list, type=0x%x count=%d", int(t), len(fds)) - return -} - -func (s *Storage) GetMeta() (fd storage.FileDesc, err error) { - fd, err = s.Storage.GetMeta() - if err != nil { - if !os.IsNotExist(err) { - s.logI("get meta failed, err=%v", err) - } - return - } - s.logI("get meta, fd=%s", fd) - return -} - -func (s *Storage) SetMeta(fd storage.FileDesc) error { - ExpectWithOffset(1, fd.Type).To(Equal(storage.TypeManifest)) - err := s.Storage.SetMeta(fd) - if err != nil { - s.logI("set meta failed, fd=%s err=%v", fd, err) - } else { - s.logI("set meta, fd=%s", fd) - } - return err -} - -func (s *Storage) fileClose(fd storage.FileDesc, closer io.Closer) (err error) { - err = s.emulateError(ModeClose, fd.Type) - if err == nil { - s.stall(ModeClose, fd.Type) - } - x := packFile(fd) - s.mu.Lock() - defer s.mu.Unlock() - if err == nil { - ExpectWithOffset(2, s.opens).To(HaveKey(x), "File closed, fd=%s", fd) - err = closer.Close() - } - s.countNB(ModeClose, fd.Type, 0) - writer := s.opens[x] - if err != nil { - s.logISkip(1, "file close failed, fd=%s writer=%v err=%v", fd, writer, err) - } else { - s.logISkip(1, "file closed, fd=%s writer=%v", fd, writer) - delete(s.opens, x) - } - return -} - -func (s *Storage) assertOpen(fd storage.FileDesc) { - x := packFile(fd) - ExpectWithOffset(2, s.opens).NotTo(HaveKey(x), "File open, fd=%s writer=%v", fd, s.opens[x]) -} - -func (s *Storage) Open(fd storage.FileDesc) (r storage.Reader, err error) { - err = s.emulateError(ModeOpen, fd.Type) - if err == nil { - s.stall(ModeOpen, fd.Type) - } - s.mu.Lock() - defer s.mu.Unlock() - if err == nil { - s.assertOpen(fd) - s.countNB(ModeOpen, fd.Type, 0) - r, err = s.Storage.Open(fd) - } - if err != nil { - s.logI("file open failed, fd=%s err=%v", fd, err) - } else { - s.logI("file opened, fd=%s", fd) - s.opens[packFile(fd)] = false - r = &reader{s, fd, r} - } - return -} - -func (s *Storage) Create(fd storage.FileDesc) (w storage.Writer, err error) { - err = s.emulateError(ModeCreate, fd.Type) - if err == nil { - s.stall(ModeCreate, fd.Type) - } - s.mu.Lock() - defer s.mu.Unlock() - if err == nil { - s.assertOpen(fd) - s.countNB(ModeCreate, fd.Type, 0) - w, err = s.Storage.Create(fd) - } - if err != nil { - s.logI("file create failed, fd=%s err=%v", fd, err) - } else { - s.logI("file created, fd=%s", fd) - s.opens[packFile(fd)] = true - w = &writer{s, fd, w} - } - return -} - -func (s *Storage) Remove(fd storage.FileDesc) (err error) { - err = s.emulateError(ModeRemove, fd.Type) - if err == nil { - s.stall(ModeRemove, fd.Type) - } - s.mu.Lock() - defer s.mu.Unlock() - if err == nil { - s.assertOpen(fd) - s.countNB(ModeRemove, fd.Type, 0) - err = s.Storage.Remove(fd) - } - if err != nil { - s.logI("file remove failed, fd=%s err=%v", fd, err) - } else { - s.logI("file removed, fd=%s", fd) - } - return -} - -func (s *Storage) ForceRemove(fd storage.FileDesc) (err error) { - s.countNB(ModeRemove, fd.Type, 0) - if err = s.Storage.Remove(fd); err != nil { - s.logI("file remove failed (forced), fd=%s err=%v", fd, err) - } else { - s.logI("file removed (forced), fd=%s", fd) - } - return -} - -func (s *Storage) Rename(oldfd, newfd storage.FileDesc) (err error) { - err = s.emulateError(ModeRename, oldfd.Type) - if err == nil { - s.stall(ModeRename, oldfd.Type) - } - s.mu.Lock() - defer s.mu.Unlock() - if err == nil { - s.assertOpen(oldfd) - s.assertOpen(newfd) - s.countNB(ModeRename, oldfd.Type, 0) - err = s.Storage.Rename(oldfd, newfd) - } - if err != nil { - s.logI("file rename failed, oldfd=%s newfd=%s err=%v", oldfd, newfd, err) - } else { - s.logI("file renamed, oldfd=%s newfd=%s", oldfd, newfd) - } - return -} - -func (s *Storage) ForceRename(oldfd, newfd storage.FileDesc) (err error) { - s.countNB(ModeRename, oldfd.Type, 0) - if err = s.Storage.Rename(oldfd, newfd); err != nil { - s.logI("file rename failed (forced), oldfd=%s newfd=%s err=%v", oldfd, newfd, err) - } else { - s.logI("file renamed (forced), oldfd=%s newfd=%s", oldfd, newfd) - } - return -} - -func (s *Storage) openFiles() string { - out := "Open files:" - for x, writer := range s.opens { - fd := unpackFile(x) - out += fmt.Sprintf("\n · fd=%s writer=%v", fd, writer) - } - return out -} - -func (s *Storage) CloseCheck() { - s.mu.Lock() - defer s.mu.Unlock() - ExpectWithOffset(1, s.opens).To(BeEmpty(), s.openFiles()) -} - -func (s *Storage) OnClose(onClose func() (preserve bool, err error)) { - s.mu.Lock() - s.onClose = onClose - s.mu.Unlock() -} - -func (s *Storage) Close() error { - s.mu.Lock() - defer s.mu.Unlock() - ExpectWithOffset(1, s.opens).To(BeEmpty(), s.openFiles()) - err := s.Storage.Close() - if err != nil { - s.logI("storage closing failed, err=%v", err) - } else { - s.logI("storage closed") - } - var preserve bool - if s.onClose != nil { - var err0 error - if preserve, err0 = s.onClose(); err0 != nil { - s.logI("onClose error, err=%v", err0) - } - } - if s.path != "" { - if storageKeepFS || preserve { - s.logI("storage is preserved, path=%v", s.path) - } else { - if err1 := os.RemoveAll(s.path); err1 != nil { - s.logI("cannot remove storage, err=%v", err1) - } else { - s.logI("storage has been removed") - } - } - } - return err -} - -func (s *Storage) countNB(m StorageMode, t storage.FileType, n int) { - s.counters[flattenType(m, t)]++ - s.bytesCounter[flattenType(m, t)] += int64(n) -} - -func (s *Storage) count(m StorageMode, t storage.FileType, n int) { - s.mu.Lock() - defer s.mu.Unlock() - s.countNB(m, t, n) -} - -func (s *Storage) ResetCounter(m StorageMode, t storage.FileType) { - for _, x := range listFlattenType(m, t) { - s.counters[x] = 0 - s.bytesCounter[x] = 0 - } -} - -func (s *Storage) Counter(m StorageMode, t storage.FileType) (count int, bytes int64) { - for _, x := range listFlattenType(m, t) { - count += s.counters[x] - bytes += s.bytesCounter[x] - } - return -} - -func (s *Storage) emulateError(m StorageMode, t storage.FileType) error { - s.mu.Lock() - defer s.mu.Unlock() - x := flattenType(m, t) - if err := s.emulatedError[x]; err != nil { - if s.emulatedErrorOnce[x] { - s.emulatedError[x] = nil - } - return emulatedError{err} - } - if err := s.emulatedRandomError[x]; err != nil && s.rand.Float64() < s.emulatedRandomErrorProb[x] { - return emulatedError{err} - } - return nil -} - -func (s *Storage) EmulateError(m StorageMode, t storage.FileType, err error) { - s.mu.Lock() - defer s.mu.Unlock() - for _, x := range listFlattenType(m, t) { - s.emulatedError[x] = err - s.emulatedErrorOnce[x] = false - } -} - -func (s *Storage) EmulateErrorOnce(m StorageMode, t storage.FileType, err error) { - s.mu.Lock() - defer s.mu.Unlock() - for _, x := range listFlattenType(m, t) { - s.emulatedError[x] = err - s.emulatedErrorOnce[x] = true - } -} - -func (s *Storage) EmulateRandomError(m StorageMode, t storage.FileType, prob float64, err error) { - s.mu.Lock() - defer s.mu.Unlock() - for _, x := range listFlattenType(m, t) { - s.emulatedRandomError[x] = err - s.emulatedRandomErrorProb[x] = prob - } -} - -func (s *Storage) stall(m StorageMode, t storage.FileType) { - x := flattenType(m, t) - s.mu.Lock() - defer s.mu.Unlock() - for s.stalled[x] { - s.stallCond.Wait() - } -} - -func (s *Storage) Stall(m StorageMode, t storage.FileType) { - s.mu.Lock() - defer s.mu.Unlock() - for _, x := range listFlattenType(m, t) { - s.stalled[x] = true - } -} - -func (s *Storage) Release(m StorageMode, t storage.FileType) { - s.mu.Lock() - defer s.mu.Unlock() - for _, x := range listFlattenType(m, t) { - s.stalled[x] = false - } - s.stallCond.Broadcast() -} - -func NewStorage() *Storage { - var ( - stor storage.Storage - path string - ) - if storageUseFS { - for { - storageMu.Lock() - num := storageNum - storageNum++ - storageMu.Unlock() - path = filepath.Join(os.TempDir(), fmt.Sprintf("goleveldb-test%d0%d0%d", os.Getuid(), os.Getpid(), num)) - if _, err := os.Stat(path); os.IsNotExist(err) { - stor, err = storage.OpenFile(path, false) - ExpectWithOffset(1, err).NotTo(HaveOccurred(), "creating storage at %s", path) - break - } - } - } else { - stor = storage.NewMemStorage() - } - s := &Storage{ - Storage: stor, - path: path, - rand: NewRand(), - opens: make(map[uint64]bool), - } - s.stallCond.L = &s.mu - if s.path != "" { - s.logI("using FS storage") - s.logI("storage path: %s", s.path) - } else { - s.logI("using MEM storage") - } - return s -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/testutil/util.go b/vendor/github.com/syndtr/goleveldb/leveldb/testutil/util.go deleted file mode 100644 index 97c5294..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/testutil/util.go +++ /dev/null @@ -1,171 +0,0 @@ -// Copyright (c) 2014, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package testutil - -import ( - "bytes" - "flag" - "math/rand" - "reflect" - "sync" - - "github.com/onsi/ginkgo/config" - - "github.com/syndtr/goleveldb/leveldb/comparer" -) - -var ( - runfn = make(map[string][]func()) - runmu sync.Mutex -) - -func Defer(args ...interface{}) bool { - var ( - group string - fn func() - ) - for _, arg := range args { - v := reflect.ValueOf(arg) - switch v.Kind() { - case reflect.String: - group = v.String() - case reflect.Func: - r := reflect.ValueOf(&fn).Elem() - r.Set(v) - } - } - if fn != nil { - runmu.Lock() - runfn[group] = append(runfn[group], fn) - runmu.Unlock() - } - return true -} - -func RunDefer(groups ...string) bool { - if len(groups) == 0 { - groups = append(groups, "") - } - runmu.Lock() - var runfn_ []func() - for _, group := range groups { - runfn_ = append(runfn_, runfn[group]...) - delete(runfn, group) - } - runmu.Unlock() - for _, fn := range runfn_ { - fn() - } - return runfn_ != nil -} - -func RandomSeed() int64 { - if !flag.Parsed() { - panic("random seed not initialized") - } - return config.GinkgoConfig.RandomSeed -} - -func NewRand() *rand.Rand { - return rand.New(rand.NewSource(RandomSeed())) -} - -var cmp = comparer.DefaultComparer - -func BytesSeparator(a, b []byte) []byte { - if bytes.Equal(a, b) { - return b - } - i, n := 0, len(a) - if n > len(b) { - n = len(b) - } - for ; i < n && (a[i] == b[i]); i++ { - } - x := append([]byte{}, a[:i]...) - if i < n { - if c := a[i] + 1; c < b[i] { - return append(x, c) - } - x = append(x, a[i]) - i++ - } - for ; i < len(a); i++ { - if c := a[i]; c < 0xff { - return append(x, c+1) - } else { - x = append(x, c) - } - } - if len(b) > i && b[i] > 0 { - return append(x, b[i]-1) - } - return append(x, 'x') -} - -func BytesAfter(b []byte) []byte { - var x []byte - for _, c := range b { - if c < 0xff { - return append(x, c+1) - } else { - x = append(x, c) - } - } - return append(x, 'x') -} - -func RandomIndex(rnd *rand.Rand, n, round int, fn func(i int)) { - if rnd == nil { - rnd = NewRand() - } - for x := 0; x < round; x++ { - fn(rnd.Intn(n)) - } - return -} - -func ShuffledIndex(rnd *rand.Rand, n, round int, fn func(i int)) { - if rnd == nil { - rnd = NewRand() - } - for x := 0; x < round; x++ { - for _, i := range rnd.Perm(n) { - fn(i) - } - } - return -} - -func RandomRange(rnd *rand.Rand, n, round int, fn func(start, limit int)) { - if rnd == nil { - rnd = NewRand() - } - for x := 0; x < round; x++ { - start := rnd.Intn(n) - length := 0 - if j := n - start; j > 0 { - length = rnd.Intn(j) - } - fn(start, start+length) - } - return -} - -func Max(x, y int) int { - if x > y { - return x - } - return y -} - -func Min(x, y int) int { - if x < y { - return x - } - return y -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/testutil_test.go b/vendor/github.com/syndtr/goleveldb/leveldb/testutil_test.go deleted file mode 100644 index c8cb44c..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/testutil_test.go +++ /dev/null @@ -1,91 +0,0 @@ -// Copyright (c) 2014, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package leveldb - -import ( - . "github.com/onsi/gomega" - - "github.com/syndtr/goleveldb/leveldb/iterator" - "github.com/syndtr/goleveldb/leveldb/opt" - "github.com/syndtr/goleveldb/leveldb/testutil" - "github.com/syndtr/goleveldb/leveldb/util" -) - -type testingDB struct { - *DB - ro *opt.ReadOptions - wo *opt.WriteOptions - stor *testutil.Storage -} - -func (t *testingDB) TestPut(key []byte, value []byte) error { - return t.Put(key, value, t.wo) -} - -func (t *testingDB) TestDelete(key []byte) error { - return t.Delete(key, t.wo) -} - -func (t *testingDB) TestGet(key []byte) (value []byte, err error) { - return t.Get(key, t.ro) -} - -func (t *testingDB) TestHas(key []byte) (ret bool, err error) { - return t.Has(key, t.ro) -} - -func (t *testingDB) TestNewIterator(slice *util.Range) iterator.Iterator { - return t.NewIterator(slice, t.ro) -} - -func (t *testingDB) TestClose() { - err := t.Close() - ExpectWithOffset(1, err).NotTo(HaveOccurred()) - err = t.stor.Close() - ExpectWithOffset(1, err).NotTo(HaveOccurred()) -} - -func newTestingDB(o *opt.Options, ro *opt.ReadOptions, wo *opt.WriteOptions) *testingDB { - stor := testutil.NewStorage() - db, err := Open(stor, o) - // FIXME: This may be called from outside It, which may cause panic. - Expect(err).NotTo(HaveOccurred()) - return &testingDB{ - DB: db, - ro: ro, - wo: wo, - stor: stor, - } -} - -type testingTransaction struct { - *Transaction - ro *opt.ReadOptions - wo *opt.WriteOptions -} - -func (t *testingTransaction) TestPut(key []byte, value []byte) error { - return t.Put(key, value, t.wo) -} - -func (t *testingTransaction) TestDelete(key []byte) error { - return t.Delete(key, t.wo) -} - -func (t *testingTransaction) TestGet(key []byte) (value []byte, err error) { - return t.Get(key, t.ro) -} - -func (t *testingTransaction) TestHas(key []byte) (ret bool, err error) { - return t.Has(key, t.ro) -} - -func (t *testingTransaction) TestNewIterator(slice *util.Range) iterator.Iterator { - return t.NewIterator(slice, t.ro) -} - -func (t *testingTransaction) TestClose() {} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/util.go b/vendor/github.com/syndtr/goleveldb/leveldb/util.go deleted file mode 100644 index e572a32..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/util.go +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright (c) 2012, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package leveldb - -import ( - "fmt" - "sort" - - "github.com/syndtr/goleveldb/leveldb/storage" -) - -func shorten(str string) string { - if len(str) <= 8 { - return str - } - return str[:3] + ".." + str[len(str)-3:] -} - -var bunits = [...]string{"", "Ki", "Mi", "Gi"} - -func shortenb(bytes int) string { - i := 0 - for ; bytes > 1024 && i < 4; i++ { - bytes /= 1024 - } - return fmt.Sprintf("%d%sB", bytes, bunits[i]) -} - -func sshortenb(bytes int) string { - if bytes == 0 { - return "~" - } - sign := "+" - if bytes < 0 { - sign = "-" - bytes *= -1 - } - i := 0 - for ; bytes > 1024 && i < 4; i++ { - bytes /= 1024 - } - return fmt.Sprintf("%s%d%sB", sign, bytes, bunits[i]) -} - -func sint(x int) string { - if x == 0 { - return "~" - } - sign := "+" - if x < 0 { - sign = "-" - x *= -1 - } - return fmt.Sprintf("%s%d", sign, x) -} - -func minInt(a, b int) int { - if a < b { - return a - } - return b -} - -func maxInt(a, b int) int { - if a > b { - return a - } - return b -} - -type fdSorter []storage.FileDesc - -func (p fdSorter) Len() int { - return len(p) -} - -func (p fdSorter) Less(i, j int) bool { - return p[i].Num < p[j].Num -} - -func (p fdSorter) Swap(i, j int) { - p[i], p[j] = p[j], p[i] -} - -func sortFds(fds []storage.FileDesc) { - sort.Sort(fdSorter(fds)) -} - -func ensureBuffer(b []byte, n int) []byte { - if cap(b) < n { - return make([]byte, n) - } - return b[:n] -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/util/buffer.go b/vendor/github.com/syndtr/goleveldb/leveldb/util/buffer.go deleted file mode 100644 index 21de242..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/util/buffer.go +++ /dev/null @@ -1,293 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package util - -// This a copy of Go std bytes.Buffer with some modification -// and some features stripped. - -import ( - "bytes" - "io" -) - -// A Buffer is a variable-sized buffer of bytes with Read and Write methods. -// The zero value for Buffer is an empty buffer ready to use. -type Buffer struct { - buf []byte // contents are the bytes buf[off : len(buf)] - off int // read at &buf[off], write at &buf[len(buf)] - bootstrap [64]byte // memory to hold first slice; helps small buffers (Printf) avoid allocation. -} - -// Bytes returns a slice of the contents of the unread portion of the buffer; -// len(b.Bytes()) == b.Len(). If the caller changes the contents of the -// returned slice, the contents of the buffer will change provided there -// are no intervening method calls on the Buffer. -func (b *Buffer) Bytes() []byte { return b.buf[b.off:] } - -// String returns the contents of the unread portion of the buffer -// as a string. If the Buffer is a nil pointer, it returns "". -func (b *Buffer) String() string { - if b == nil { - // Special case, useful in debugging. - return "" - } - return string(b.buf[b.off:]) -} - -// Len returns the number of bytes of the unread portion of the buffer; -// b.Len() == len(b.Bytes()). -func (b *Buffer) Len() int { return len(b.buf) - b.off } - -// Truncate discards all but the first n unread bytes from the buffer. -// It panics if n is negative or greater than the length of the buffer. -func (b *Buffer) Truncate(n int) { - switch { - case n < 0 || n > b.Len(): - panic("leveldb/util.Buffer: truncation out of range") - case n == 0: - // Reuse buffer space. - b.off = 0 - } - b.buf = b.buf[0 : b.off+n] -} - -// Reset resets the buffer so it has no content. -// b.Reset() is the same as b.Truncate(0). -func (b *Buffer) Reset() { b.Truncate(0) } - -// grow grows the buffer to guarantee space for n more bytes. -// It returns the index where bytes should be written. -// If the buffer can't grow it will panic with bytes.ErrTooLarge. -func (b *Buffer) grow(n int) int { - m := b.Len() - // If buffer is empty, reset to recover space. - if m == 0 && b.off != 0 { - b.Truncate(0) - } - if len(b.buf)+n > cap(b.buf) { - var buf []byte - if b.buf == nil && n <= len(b.bootstrap) { - buf = b.bootstrap[0:] - } else if m+n <= cap(b.buf)/2 { - // We can slide things down instead of allocating a new - // slice. We only need m+n <= cap(b.buf) to slide, but - // we instead let capacity get twice as large so we - // don't spend all our time copying. - copy(b.buf[:], b.buf[b.off:]) - buf = b.buf[:m] - } else { - // not enough space anywhere - buf = makeSlice(2*cap(b.buf) + n) - copy(buf, b.buf[b.off:]) - } - b.buf = buf - b.off = 0 - } - b.buf = b.buf[0 : b.off+m+n] - return b.off + m -} - -// Alloc allocs n bytes of slice from the buffer, growing the buffer as -// needed. If n is negative, Alloc will panic. -// If the buffer can't grow it will panic with bytes.ErrTooLarge. -func (b *Buffer) Alloc(n int) []byte { - if n < 0 { - panic("leveldb/util.Buffer.Alloc: negative count") - } - m := b.grow(n) - return b.buf[m:] -} - -// Grow grows the buffer's capacity, if necessary, to guarantee space for -// another n bytes. After Grow(n), at least n bytes can be written to the -// buffer without another allocation. -// If n is negative, Grow will panic. -// If the buffer can't grow it will panic with bytes.ErrTooLarge. -func (b *Buffer) Grow(n int) { - if n < 0 { - panic("leveldb/util.Buffer.Grow: negative count") - } - m := b.grow(n) - b.buf = b.buf[0:m] -} - -// Write appends the contents of p to the buffer, growing the buffer as -// needed. The return value n is the length of p; err is always nil. If the -// buffer becomes too large, Write will panic with bytes.ErrTooLarge. -func (b *Buffer) Write(p []byte) (n int, err error) { - m := b.grow(len(p)) - return copy(b.buf[m:], p), nil -} - -// MinRead is the minimum slice size passed to a Read call by -// Buffer.ReadFrom. As long as the Buffer has at least MinRead bytes beyond -// what is required to hold the contents of r, ReadFrom will not grow the -// underlying buffer. -const MinRead = 512 - -// ReadFrom reads data from r until EOF and appends it to the buffer, growing -// the buffer as needed. The return value n is the number of bytes read. Any -// error except io.EOF encountered during the read is also returned. If the -// buffer becomes too large, ReadFrom will panic with bytes.ErrTooLarge. -func (b *Buffer) ReadFrom(r io.Reader) (n int64, err error) { - // If buffer is empty, reset to recover space. - if b.off >= len(b.buf) { - b.Truncate(0) - } - for { - if free := cap(b.buf) - len(b.buf); free < MinRead { - // not enough space at end - newBuf := b.buf - if b.off+free < MinRead { - // not enough space using beginning of buffer; - // double buffer capacity - newBuf = makeSlice(2*cap(b.buf) + MinRead) - } - copy(newBuf, b.buf[b.off:]) - b.buf = newBuf[:len(b.buf)-b.off] - b.off = 0 - } - m, e := r.Read(b.buf[len(b.buf):cap(b.buf)]) - b.buf = b.buf[0 : len(b.buf)+m] - n += int64(m) - if e == io.EOF { - break - } - if e != nil { - return n, e - } - } - return n, nil // err is EOF, so return nil explicitly -} - -// makeSlice allocates a slice of size n. If the allocation fails, it panics -// with bytes.ErrTooLarge. -func makeSlice(n int) []byte { - // If the make fails, give a known error. - defer func() { - if recover() != nil { - panic(bytes.ErrTooLarge) - } - }() - return make([]byte, n) -} - -// WriteTo writes data to w until the buffer is drained or an error occurs. -// The return value n is the number of bytes written; it always fits into an -// int, but it is int64 to match the io.WriterTo interface. Any error -// encountered during the write is also returned. -func (b *Buffer) WriteTo(w io.Writer) (n int64, err error) { - if b.off < len(b.buf) { - nBytes := b.Len() - m, e := w.Write(b.buf[b.off:]) - if m > nBytes { - panic("leveldb/util.Buffer.WriteTo: invalid Write count") - } - b.off += m - n = int64(m) - if e != nil { - return n, e - } - // all bytes should have been written, by definition of - // Write method in io.Writer - if m != nBytes { - return n, io.ErrShortWrite - } - } - // Buffer is now empty; reset. - b.Truncate(0) - return -} - -// WriteByte appends the byte c to the buffer, growing the buffer as needed. -// The returned error is always nil, but is included to match bufio.Writer's -// WriteByte. If the buffer becomes too large, WriteByte will panic with -// bytes.ErrTooLarge. -func (b *Buffer) WriteByte(c byte) error { - m := b.grow(1) - b.buf[m] = c - return nil -} - -// Read reads the next len(p) bytes from the buffer or until the buffer -// is drained. The return value n is the number of bytes read. If the -// buffer has no data to return, err is io.EOF (unless len(p) is zero); -// otherwise it is nil. -func (b *Buffer) Read(p []byte) (n int, err error) { - if b.off >= len(b.buf) { - // Buffer is empty, reset to recover space. - b.Truncate(0) - if len(p) == 0 { - return - } - return 0, io.EOF - } - n = copy(p, b.buf[b.off:]) - b.off += n - return -} - -// Next returns a slice containing the next n bytes from the buffer, -// advancing the buffer as if the bytes had been returned by Read. -// If there are fewer than n bytes in the buffer, Next returns the entire buffer. -// The slice is only valid until the next call to a read or write method. -func (b *Buffer) Next(n int) []byte { - m := b.Len() - if n > m { - n = m - } - data := b.buf[b.off : b.off+n] - b.off += n - return data -} - -// ReadByte reads and returns the next byte from the buffer. -// If no byte is available, it returns error io.EOF. -func (b *Buffer) ReadByte() (c byte, err error) { - if b.off >= len(b.buf) { - // Buffer is empty, reset to recover space. - b.Truncate(0) - return 0, io.EOF - } - c = b.buf[b.off] - b.off++ - return c, nil -} - -// ReadBytes reads until the first occurrence of delim in the input, -// returning a slice containing the data up to and including the delimiter. -// If ReadBytes encounters an error before finding a delimiter, -// it returns the data read before the error and the error itself (often io.EOF). -// ReadBytes returns err != nil if and only if the returned data does not end in -// delim. -func (b *Buffer) ReadBytes(delim byte) (line []byte, err error) { - slice, err := b.readSlice(delim) - // return a copy of slice. The buffer's backing array may - // be overwritten by later calls. - line = append(line, slice...) - return -} - -// readSlice is like ReadBytes but returns a reference to internal buffer data. -func (b *Buffer) readSlice(delim byte) (line []byte, err error) { - i := bytes.IndexByte(b.buf[b.off:], delim) - end := b.off + i + 1 - if i < 0 { - end = len(b.buf) - err = io.EOF - } - line = b.buf[b.off:end] - b.off = end - return line, err -} - -// NewBuffer creates and initializes a new Buffer using buf as its initial -// contents. It is intended to prepare a Buffer to read existing data. It -// can also be used to size the internal buffer for writing. To do that, -// buf should have the desired capacity but a length of zero. -// -// In most cases, new(Buffer) (or just declaring a Buffer variable) is -// sufficient to initialize a Buffer. -func NewBuffer(buf []byte) *Buffer { return &Buffer{buf: buf} } diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/util/buffer_pool.go b/vendor/github.com/syndtr/goleveldb/leveldb/util/buffer_pool.go deleted file mode 100644 index 2f3db97..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/util/buffer_pool.go +++ /dev/null @@ -1,239 +0,0 @@ -// Copyright (c) 2014, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package util - -import ( - "fmt" - "sync" - "sync/atomic" - "time" -) - -type buffer struct { - b []byte - miss int -} - -// BufferPool is a 'buffer pool'. -type BufferPool struct { - pool [6]chan []byte - size [5]uint32 - sizeMiss [5]uint32 - sizeHalf [5]uint32 - baseline [4]int - baseline0 int - - mu sync.RWMutex - closed bool - closeC chan struct{} - - get uint32 - put uint32 - half uint32 - less uint32 - equal uint32 - greater uint32 - miss uint32 -} - -func (p *BufferPool) poolNum(n int) int { - if n <= p.baseline0 && n > p.baseline0/2 { - return 0 - } - for i, x := range p.baseline { - if n <= x { - return i + 1 - } - } - return len(p.baseline) + 1 -} - -// Get returns buffer with length of n. -func (p *BufferPool) Get(n int) []byte { - if p == nil { - return make([]byte, n) - } - - p.mu.RLock() - defer p.mu.RUnlock() - - if p.closed { - return make([]byte, n) - } - - atomic.AddUint32(&p.get, 1) - - poolNum := p.poolNum(n) - pool := p.pool[poolNum] - if poolNum == 0 { - // Fast path. - select { - case b := <-pool: - switch { - case cap(b) > n: - if cap(b)-n >= n { - atomic.AddUint32(&p.half, 1) - select { - case pool <- b: - default: - } - return make([]byte, n) - } else { - atomic.AddUint32(&p.less, 1) - return b[:n] - } - case cap(b) == n: - atomic.AddUint32(&p.equal, 1) - return b[:n] - default: - atomic.AddUint32(&p.greater, 1) - } - default: - atomic.AddUint32(&p.miss, 1) - } - - return make([]byte, n, p.baseline0) - } else { - sizePtr := &p.size[poolNum-1] - - select { - case b := <-pool: - switch { - case cap(b) > n: - if cap(b)-n >= n { - atomic.AddUint32(&p.half, 1) - sizeHalfPtr := &p.sizeHalf[poolNum-1] - if atomic.AddUint32(sizeHalfPtr, 1) == 20 { - atomic.StoreUint32(sizePtr, uint32(cap(b)/2)) - atomic.StoreUint32(sizeHalfPtr, 0) - } else { - select { - case pool <- b: - default: - } - } - return make([]byte, n) - } else { - atomic.AddUint32(&p.less, 1) - return b[:n] - } - case cap(b) == n: - atomic.AddUint32(&p.equal, 1) - return b[:n] - default: - atomic.AddUint32(&p.greater, 1) - if uint32(cap(b)) >= atomic.LoadUint32(sizePtr) { - select { - case pool <- b: - default: - } - } - } - default: - atomic.AddUint32(&p.miss, 1) - } - - if size := atomic.LoadUint32(sizePtr); uint32(n) > size { - if size == 0 { - atomic.CompareAndSwapUint32(sizePtr, 0, uint32(n)) - } else { - sizeMissPtr := &p.sizeMiss[poolNum-1] - if atomic.AddUint32(sizeMissPtr, 1) == 20 { - atomic.StoreUint32(sizePtr, uint32(n)) - atomic.StoreUint32(sizeMissPtr, 0) - } - } - return make([]byte, n) - } else { - return make([]byte, n, size) - } - } -} - -// Put adds given buffer to the pool. -func (p *BufferPool) Put(b []byte) { - if p == nil { - return - } - - p.mu.RLock() - defer p.mu.RUnlock() - - if p.closed { - return - } - - atomic.AddUint32(&p.put, 1) - - pool := p.pool[p.poolNum(cap(b))] - select { - case pool <- b: - default: - } - -} - -func (p *BufferPool) Close() { - if p == nil { - return - } - - p.mu.Lock() - if !p.closed { - p.closed = true - p.closeC <- struct{}{} - } - p.mu.Unlock() -} - -func (p *BufferPool) String() string { - if p == nil { - return "" - } - - return fmt.Sprintf("BufferPool{B·%d Z·%v Zm·%v Zh·%v G·%d P·%d H·%d <·%d =·%d >·%d M·%d}", - p.baseline0, p.size, p.sizeMiss, p.sizeHalf, p.get, p.put, p.half, p.less, p.equal, p.greater, p.miss) -} - -func (p *BufferPool) drain() { - ticker := time.NewTicker(2 * time.Second) - defer ticker.Stop() - for { - select { - case <-ticker.C: - for _, ch := range p.pool { - select { - case <-ch: - default: - } - } - case <-p.closeC: - close(p.closeC) - for _, ch := range p.pool { - close(ch) - } - return - } - } -} - -// NewBufferPool creates a new initialized 'buffer pool'. -func NewBufferPool(baseline int) *BufferPool { - if baseline <= 0 { - panic("baseline can't be <= 0") - } - p := &BufferPool{ - baseline0: baseline, - baseline: [...]int{baseline / 4, baseline / 2, baseline * 2, baseline * 4}, - closeC: make(chan struct{}, 1), - } - for i, cap := range []int{2, 2, 4, 4, 2, 1} { - p.pool[i] = make(chan []byte, cap) - } - go p.drain() - return p -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/util/buffer_test.go b/vendor/github.com/syndtr/goleveldb/leveldb/util/buffer_test.go deleted file mode 100644 index 87d9673..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/util/buffer_test.go +++ /dev/null @@ -1,369 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package util - -import ( - "bytes" - "io" - "math/rand" - "runtime" - "testing" -) - -const N = 10000 // make this bigger for a larger (and slower) test -var data string // test data for write tests -var testBytes []byte // test data; same as data but as a slice. - -func init() { - testBytes = make([]byte, N) - for i := 0; i < N; i++ { - testBytes[i] = 'a' + byte(i%26) - } - data = string(testBytes) -} - -// Verify that contents of buf match the string s. -func check(t *testing.T, testname string, buf *Buffer, s string) { - bytes := buf.Bytes() - str := buf.String() - if buf.Len() != len(bytes) { - t.Errorf("%s: buf.Len() == %d, len(buf.Bytes()) == %d", testname, buf.Len(), len(bytes)) - } - - if buf.Len() != len(str) { - t.Errorf("%s: buf.Len() == %d, len(buf.String()) == %d", testname, buf.Len(), len(str)) - } - - if buf.Len() != len(s) { - t.Errorf("%s: buf.Len() == %d, len(s) == %d", testname, buf.Len(), len(s)) - } - - if string(bytes) != s { - t.Errorf("%s: string(buf.Bytes()) == %q, s == %q", testname, string(bytes), s) - } -} - -// Fill buf through n writes of byte slice fub. -// The initial contents of buf corresponds to the string s; -// the result is the final contents of buf returned as a string. -func fillBytes(t *testing.T, testname string, buf *Buffer, s string, n int, fub []byte) string { - check(t, testname+" (fill 1)", buf, s) - for ; n > 0; n-- { - m, err := buf.Write(fub) - if m != len(fub) { - t.Errorf(testname+" (fill 2): m == %d, expected %d", m, len(fub)) - } - if err != nil { - t.Errorf(testname+" (fill 3): err should always be nil, found err == %s", err) - } - s += string(fub) - check(t, testname+" (fill 4)", buf, s) - } - return s -} - -func TestNewBuffer(t *testing.T) { - buf := NewBuffer(testBytes) - check(t, "NewBuffer", buf, data) -} - -// Empty buf through repeated reads into fub. -// The initial contents of buf corresponds to the string s. -func empty(t *testing.T, testname string, buf *Buffer, s string, fub []byte) { - check(t, testname+" (empty 1)", buf, s) - - for { - n, err := buf.Read(fub) - if n == 0 { - break - } - if err != nil { - t.Errorf(testname+" (empty 2): err should always be nil, found err == %s", err) - } - s = s[n:] - check(t, testname+" (empty 3)", buf, s) - } - - check(t, testname+" (empty 4)", buf, "") -} - -func TestBasicOperations(t *testing.T) { - var buf Buffer - - for i := 0; i < 5; i++ { - check(t, "TestBasicOperations (1)", &buf, "") - - buf.Reset() - check(t, "TestBasicOperations (2)", &buf, "") - - buf.Truncate(0) - check(t, "TestBasicOperations (3)", &buf, "") - - n, err := buf.Write([]byte(data[0:1])) - if n != 1 { - t.Errorf("wrote 1 byte, but n == %d", n) - } - if err != nil { - t.Errorf("err should always be nil, but err == %s", err) - } - check(t, "TestBasicOperations (4)", &buf, "a") - - buf.WriteByte(data[1]) - check(t, "TestBasicOperations (5)", &buf, "ab") - - n, err = buf.Write([]byte(data[2:26])) - if n != 24 { - t.Errorf("wrote 25 bytes, but n == %d", n) - } - check(t, "TestBasicOperations (6)", &buf, string(data[0:26])) - - buf.Truncate(26) - check(t, "TestBasicOperations (7)", &buf, string(data[0:26])) - - buf.Truncate(20) - check(t, "TestBasicOperations (8)", &buf, string(data[0:20])) - - empty(t, "TestBasicOperations (9)", &buf, string(data[0:20]), make([]byte, 5)) - empty(t, "TestBasicOperations (10)", &buf, "", make([]byte, 100)) - - buf.WriteByte(data[1]) - c, err := buf.ReadByte() - if err != nil { - t.Error("ReadByte unexpected eof") - } - if c != data[1] { - t.Errorf("ReadByte wrong value c=%v", c) - } - c, err = buf.ReadByte() - if err == nil { - t.Error("ReadByte unexpected not eof") - } - } -} - -func TestLargeByteWrites(t *testing.T) { - var buf Buffer - limit := 30 - if testing.Short() { - limit = 9 - } - for i := 3; i < limit; i += 3 { - s := fillBytes(t, "TestLargeWrites (1)", &buf, "", 5, testBytes) - empty(t, "TestLargeByteWrites (2)", &buf, s, make([]byte, len(data)/i)) - } - check(t, "TestLargeByteWrites (3)", &buf, "") -} - -func TestLargeByteReads(t *testing.T) { - var buf Buffer - for i := 3; i < 30; i += 3 { - s := fillBytes(t, "TestLargeReads (1)", &buf, "", 5, testBytes[0:len(testBytes)/i]) - empty(t, "TestLargeReads (2)", &buf, s, make([]byte, len(data))) - } - check(t, "TestLargeByteReads (3)", &buf, "") -} - -func TestMixedReadsAndWrites(t *testing.T) { - var buf Buffer - s := "" - for i := 0; i < 50; i++ { - wlen := rand.Intn(len(data)) - s = fillBytes(t, "TestMixedReadsAndWrites (1)", &buf, s, 1, testBytes[0:wlen]) - rlen := rand.Intn(len(data)) - fub := make([]byte, rlen) - n, _ := buf.Read(fub) - s = s[n:] - } - empty(t, "TestMixedReadsAndWrites (2)", &buf, s, make([]byte, buf.Len())) -} - -func TestNil(t *testing.T) { - var b *Buffer - if b.String() != "" { - t.Errorf("expected ; got %q", b.String()) - } -} - -func TestReadFrom(t *testing.T) { - var buf Buffer - for i := 3; i < 30; i += 3 { - s := fillBytes(t, "TestReadFrom (1)", &buf, "", 5, testBytes[0:len(testBytes)/i]) - var b Buffer - b.ReadFrom(&buf) - empty(t, "TestReadFrom (2)", &b, s, make([]byte, len(data))) - } -} - -func TestWriteTo(t *testing.T) { - var buf Buffer - for i := 3; i < 30; i += 3 { - s := fillBytes(t, "TestWriteTo (1)", &buf, "", 5, testBytes[0:len(testBytes)/i]) - var b Buffer - buf.WriteTo(&b) - empty(t, "TestWriteTo (2)", &b, s, make([]byte, len(data))) - } -} - -func TestNext(t *testing.T) { - b := []byte{0, 1, 2, 3, 4} - tmp := make([]byte, 5) - for i := 0; i <= 5; i++ { - for j := i; j <= 5; j++ { - for k := 0; k <= 6; k++ { - // 0 <= i <= j <= 5; 0 <= k <= 6 - // Check that if we start with a buffer - // of length j at offset i and ask for - // Next(k), we get the right bytes. - buf := NewBuffer(b[0:j]) - n, _ := buf.Read(tmp[0:i]) - if n != i { - t.Fatalf("Read %d returned %d", i, n) - } - bb := buf.Next(k) - want := k - if want > j-i { - want = j - i - } - if len(bb) != want { - t.Fatalf("in %d,%d: len(Next(%d)) == %d", i, j, k, len(bb)) - } - for l, v := range bb { - if v != byte(l+i) { - t.Fatalf("in %d,%d: Next(%d)[%d] = %d, want %d", i, j, k, l, v, l+i) - } - } - } - } - } -} - -var readBytesTests = []struct { - buffer string - delim byte - expected []string - err error -}{ - {"", 0, []string{""}, io.EOF}, - {"a\x00", 0, []string{"a\x00"}, nil}, - {"abbbaaaba", 'b', []string{"ab", "b", "b", "aaab"}, nil}, - {"hello\x01world", 1, []string{"hello\x01"}, nil}, - {"foo\nbar", 0, []string{"foo\nbar"}, io.EOF}, - {"alpha\nbeta\ngamma\n", '\n', []string{"alpha\n", "beta\n", "gamma\n"}, nil}, - {"alpha\nbeta\ngamma", '\n', []string{"alpha\n", "beta\n", "gamma"}, io.EOF}, -} - -func TestReadBytes(t *testing.T) { - for _, test := range readBytesTests { - buf := NewBuffer([]byte(test.buffer)) - var err error - for _, expected := range test.expected { - var bytes []byte - bytes, err = buf.ReadBytes(test.delim) - if string(bytes) != expected { - t.Errorf("expected %q, got %q", expected, bytes) - } - if err != nil { - break - } - } - if err != test.err { - t.Errorf("expected error %v, got %v", test.err, err) - } - } -} - -func TestGrow(t *testing.T) { - x := []byte{'x'} - y := []byte{'y'} - tmp := make([]byte, 72) - for _, startLen := range []int{0, 100, 1000, 10000, 100000} { - xBytes := bytes.Repeat(x, startLen) - for _, growLen := range []int{0, 100, 1000, 10000, 100000} { - buf := NewBuffer(xBytes) - // If we read, this affects buf.off, which is good to test. - readBytes, _ := buf.Read(tmp) - buf.Grow(growLen) - yBytes := bytes.Repeat(y, growLen) - // Check no allocation occurs in write, as long as we're single-threaded. - var m1, m2 runtime.MemStats - runtime.ReadMemStats(&m1) - buf.Write(yBytes) - runtime.ReadMemStats(&m2) - if runtime.GOMAXPROCS(-1) == 1 && m1.Mallocs != m2.Mallocs { - t.Errorf("allocation occurred during write") - } - // Check that buffer has correct data. - if !bytes.Equal(buf.Bytes()[0:startLen-readBytes], xBytes[readBytes:]) { - t.Errorf("bad initial data at %d %d", startLen, growLen) - } - if !bytes.Equal(buf.Bytes()[startLen-readBytes:startLen-readBytes+growLen], yBytes) { - t.Errorf("bad written data at %d %d", startLen, growLen) - } - } - } -} - -// Was a bug: used to give EOF reading empty slice at EOF. -func TestReadEmptyAtEOF(t *testing.T) { - b := new(Buffer) - slice := make([]byte, 0) - n, err := b.Read(slice) - if err != nil { - t.Errorf("read error: %v", err) - } - if n != 0 { - t.Errorf("wrong count; got %d want 0", n) - } -} - -// Tests that we occasionally compact. Issue 5154. -func TestBufferGrowth(t *testing.T) { - var b Buffer - buf := make([]byte, 1024) - b.Write(buf[0:1]) - var cap0 int - for i := 0; i < 5<<10; i++ { - b.Write(buf) - b.Read(buf) - if i == 0 { - cap0 = cap(b.buf) - } - } - cap1 := cap(b.buf) - // (*Buffer).grow allows for 2x capacity slop before sliding, - // so set our error threshold at 3x. - if cap1 > cap0*3 { - t.Errorf("buffer cap = %d; too big (grew from %d)", cap1, cap0) - } -} - -// From Issue 5154. -func BenchmarkBufferNotEmptyWriteRead(b *testing.B) { - buf := make([]byte, 1024) - for i := 0; i < b.N; i++ { - var b Buffer - b.Write(buf[0:1]) - for i := 0; i < 5<<10; i++ { - b.Write(buf) - b.Read(buf) - } - } -} - -// Check that we don't compact too often. From Issue 5154. -func BenchmarkBufferFullSmallReads(b *testing.B) { - buf := make([]byte, 1024) - for i := 0; i < b.N; i++ { - var b Buffer - b.Write(buf) - for b.Len()+20 < cap(b.buf) { - b.Write(buf[:10]) - } - for i := 0; i < 5<<10; i++ { - b.Read(buf[:1]) - b.Write(buf[:1]) - } - } -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/util/crc32.go b/vendor/github.com/syndtr/goleveldb/leveldb/util/crc32.go deleted file mode 100644 index 631c9d6..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/util/crc32.go +++ /dev/null @@ -1,30 +0,0 @@ -// Copyright 2011 The LevelDB-Go Authors. All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package util - -import ( - "hash/crc32" -) - -var table = crc32.MakeTable(crc32.Castagnoli) - -// CRC is a CRC-32 checksum computed using Castagnoli's polynomial. -type CRC uint32 - -// NewCRC creates a new crc based on the given bytes. -func NewCRC(b []byte) CRC { - return CRC(0).Update(b) -} - -// Update updates the crc with the given bytes. -func (c CRC) Update(b []byte) CRC { - return CRC(crc32.Update(uint32(c), table, b)) -} - -// Value returns a masked crc. -func (c CRC) Value() uint32 { - return uint32(c>>15|c<<17) + 0xa282ead8 -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/util/hash.go b/vendor/github.com/syndtr/goleveldb/leveldb/util/hash.go deleted file mode 100644 index 7f3fa4e..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/util/hash.go +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright (c) 2012, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package util - -import ( - "encoding/binary" -) - -// Hash return hash of the given data. -func Hash(data []byte, seed uint32) uint32 { - // Similar to murmur hash - const ( - m = uint32(0xc6a4a793) - r = uint32(24) - ) - var ( - h = seed ^ (uint32(len(data)) * m) - i int - ) - - for n := len(data) - len(data)%4; i < n; i += 4 { - h += binary.LittleEndian.Uint32(data[i:]) - h *= m - h ^= (h >> 16) - } - - switch len(data) - i { - default: - panic("not reached") - case 3: - h += uint32(data[i+2]) << 16 - fallthrough - case 2: - h += uint32(data[i+1]) << 8 - fallthrough - case 1: - h += uint32(data[i]) - h *= m - h ^= (h >> r) - case 0: - } - - return h -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/util/hash_test.go b/vendor/github.com/syndtr/goleveldb/leveldb/util/hash_test.go deleted file mode 100644 index a35d273..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/util/hash_test.go +++ /dev/null @@ -1,46 +0,0 @@ -// Copyright (c) 2012, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package util - -import ( - "testing" -) - -var hashTests = []struct { - data []byte - seed uint32 - hash uint32 -}{ - {nil, 0xbc9f1d34, 0xbc9f1d34}, - {[]byte{0x62}, 0xbc9f1d34, 0xef1345c4}, - {[]byte{0xc3, 0x97}, 0xbc9f1d34, 0x5b663814}, - {[]byte{0xe2, 0x99, 0xa5}, 0xbc9f1d34, 0x323c078f}, - {[]byte{0xe1, 0x80, 0xb9, 0x32}, 0xbc9f1d34, 0xed21633a}, - {[]byte{ - 0x01, 0xc0, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x14, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x04, 0x00, - 0x00, 0x00, 0x00, 0x14, - 0x00, 0x00, 0x00, 0x18, - 0x28, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - 0x02, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, - }, 0x12345678, 0xf333dabb}, -} - -func TestHash(t *testing.T) { - for i, x := range hashTests { - h := Hash(x.data, x.seed) - if h != x.hash { - t.Fatalf("test-%d: invalid hash, %#x vs %#x", i, h, x.hash) - } - } -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/util/range.go b/vendor/github.com/syndtr/goleveldb/leveldb/util/range.go deleted file mode 100644 index 8515958..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/util/range.go +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright (c) 2014, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package util - -// Range is a key range. -type Range struct { - // Start of the key range, include in the range. - Start []byte - - // Limit of the key range, not include in the range. - Limit []byte -} - -// BytesPrefix returns key range that satisfy the given prefix. -// This only applicable for the standard 'bytes comparer'. -func BytesPrefix(prefix []byte) *Range { - var limit []byte - for i := len(prefix) - 1; i >= 0; i-- { - c := prefix[i] - if c < 0xff { - limit = make([]byte, i+1) - copy(limit, prefix) - limit[i] = c + 1 - break - } - } - return &Range{prefix, limit} -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/util/util.go b/vendor/github.com/syndtr/goleveldb/leveldb/util/util.go deleted file mode 100644 index f359768..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/util/util.go +++ /dev/null @@ -1,73 +0,0 @@ -// Copyright (c) 2013, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -// Package util provides utilities used throughout leveldb. -package util - -import ( - "errors" -) - -var ( - ErrReleased = errors.New("leveldb: resource already relesed") - ErrHasReleaser = errors.New("leveldb: releaser already defined") -) - -// Releaser is the interface that wraps the basic Release method. -type Releaser interface { - // Release releases associated resources. Release should always success - // and can be called multipe times without causing error. - Release() -} - -// ReleaseSetter is the interface that wraps the basic SetReleaser method. -type ReleaseSetter interface { - // SetReleaser associates the given releaser to the resources. The - // releaser will be called once coresponding resources released. - // Calling SetReleaser with nil will clear the releaser. - // - // This will panic if a releaser already present or coresponding - // resource is already released. Releaser should be cleared first - // before assigned a new one. - SetReleaser(releaser Releaser) -} - -// BasicReleaser provides basic implementation of Releaser and ReleaseSetter. -type BasicReleaser struct { - releaser Releaser - released bool -} - -// Released returns whether Release method already called. -func (r *BasicReleaser) Released() bool { - return r.released -} - -// Release implements Releaser.Release. -func (r *BasicReleaser) Release() { - if !r.released { - if r.releaser != nil { - r.releaser.Release() - r.releaser = nil - } - r.released = true - } -} - -// SetReleaser implements ReleaseSetter.SetReleaser. -func (r *BasicReleaser) SetReleaser(releaser Releaser) { - if r.released { - panic(ErrReleased) - } - if r.releaser != nil && releaser != nil { - panic(ErrHasReleaser) - } - r.releaser = releaser -} - -type NoopReleaser struct{} - -func (NoopReleaser) Release() {} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/version.go b/vendor/github.com/syndtr/goleveldb/leveldb/version.go deleted file mode 100644 index 73f272a..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/version.go +++ /dev/null @@ -1,528 +0,0 @@ -// Copyright (c) 2012, Suryandaru Triandana -// All rights reserved. -// -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -package leveldb - -import ( - "fmt" - "sync/atomic" - "unsafe" - - "github.com/syndtr/goleveldb/leveldb/iterator" - "github.com/syndtr/goleveldb/leveldb/opt" - "github.com/syndtr/goleveldb/leveldb/util" -) - -type tSet struct { - level int - table *tFile -} - -type version struct { - s *session - - levels []tFiles - - // Level that should be compacted next and its compaction score. - // Score < 1 means compaction is not strictly needed. These fields - // are initialized by computeCompaction() - cLevel int - cScore float64 - - cSeek unsafe.Pointer - - closing bool - ref int - released bool -} - -func newVersion(s *session) *version { - return &version{s: s} -} - -func (v *version) incref() { - if v.released { - panic("already released") - } - - v.ref++ - if v.ref == 1 { - // Incr file ref. - for _, tt := range v.levels { - for _, t := range tt { - v.s.addFileRef(t.fd, 1) - } - } - } -} - -func (v *version) releaseNB() { - v.ref-- - if v.ref > 0 { - return - } else if v.ref < 0 { - panic("negative version ref") - } - - for _, tt := range v.levels { - for _, t := range tt { - if v.s.addFileRef(t.fd, -1) == 0 { - v.s.tops.remove(t) - } - } - } - - v.released = true -} - -func (v *version) release() { - v.s.vmu.Lock() - v.releaseNB() - v.s.vmu.Unlock() -} - -func (v *version) walkOverlapping(aux tFiles, ikey internalKey, f func(level int, t *tFile) bool, lf func(level int) bool) { - ukey := ikey.ukey() - - // Aux level. - if aux != nil { - for _, t := range aux { - if t.overlaps(v.s.icmp, ukey, ukey) { - if !f(-1, t) { - return - } - } - } - - if lf != nil && !lf(-1) { - return - } - } - - // Walk tables level-by-level. - for level, tables := range v.levels { - if len(tables) == 0 { - continue - } - - if level == 0 { - // Level-0 files may overlap each other. Find all files that - // overlap ukey. - for _, t := range tables { - if t.overlaps(v.s.icmp, ukey, ukey) { - if !f(level, t) { - return - } - } - } - } else { - if i := tables.searchMax(v.s.icmp, ikey); i < len(tables) { - t := tables[i] - if v.s.icmp.uCompare(ukey, t.imin.ukey()) >= 0 { - if !f(level, t) { - return - } - } - } - } - - if lf != nil && !lf(level) { - return - } - } -} - -func (v *version) get(aux tFiles, ikey internalKey, ro *opt.ReadOptions, noValue bool) (value []byte, tcomp bool, err error) { - if v.closing { - return nil, false, ErrClosed - } - - ukey := ikey.ukey() - - var ( - tset *tSet - tseek bool - - // Level-0. - zfound bool - zseq uint64 - zkt keyType - zval []byte - ) - - err = ErrNotFound - - // Since entries never hop across level, finding key/value - // in smaller level make later levels irrelevant. - v.walkOverlapping(aux, ikey, func(level int, t *tFile) bool { - if level >= 0 && !tseek { - if tset == nil { - tset = &tSet{level, t} - } else { - tseek = true - } - } - - var ( - fikey, fval []byte - ferr error - ) - if noValue { - fikey, ferr = v.s.tops.findKey(t, ikey, ro) - } else { - fikey, fval, ferr = v.s.tops.find(t, ikey, ro) - } - - switch ferr { - case nil: - case ErrNotFound: - return true - default: - err = ferr - return false - } - - if fukey, fseq, fkt, fkerr := parseInternalKey(fikey); fkerr == nil { - if v.s.icmp.uCompare(ukey, fukey) == 0 { - // Level <= 0 may overlaps each-other. - if level <= 0 { - if fseq >= zseq { - zfound = true - zseq = fseq - zkt = fkt - zval = fval - } - } else { - switch fkt { - case keyTypeVal: - value = fval - err = nil - case keyTypeDel: - default: - panic("leveldb: invalid internalKey type") - } - return false - } - } - } else { - err = fkerr - return false - } - - return true - }, func(level int) bool { - if zfound { - switch zkt { - case keyTypeVal: - value = zval - err = nil - case keyTypeDel: - default: - panic("leveldb: invalid internalKey type") - } - return false - } - - return true - }) - - if tseek && tset.table.consumeSeek() <= 0 { - tcomp = atomic.CompareAndSwapPointer(&v.cSeek, nil, unsafe.Pointer(tset)) - } - - return -} - -func (v *version) sampleSeek(ikey internalKey) (tcomp bool) { - var tset *tSet - - v.walkOverlapping(nil, ikey, func(level int, t *tFile) bool { - if tset == nil { - tset = &tSet{level, t} - return true - } - if tset.table.consumeSeek() <= 0 { - tcomp = atomic.CompareAndSwapPointer(&v.cSeek, nil, unsafe.Pointer(tset)) - } - return false - }, nil) - - return -} - -func (v *version) getIterators(slice *util.Range, ro *opt.ReadOptions) (its []iterator.Iterator) { - strict := opt.GetStrict(v.s.o.Options, ro, opt.StrictReader) - for level, tables := range v.levels { - if level == 0 { - // Merge all level zero files together since they may overlap. - for _, t := range tables { - its = append(its, v.s.tops.newIterator(t, slice, ro)) - } - } else if len(tables) != 0 { - its = append(its, iterator.NewIndexedIterator(tables.newIndexIterator(v.s.tops, v.s.icmp, slice, ro), strict)) - } - } - return -} - -func (v *version) newStaging() *versionStaging { - return &versionStaging{base: v} -} - -// Spawn a new version based on this version. -func (v *version) spawn(r *sessionRecord) *version { - staging := v.newStaging() - staging.commit(r) - return staging.finish() -} - -func (v *version) fillRecord(r *sessionRecord) { - for level, tables := range v.levels { - for _, t := range tables { - r.addTableFile(level, t) - } - } -} - -func (v *version) tLen(level int) int { - if level < len(v.levels) { - return len(v.levels[level]) - } - return 0 -} - -func (v *version) offsetOf(ikey internalKey) (n int64, err error) { - for level, tables := range v.levels { - for _, t := range tables { - if v.s.icmp.Compare(t.imax, ikey) <= 0 { - // Entire file is before "ikey", so just add the file size - n += t.size - } else if v.s.icmp.Compare(t.imin, ikey) > 0 { - // Entire file is after "ikey", so ignore - if level > 0 { - // Files other than level 0 are sorted by meta->min, so - // no further files in this level will contain data for - // "ikey". - break - } - } else { - // "ikey" falls in the range for this table. Add the - // approximate offset of "ikey" within the table. - if m, err := v.s.tops.offsetOf(t, ikey); err == nil { - n += m - } else { - return 0, err - } - } - } - } - - return -} - -func (v *version) pickMemdbLevel(umin, umax []byte, maxLevel int) (level int) { - if maxLevel > 0 { - if len(v.levels) == 0 { - return maxLevel - } - if !v.levels[0].overlaps(v.s.icmp, umin, umax, true) { - var overlaps tFiles - for ; level < maxLevel; level++ { - if pLevel := level + 1; pLevel >= len(v.levels) { - return maxLevel - } else if v.levels[pLevel].overlaps(v.s.icmp, umin, umax, false) { - break - } - if gpLevel := level + 2; gpLevel < len(v.levels) { - overlaps = v.levels[gpLevel].getOverlaps(overlaps, v.s.icmp, umin, umax, false) - if overlaps.size() > int64(v.s.o.GetCompactionGPOverlaps(level)) { - break - } - } - } - } - } - return -} - -func (v *version) computeCompaction() { - // Precomputed best level for next compaction - bestLevel := int(-1) - bestScore := float64(-1) - - statFiles := make([]int, len(v.levels)) - statSizes := make([]string, len(v.levels)) - statScore := make([]string, len(v.levels)) - statTotSize := int64(0) - - for level, tables := range v.levels { - var score float64 - size := tables.size() - if level == 0 { - // We treat level-0 specially by bounding the number of files - // instead of number of bytes for two reasons: - // - // (1) With larger write-buffer sizes, it is nice not to do too - // many level-0 compaction. - // - // (2) The files in level-0 are merged on every read and - // therefore we wish to avoid too many files when the individual - // file size is small (perhaps because of a small write-buffer - // setting, or very high compression ratios, or lots of - // overwrites/deletions). - score = float64(len(tables)) / float64(v.s.o.GetCompactionL0Trigger()) - } else { - score = float64(size) / float64(v.s.o.GetCompactionTotalSize(level)) - } - - if score > bestScore { - bestLevel = level - bestScore = score - } - - statFiles[level] = len(tables) - statSizes[level] = shortenb(int(size)) - statScore[level] = fmt.Sprintf("%.2f", score) - statTotSize += size - } - - v.cLevel = bestLevel - v.cScore = bestScore - - v.s.logf("version@stat F·%v S·%s%v Sc·%v", statFiles, shortenb(int(statTotSize)), statSizes, statScore) -} - -func (v *version) needCompaction() bool { - return v.cScore >= 1 || atomic.LoadPointer(&v.cSeek) != nil -} - -type tablesScratch struct { - added map[int64]atRecord - deleted map[int64]struct{} -} - -type versionStaging struct { - base *version - levels []tablesScratch -} - -func (p *versionStaging) getScratch(level int) *tablesScratch { - if level >= len(p.levels) { - newLevels := make([]tablesScratch, level+1) - copy(newLevels, p.levels) - p.levels = newLevels - } - return &(p.levels[level]) -} - -func (p *versionStaging) commit(r *sessionRecord) { - // Deleted tables. - for _, r := range r.deletedTables { - scratch := p.getScratch(r.level) - if r.level < len(p.base.levels) && len(p.base.levels[r.level]) > 0 { - if scratch.deleted == nil { - scratch.deleted = make(map[int64]struct{}) - } - scratch.deleted[r.num] = struct{}{} - } - if scratch.added != nil { - delete(scratch.added, r.num) - } - } - - // New tables. - for _, r := range r.addedTables { - scratch := p.getScratch(r.level) - if scratch.added == nil { - scratch.added = make(map[int64]atRecord) - } - scratch.added[r.num] = r - if scratch.deleted != nil { - delete(scratch.deleted, r.num) - } - } -} - -func (p *versionStaging) finish() *version { - // Build new version. - nv := newVersion(p.base.s) - numLevel := len(p.levels) - if len(p.base.levels) > numLevel { - numLevel = len(p.base.levels) - } - nv.levels = make([]tFiles, numLevel) - for level := 0; level < numLevel; level++ { - var baseTabels tFiles - if level < len(p.base.levels) { - baseTabels = p.base.levels[level] - } - - if level < len(p.levels) { - scratch := p.levels[level] - - var nt tFiles - // Prealloc list if possible. - if n := len(baseTabels) + len(scratch.added) - len(scratch.deleted); n > 0 { - nt = make(tFiles, 0, n) - } - - // Base tables. - for _, t := range baseTabels { - if _, ok := scratch.deleted[t.fd.Num]; ok { - continue - } - if _, ok := scratch.added[t.fd.Num]; ok { - continue - } - nt = append(nt, t) - } - - // New tables. - for _, r := range scratch.added { - nt = append(nt, tableFileFromRecord(r)) - } - - if len(nt) != 0 { - // Sort tables. - if level == 0 { - nt.sortByNum() - } else { - nt.sortByKey(p.base.s.icmp) - } - - nv.levels[level] = nt - } - } else { - nv.levels[level] = baseTabels - } - } - - // Trim levels. - n := len(nv.levels) - for ; n > 0 && nv.levels[n-1] == nil; n-- { - } - nv.levels = nv.levels[:n] - - // Compute compaction score for new version. - nv.computeCompaction() - - return nv -} - -type versionReleaser struct { - v *version - once bool -} - -func (vr *versionReleaser) Release() { - v := vr.v - v.s.vmu.Lock() - if !vr.once { - v.releaseNB() - vr.once = true - } - v.s.vmu.Unlock() -} diff --git a/vendor/github.com/syndtr/goleveldb/leveldb/version_test.go b/vendor/github.com/syndtr/goleveldb/leveldb/version_test.go deleted file mode 100644 index a643be1..0000000 --- a/vendor/github.com/syndtr/goleveldb/leveldb/version_test.go +++ /dev/null @@ -1,181 +0,0 @@ -package leveldb - -import ( - "encoding/binary" - "reflect" - "testing" - - "github.com/onsi/gomega" - - "github.com/syndtr/goleveldb/leveldb/testutil" -) - -type testFileRec struct { - level int - num int64 -} - -func TestVersionStaging(t *testing.T) { - gomega.RegisterTestingT(t) - stor := testutil.NewStorage() - defer stor.Close() - s, err := newSession(stor, nil) - if err != nil { - t.Fatal(err) - } - - v := newVersion(s) - v.newStaging() - - tmp := make([]byte, 4) - mik := func(i uint64) []byte { - binary.BigEndian.PutUint32(tmp, uint32(i)) - return []byte(makeInternalKey(nil, tmp, 0, keyTypeVal)) - } - - for i, x := range []struct { - add, del []testFileRec - levels [][]int64 - }{ - { - add: []testFileRec{ - {1, 1}, - }, - levels: [][]int64{ - {}, - {1}, - }, - }, - { - add: []testFileRec{ - {1, 1}, - }, - levels: [][]int64{ - {}, - {1}, - }, - }, - { - del: []testFileRec{ - {1, 1}, - }, - levels: [][]int64{}, - }, - { - add: []testFileRec{ - {0, 1}, - {0, 3}, - {0, 2}, - {2, 5}, - {1, 4}, - }, - levels: [][]int64{ - {3, 2, 1}, - {4}, - {5}, - }, - }, - { - add: []testFileRec{ - {1, 6}, - {2, 5}, - }, - del: []testFileRec{ - {0, 1}, - {0, 4}, - }, - levels: [][]int64{ - {3, 2}, - {4, 6}, - {5}, - }, - }, - { - del: []testFileRec{ - {0, 3}, - {0, 2}, - {1, 4}, - {1, 6}, - {2, 5}, - }, - levels: [][]int64{}, - }, - { - add: []testFileRec{ - {0, 1}, - }, - levels: [][]int64{ - {1}, - }, - }, - { - add: []testFileRec{ - {1, 2}, - }, - levels: [][]int64{ - {1}, - {2}, - }, - }, - { - add: []testFileRec{ - {0, 3}, - }, - levels: [][]int64{ - {3, 1}, - {2}, - }, - }, - { - add: []testFileRec{ - {6, 9}, - }, - levels: [][]int64{ - {3, 1}, - {2}, - {}, - {}, - {}, - {}, - {9}, - }, - }, - { - del: []testFileRec{ - {6, 9}, - }, - levels: [][]int64{ - {3, 1}, - {2}, - }, - }, - } { - rec := &sessionRecord{} - for _, f := range x.add { - ik := mik(uint64(f.num)) - rec.addTable(f.level, f.num, 1, ik, ik) - } - for _, f := range x.del { - rec.delTable(f.level, f.num) - } - vs := v.newStaging() - vs.commit(rec) - v = vs.finish() - if len(v.levels) != len(x.levels) { - t.Fatalf("#%d: invalid level count: want=%d got=%d", i, len(x.levels), len(v.levels)) - } - for j, want := range x.levels { - tables := v.levels[j] - if len(want) != len(tables) { - t.Fatalf("#%d.%d: invalid tables count: want=%d got=%d", i, j, len(want), len(tables)) - } - got := make([]int64, len(tables)) - for k, t := range tables { - got[k] = t.fd.Num - } - if !reflect.DeepEqual(want, got) { - t.Fatalf("#%d.%d: invalid tables: want=%v got=%v", i, j, want, got) - } - } - } -} diff --git a/vendor/github.com/syndtr/goleveldb/manualtest/dbstress/key.go b/vendor/github.com/syndtr/goleveldb/manualtest/dbstress/key.go deleted file mode 100644 index c9f6963..0000000 --- a/vendor/github.com/syndtr/goleveldb/manualtest/dbstress/key.go +++ /dev/null @@ -1,137 +0,0 @@ -package main - -import ( - "encoding/binary" - "fmt" - - "github.com/syndtr/goleveldb/leveldb/errors" - "github.com/syndtr/goleveldb/leveldb/storage" -) - -type ErrIkeyCorrupted struct { - Ikey []byte - Reason string -} - -func (e *ErrIkeyCorrupted) Error() string { - return fmt.Sprintf("leveldb: iKey %q corrupted: %s", e.Ikey, e.Reason) -} - -func newErrIkeyCorrupted(ikey []byte, reason string) error { - return errors.NewErrCorrupted(storage.FileDesc{}, &ErrIkeyCorrupted{append([]byte{}, ikey...), reason}) -} - -type kType int - -func (kt kType) String() string { - switch kt { - case ktDel: - return "d" - case ktVal: - return "v" - } - return "x" -} - -// Value types encoded as the last component of internal keys. -// Don't modify; this value are saved to disk. -const ( - ktDel kType = iota - ktVal -) - -// ktSeek defines the kType that should be passed when constructing an -// internal key for seeking to a particular sequence number (since we -// sort sequence numbers in decreasing order and the value type is -// embedded as the low 8 bits in the sequence number in internal keys, -// we need to use the highest-numbered ValueType, not the lowest). -const ktSeek = ktVal - -const ( - // Maximum value possible for sequence number; the 8-bits are - // used by value type, so its can packed together in single - // 64-bit integer. - kMaxSeq uint64 = (uint64(1) << 56) - 1 - // Maximum value possible for packed sequence number and type. - kMaxNum uint64 = (kMaxSeq << 8) | uint64(ktSeek) -) - -// Maximum number encoded in bytes. -var kMaxNumBytes = make([]byte, 8) - -func init() { - binary.LittleEndian.PutUint64(kMaxNumBytes, kMaxNum) -} - -type iKey []byte - -func newIkey(ukey []byte, seq uint64, kt kType) iKey { - if seq > kMaxSeq { - panic("leveldb: invalid sequence number") - } else if kt > ktVal { - panic("leveldb: invalid type") - } - - ik := make(iKey, len(ukey)+8) - copy(ik, ukey) - binary.LittleEndian.PutUint64(ik[len(ukey):], (seq<<8)|uint64(kt)) - return ik -} - -func parseIkey(ik []byte) (ukey []byte, seq uint64, kt kType, err error) { - if len(ik) < 8 { - return nil, 0, 0, newErrIkeyCorrupted(ik, "invalid length") - } - num := binary.LittleEndian.Uint64(ik[len(ik)-8:]) - seq, kt = uint64(num>>8), kType(num&0xff) - if kt > ktVal { - return nil, 0, 0, newErrIkeyCorrupted(ik, "invalid type") - } - ukey = ik[:len(ik)-8] - return -} - -func validIkey(ik []byte) bool { - _, _, _, err := parseIkey(ik) - return err == nil -} - -func (ik iKey) assert() { - if ik == nil { - panic("leveldb: nil iKey") - } - if len(ik) < 8 { - panic(fmt.Sprintf("leveldb: iKey %q, len=%d: invalid length", ik, len(ik))) - } -} - -func (ik iKey) ukey() []byte { - ik.assert() - return ik[:len(ik)-8] -} - -func (ik iKey) num() uint64 { - ik.assert() - return binary.LittleEndian.Uint64(ik[len(ik)-8:]) -} - -func (ik iKey) parseNum() (seq uint64, kt kType) { - num := ik.num() - seq, kt = uint64(num>>8), kType(num&0xff) - if kt > ktVal { - panic(fmt.Sprintf("leveldb: iKey %q, len=%d: invalid type %#x", ik, len(ik), kt)) - } - return -} - -func (ik iKey) String() string { - if ik == nil { - return "" - } - - if ukey, seq, kt, err := parseIkey(ik); err == nil { - return fmt.Sprintf("%x,%s%d", ukey, kt, seq) - } else { - return "" - } -} diff --git a/vendor/github.com/syndtr/goleveldb/manualtest/dbstress/main.go b/vendor/github.com/syndtr/goleveldb/manualtest/dbstress/main.go deleted file mode 100644 index 2886475..0000000 --- a/vendor/github.com/syndtr/goleveldb/manualtest/dbstress/main.go +++ /dev/null @@ -1,628 +0,0 @@ -package main - -import ( - "crypto/rand" - "encoding/binary" - "flag" - "fmt" - "log" - mrand "math/rand" - "net/http" - _ "net/http/pprof" - "os" - "os/signal" - "path" - "runtime" - "strconv" - "strings" - "sync" - "sync/atomic" - "time" - - "github.com/syndtr/goleveldb/leveldb" - "github.com/syndtr/goleveldb/leveldb/errors" - "github.com/syndtr/goleveldb/leveldb/opt" - "github.com/syndtr/goleveldb/leveldb/storage" - "github.com/syndtr/goleveldb/leveldb/table" - "github.com/syndtr/goleveldb/leveldb/util" -) - -var ( - dbPath = path.Join(os.TempDir(), "goleveldb-testdb") - openFilesCacheCapacity = 500 - keyLen = 63 - valueLen = 256 - numKeys = arrayInt{100000, 1332, 531, 1234, 9553, 1024, 35743} - httpProf = "127.0.0.1:5454" - transactionProb = 0.5 - enableBlockCache = false - enableCompression = false - enableBufferPool = false - - wg = new(sync.WaitGroup) - done, fail uint32 - - bpool *util.BufferPool -) - -type arrayInt []int - -func (a arrayInt) String() string { - var str string - for i, n := range a { - if i > 0 { - str += "," - } - str += strconv.Itoa(n) - } - return str -} - -func (a *arrayInt) Set(str string) error { - var na arrayInt - for _, s := range strings.Split(str, ",") { - s = strings.TrimSpace(s) - if s != "" { - n, err := strconv.Atoi(s) - if err != nil { - return err - } - na = append(na, n) - } - } - *a = na - return nil -} - -func init() { - flag.StringVar(&dbPath, "db", dbPath, "testdb path") - flag.IntVar(&openFilesCacheCapacity, "openfilescachecap", openFilesCacheCapacity, "open files cache capacity") - flag.IntVar(&keyLen, "keylen", keyLen, "key length") - flag.IntVar(&valueLen, "valuelen", valueLen, "value length") - flag.Var(&numKeys, "numkeys", "num keys") - flag.StringVar(&httpProf, "httpprof", httpProf, "http pprof listen addr") - flag.Float64Var(&transactionProb, "transactionprob", transactionProb, "probablity of writes using transaction") - flag.BoolVar(&enableBufferPool, "enablebufferpool", enableBufferPool, "enable buffer pool") - flag.BoolVar(&enableBlockCache, "enableblockcache", enableBlockCache, "enable block cache") - flag.BoolVar(&enableCompression, "enablecompression", enableCompression, "enable block compression") -} - -func randomData(dst []byte, ns, prefix byte, i uint32, dataLen int) []byte { - if dataLen < (2+4+4)*2+4 { - panic("dataLen is too small") - } - if cap(dst) < dataLen { - dst = make([]byte, dataLen) - } else { - dst = dst[:dataLen] - } - half := (dataLen - 4) / 2 - if _, err := rand.Reader.Read(dst[2 : half-8]); err != nil { - panic(err) - } - dst[0] = ns - dst[1] = prefix - binary.LittleEndian.PutUint32(dst[half-8:], i) - binary.LittleEndian.PutUint32(dst[half-8:], i) - binary.LittleEndian.PutUint32(dst[half-4:], util.NewCRC(dst[:half-4]).Value()) - full := half * 2 - copy(dst[half:full], dst[:half]) - if full < dataLen-4 { - if _, err := rand.Reader.Read(dst[full : dataLen-4]); err != nil { - panic(err) - } - } - binary.LittleEndian.PutUint32(dst[dataLen-4:], util.NewCRC(dst[:dataLen-4]).Value()) - return dst -} - -func dataSplit(data []byte) (data0, data1 []byte) { - n := (len(data) - 4) / 2 - return data[:n], data[n : n+n] -} - -func dataNS(data []byte) byte { - return data[0] -} - -func dataPrefix(data []byte) byte { - return data[1] -} - -func dataI(data []byte) uint32 { - return binary.LittleEndian.Uint32(data[(len(data)-4)/2-8:]) -} - -func dataChecksum(data []byte) (uint32, uint32) { - checksum0 := binary.LittleEndian.Uint32(data[len(data)-4:]) - checksum1 := util.NewCRC(data[:len(data)-4]).Value() - return checksum0, checksum1 -} - -func dataPrefixSlice(ns, prefix byte) *util.Range { - return util.BytesPrefix([]byte{ns, prefix}) -} - -func dataNsSlice(ns byte) *util.Range { - return util.BytesPrefix([]byte{ns}) -} - -type testingStorage struct { - storage.Storage -} - -func (ts *testingStorage) scanTable(fd storage.FileDesc, checksum bool) (corrupted bool) { - r, err := ts.Open(fd) - if err != nil { - log.Fatal(err) - } - defer r.Close() - - size, err := r.Seek(0, os.SEEK_END) - if err != nil { - log.Fatal(err) - } - - o := &opt.Options{ - DisableLargeBatchTransaction: true, - Strict: opt.NoStrict, - } - if checksum { - o.Strict = opt.StrictBlockChecksum | opt.StrictReader - } - tr, err := table.NewReader(r, size, fd, nil, bpool, o) - if err != nil { - log.Fatal(err) - } - defer tr.Release() - - checkData := func(i int, t string, data []byte) bool { - if len(data) == 0 { - panic(fmt.Sprintf("[%v] nil data: i=%d t=%s", fd, i, t)) - } - - checksum0, checksum1 := dataChecksum(data) - if checksum0 != checksum1 { - atomic.StoreUint32(&fail, 1) - atomic.StoreUint32(&done, 1) - corrupted = true - - data0, data1 := dataSplit(data) - data0c0, data0c1 := dataChecksum(data0) - data1c0, data1c1 := dataChecksum(data1) - log.Printf("FATAL: [%v] Corrupted data i=%d t=%s (%#x != %#x): %x(%v) vs %x(%v)", - fd, i, t, checksum0, checksum1, data0, data0c0 == data0c1, data1, data1c0 == data1c1) - return true - } - return false - } - - iter := tr.NewIterator(nil, nil) - defer iter.Release() - for i := 0; iter.Next(); i++ { - ukey, _, kt, kerr := parseIkey(iter.Key()) - if kerr != nil { - atomic.StoreUint32(&fail, 1) - atomic.StoreUint32(&done, 1) - corrupted = true - - log.Printf("FATAL: [%v] Corrupted ikey i=%d: %v", fd, i, kerr) - return - } - if checkData(i, "key", ukey) { - return - } - if kt == ktVal && checkData(i, "value", iter.Value()) { - return - } - } - if err := iter.Error(); err != nil { - if errors.IsCorrupted(err) { - atomic.StoreUint32(&fail, 1) - atomic.StoreUint32(&done, 1) - corrupted = true - - log.Printf("FATAL: [%v] Corruption detected: %v", fd, err) - } else { - log.Fatal(err) - } - } - - return -} - -func (ts *testingStorage) Remove(fd storage.FileDesc) error { - if atomic.LoadUint32(&fail) == 1 { - return nil - } - - if fd.Type == storage.TypeTable { - if ts.scanTable(fd, true) { - return nil - } - } - return ts.Storage.Remove(fd) -} - -type latencyStats struct { - mark time.Time - dur, min, max time.Duration - num int -} - -func (s *latencyStats) start() { - s.mark = time.Now() -} - -func (s *latencyStats) record(n int) { - if s.mark.IsZero() { - panic("not started") - } - dur := time.Now().Sub(s.mark) - dur1 := dur / time.Duration(n) - if dur1 < s.min || s.min == 0 { - s.min = dur1 - } - if dur1 > s.max { - s.max = dur1 - } - s.dur += dur - s.num += n - s.mark = time.Time{} -} - -func (s *latencyStats) ratePerSec() int { - durSec := s.dur / time.Second - if durSec > 0 { - return s.num / int(durSec) - } - return s.num -} - -func (s *latencyStats) avg() time.Duration { - if s.num > 0 { - return s.dur / time.Duration(s.num) - } - return 0 -} - -func (s *latencyStats) add(x *latencyStats) { - if x.min < s.min || s.min == 0 { - s.min = x.min - } - if x.max > s.max { - s.max = x.max - } - s.dur += x.dur - s.num += x.num -} - -func main() { - flag.Parse() - - if enableBufferPool { - bpool = util.NewBufferPool(opt.DefaultBlockSize + 128) - } - - log.Printf("Test DB stored at %q", dbPath) - if httpProf != "" { - log.Printf("HTTP pprof listening at %q", httpProf) - runtime.SetBlockProfileRate(1) - go func() { - if err := http.ListenAndServe(httpProf, nil); err != nil { - log.Fatalf("HTTPPROF: %v", err) - } - }() - } - - runtime.GOMAXPROCS(runtime.NumCPU()) - - os.RemoveAll(dbPath) - stor, err := storage.OpenFile(dbPath, false) - if err != nil { - log.Fatal(err) - } - tstor := &testingStorage{stor} - defer tstor.Close() - - fatalf := func(err error, format string, v ...interface{}) { - atomic.StoreUint32(&fail, 1) - atomic.StoreUint32(&done, 1) - log.Printf("FATAL: "+format, v...) - if err != nil && errors.IsCorrupted(err) { - cerr := err.(*errors.ErrCorrupted) - if !cerr.Fd.Zero() && cerr.Fd.Type == storage.TypeTable { - log.Print("FATAL: corruption detected, scanning...") - if !tstor.scanTable(storage.FileDesc{Type: storage.TypeTable, Num: cerr.Fd.Num}, false) { - log.Printf("FATAL: unable to find corrupted key/value pair in table %v", cerr.Fd) - } - } - } - runtime.Goexit() - } - - if openFilesCacheCapacity == 0 { - openFilesCacheCapacity = -1 - } - o := &opt.Options{ - OpenFilesCacheCapacity: openFilesCacheCapacity, - DisableBufferPool: !enableBufferPool, - DisableBlockCache: !enableBlockCache, - ErrorIfExist: true, - Compression: opt.NoCompression, - } - if enableCompression { - o.Compression = opt.DefaultCompression - } - - db, err := leveldb.Open(tstor, o) - if err != nil { - log.Fatal(err) - } - defer db.Close() - - var ( - mu = &sync.Mutex{} - gGetStat = &latencyStats{} - gIterStat = &latencyStats{} - gWriteStat = &latencyStats{} - gTrasactionStat = &latencyStats{} - startTime = time.Now() - - writeReq = make(chan *leveldb.Batch) - writeAck = make(chan error) - writeAckAck = make(chan struct{}) - ) - - go func() { - for b := range writeReq { - - var err error - if mrand.Float64() < transactionProb { - log.Print("> Write using transaction") - gTrasactionStat.start() - var tr *leveldb.Transaction - if tr, err = db.OpenTransaction(); err == nil { - if err = tr.Write(b, nil); err == nil { - if err = tr.Commit(); err == nil { - gTrasactionStat.record(b.Len()) - } - } else { - tr.Discard() - } - } - } else { - gWriteStat.start() - if err = db.Write(b, nil); err == nil { - gWriteStat.record(b.Len()) - } - } - writeAck <- err - <-writeAckAck - } - }() - - go func() { - for { - time.Sleep(3 * time.Second) - - log.Print("------------------------") - - log.Printf("> Elapsed=%v", time.Now().Sub(startTime)) - mu.Lock() - log.Printf("> GetLatencyMin=%v GetLatencyMax=%v GetLatencyAvg=%v GetRatePerSec=%d", - gGetStat.min, gGetStat.max, gGetStat.avg(), gGetStat.ratePerSec()) - log.Printf("> IterLatencyMin=%v IterLatencyMax=%v IterLatencyAvg=%v IterRatePerSec=%d", - gIterStat.min, gIterStat.max, gIterStat.avg(), gIterStat.ratePerSec()) - log.Printf("> WriteLatencyMin=%v WriteLatencyMax=%v WriteLatencyAvg=%v WriteRatePerSec=%d", - gWriteStat.min, gWriteStat.max, gWriteStat.avg(), gWriteStat.ratePerSec()) - log.Printf("> TransactionLatencyMin=%v TransactionLatencyMax=%v TransactionLatencyAvg=%v TransactionRatePerSec=%d", - gTrasactionStat.min, gTrasactionStat.max, gTrasactionStat.avg(), gTrasactionStat.ratePerSec()) - mu.Unlock() - - cachedblock, _ := db.GetProperty("leveldb.cachedblock") - openedtables, _ := db.GetProperty("leveldb.openedtables") - alivesnaps, _ := db.GetProperty("leveldb.alivesnaps") - aliveiters, _ := db.GetProperty("leveldb.aliveiters") - blockpool, _ := db.GetProperty("leveldb.blockpool") - log.Printf("> BlockCache=%s OpenedTables=%s AliveSnaps=%s AliveIter=%s BlockPool=%q", - cachedblock, openedtables, alivesnaps, aliveiters, blockpool) - - log.Print("------------------------") - } - }() - - for ns, numKey := range numKeys { - func(ns, numKey int) { - log.Printf("[%02d] STARTING: numKey=%d", ns, numKey) - - keys := make([][]byte, numKey) - for i := range keys { - keys[i] = randomData(nil, byte(ns), 1, uint32(i), keyLen) - } - - wg.Add(1) - go func() { - var wi uint32 - defer func() { - log.Printf("[%02d] WRITER DONE #%d", ns, wi) - wg.Done() - }() - - var ( - b = new(leveldb.Batch) - k2, v2 []byte - nReader int32 - ) - for atomic.LoadUint32(&done) == 0 { - log.Printf("[%02d] WRITER #%d", ns, wi) - - b.Reset() - for _, k1 := range keys { - k2 = randomData(k2, byte(ns), 2, wi, keyLen) - v2 = randomData(v2, byte(ns), 3, wi, valueLen) - b.Put(k2, v2) - b.Put(k1, k2) - } - writeReq <- b - if err := <-writeAck; err != nil { - writeAckAck <- struct{}{} - fatalf(err, "[%02d] WRITER #%d db.Write: %v", ns, wi, err) - } - - snap, err := db.GetSnapshot() - if err != nil { - writeAckAck <- struct{}{} - fatalf(err, "[%02d] WRITER #%d db.GetSnapshot: %v", ns, wi, err) - } - - writeAckAck <- struct{}{} - - wg.Add(1) - atomic.AddInt32(&nReader, 1) - go func(snapwi uint32, snap *leveldb.Snapshot) { - var ( - ri int - iterStat = &latencyStats{} - getStat = &latencyStats{} - ) - defer func() { - mu.Lock() - gGetStat.add(getStat) - gIterStat.add(iterStat) - mu.Unlock() - - atomic.AddInt32(&nReader, -1) - log.Printf("[%02d] READER #%d.%d DONE Snap=%v Alive=%d IterLatency=%v GetLatency=%v", ns, snapwi, ri, snap, atomic.LoadInt32(&nReader), iterStat.avg(), getStat.avg()) - snap.Release() - wg.Done() - }() - - stopi := snapwi + 3 - for (ri < 3 || atomic.LoadUint32(&wi) < stopi) && atomic.LoadUint32(&done) == 0 { - var n int - iter := snap.NewIterator(dataPrefixSlice(byte(ns), 1), nil) - iterStat.start() - for iter.Next() { - k1 := iter.Key() - k2 := iter.Value() - iterStat.record(1) - - if dataNS(k2) != byte(ns) { - fatalf(nil, "[%02d] READER #%d.%d K%d invalid in-key NS: want=%d got=%d", ns, snapwi, ri, n, ns, dataNS(k2)) - } - - kwritei := dataI(k2) - if kwritei != snapwi { - fatalf(nil, "[%02d] READER #%d.%d K%d invalid in-key iter num: %d", ns, snapwi, ri, n, kwritei) - } - - getStat.start() - v2, err := snap.Get(k2, nil) - if err != nil { - fatalf(err, "[%02d] READER #%d.%d K%d snap.Get: %v\nk1: %x\n -> k2: %x", ns, snapwi, ri, n, err, k1, k2) - } - getStat.record(1) - - if checksum0, checksum1 := dataChecksum(v2); checksum0 != checksum1 { - err := &errors.ErrCorrupted{Fd: storage.FileDesc{0xff, 0}, Err: fmt.Errorf("v2: %x: checksum mismatch: %v vs %v", v2, checksum0, checksum1)} - fatalf(err, "[%02d] READER #%d.%d K%d snap.Get: %v\nk1: %x\n -> k2: %x", ns, snapwi, ri, n, err, k1, k2) - } - - n++ - iterStat.start() - } - iter.Release() - if err := iter.Error(); err != nil { - fatalf(err, "[%02d] READER #%d.%d K%d iter.Error: %v", ns, snapwi, ri, numKey, err) - } - if n != numKey { - fatalf(nil, "[%02d] READER #%d.%d missing keys: want=%d got=%d", ns, snapwi, ri, numKey, n) - } - - ri++ - } - }(wi, snap) - - atomic.AddUint32(&wi, 1) - } - }() - - delB := new(leveldb.Batch) - wg.Add(1) - go func() { - var ( - i int - iterStat = &latencyStats{} - ) - defer func() { - log.Printf("[%02d] SCANNER DONE #%d", ns, i) - wg.Done() - }() - - time.Sleep(2 * time.Second) - - for atomic.LoadUint32(&done) == 0 { - var n int - delB.Reset() - iter := db.NewIterator(dataNsSlice(byte(ns)), nil) - iterStat.start() - for iter.Next() && atomic.LoadUint32(&done) == 0 { - k := iter.Key() - v := iter.Value() - iterStat.record(1) - - for ci, x := range [...][]byte{k, v} { - checksum0, checksum1 := dataChecksum(x) - if checksum0 != checksum1 { - if ci == 0 { - fatalf(nil, "[%02d] SCANNER %d.%d invalid key checksum: want %d, got %d\n%x -> %x", ns, i, n, checksum0, checksum1, k, v) - } else { - fatalf(nil, "[%02d] SCANNER %d.%d invalid value checksum: want %d, got %d\n%x -> %x", ns, i, n, checksum0, checksum1, k, v) - } - } - } - - if dataPrefix(k) == 2 || mrand.Int()%999 == 0 { - delB.Delete(k) - } - - n++ - iterStat.start() - } - iter.Release() - if err := iter.Error(); err != nil { - fatalf(err, "[%02d] SCANNER #%d.%d iter.Error: %v", ns, i, n, err) - } - - if n > 0 { - log.Printf("[%02d] SCANNER #%d IterLatency=%v", ns, i, iterStat.avg()) - } - - if delB.Len() > 0 && atomic.LoadUint32(&done) == 0 { - t := time.Now() - writeReq <- delB - if err := <-writeAck; err != nil { - writeAckAck <- struct{}{} - fatalf(err, "[%02d] SCANNER #%d db.Write: %v", ns, i, err) - } else { - writeAckAck <- struct{}{} - } - log.Printf("[%02d] SCANNER #%d Deleted=%d Time=%v", ns, i, delB.Len(), time.Now().Sub(t)) - } - - i++ - } - }() - }(ns, numKey) - } - - go func() { - sig := make(chan os.Signal) - signal.Notify(sig, os.Interrupt, os.Kill) - log.Printf("Got signal: %v, exiting...", <-sig) - atomic.StoreUint32(&done, 1) - }() - - wg.Wait() -} diff --git a/vendor/github.com/syndtr/goleveldb/manualtest/filelock/main.go b/vendor/github.com/syndtr/goleveldb/manualtest/filelock/main.go deleted file mode 100644 index 192951f..0000000 --- a/vendor/github.com/syndtr/goleveldb/manualtest/filelock/main.go +++ /dev/null @@ -1,85 +0,0 @@ -package main - -import ( - "bufio" - "bytes" - "flag" - "fmt" - "os" - "os/exec" - "path/filepath" - - "github.com/syndtr/goleveldb/leveldb/storage" -) - -var ( - filename string - child bool -) - -func init() { - flag.StringVar(&filename, "filename", filepath.Join(os.TempDir(), "goleveldb_filelock_test"), "Filename used for testing") - flag.BoolVar(&child, "child", false, "This is the child") -} - -func runChild() error { - var args []string - args = append(args, os.Args[1:]...) - args = append(args, "-child") - cmd := exec.Command(os.Args[0], args...) - var out bytes.Buffer - cmd.Stdout = &out - err := cmd.Run() - r := bufio.NewReader(&out) - for { - line, _, e1 := r.ReadLine() - if e1 != nil { - break - } - fmt.Println("[Child]", string(line)) - } - return err -} - -func main() { - flag.Parse() - - fmt.Printf("Using path: %s\n", filename) - if child { - fmt.Println("Child flag set.") - } - - stor, err := storage.OpenFile(filename, false) - if err != nil { - fmt.Printf("Could not open storage: %s", err) - os.Exit(10) - } - - if !child { - fmt.Println("Executing child -- first test (expecting error)") - err := runChild() - if err == nil { - fmt.Println("Expecting error from child") - } else if err.Error() != "exit status 10" { - fmt.Println("Got unexpected error from child:", err) - } else { - fmt.Printf("Got error from child: %s (expected)\n", err) - } - } - - err = stor.Close() - if err != nil { - fmt.Printf("Error when closing storage: %s", err) - os.Exit(11) - } - - if !child { - fmt.Println("Executing child -- second test") - err := runChild() - if err != nil { - fmt.Println("Got unexpected error from child:", err) - } - } - - os.RemoveAll(filename) -}