diff --git a/deps/rocksdb-rocksdb-3.4.tar.gz b/deps/rocksdb-rocksdb-3.4.tar.gz new file mode 100755 index 00000000..240554ff Binary files /dev/null and b/deps/rocksdb-rocksdb-3.4.tar.gz differ diff --git a/deps/rocksdb-rocksdb-3.5.tar.gz b/deps/rocksdb-rocksdb-3.5.tar.gz deleted file mode 100755 index 12ab5286..00000000 Binary files a/deps/rocksdb-rocksdb-3.5.tar.gz and /dev/null differ diff --git a/patch/rocksdb-3.4.patch b/patch/rocksdb-3.4.patch new file mode 100755 index 00000000..3e9e5e3f --- /dev/null +++ b/patch/rocksdb-3.4.patch @@ -0,0 +1,11 @@ +--- util/rate_limiter.cc 2014-09-18 19:43:00.509681500 +0800 ++++ util/rate_limiter.cc 2014-09-18 19:36:31.071741600 +0800 +@@ -60,7 +60,7 @@ + } + + void GenericRateLimiter::Request(int64_t bytes, const Env::IOPriority pri) { +- assert(bytes < refill_bytes_per_period_); ++ assert(bytes <= refill_bytes_per_period_); + + MutexLock g(&request_mutex_); + if (stop_) { diff --git a/patch/zookeeper.configure.ac.patch b/patch/zookeeper.configure.ac.patch new file mode 100755 index 00000000..2c7df946 --- /dev/null +++ b/patch/zookeeper.configure.ac.patch @@ -0,0 +1,18 @@ +--- configure.ac 2012-09-30 10:53:32.000000000 -0700 ++++ configure.ac 2014-07-08 18:02:20.285143803 -0700 +@@ -41,9 +41,12 @@ + CPPUNIT_CFLAGS="$CPPUNIT_CFLAGS -DZKSERVER_CMD=\"\\\"${base_dir}/src/c/tests/zkServer.sh\\\"\"" + else + CPPUNIT_CFLAGS="$CPPUNIT_CFLAGS -DZKSERVER_CMD=\"\\\"./tests/zkServer.sh\\\"\"" +-AC_CHECK_FILES([generated/zookeeper.jute.c generated/zookeeper.jute.h],[], +- [AC_MSG_ERROR([jute files are missing! Please run "ant compile_jute" while in the zookeeper top level directory.]) +-]) ++if test -f "generated/zookeeper.jute.c"; then :; else ++ AC_MSG_ERROR([jute files are missing! Please run "ant compile_jute" while in the zookeeper top level directory.]) ++fi ++if test -f "generated/zookeeper.jute.h"; then :; else ++ AC_MSG_ERROR([jute files are missing! Please run "ant compile_jute" while in the zookeeper top level directory.]) ++fi + fi + AC_SUBST(CPPUNIT_CFLAGS) + diff --git a/patch/zookeeper.mt_adaptor.patch b/patch/zookeeper.mt_adaptor.patch new file mode 100755 index 00000000..0dbc8020 --- /dev/null +++ b/patch/zookeeper.mt_adaptor.patch @@ -0,0 +1,20 @@ +--- mt_adaptor.c 2012-09-30 10:53:32.000000000 -0700 ++++ mt_adaptor.c 2014-07-01 16:05:07.387967525 -0700 +@@ -482,6 +482,9 @@ + + int32_t fetch_and_add(volatile int32_t* operand, int incr) + { ++#if defined(__GNUC__) ++ return __sync_fetch_and_add(operand, incr); ++#else + #ifndef WIN32 + int32_t result; + asm __volatile__( +@@ -503,6 +506,7 @@ + } + return result; + #endif ++#endif + } + + // make sure the static xid is initialized before any threads started diff --git a/src/Makefile b/src/Makefile index 09ca6150..aac49b3d 100755 --- a/src/Makefile +++ b/src/Makefile @@ -34,7 +34,7 @@ CXX=g++ CC=gcc -ARDB_VERSION=0.8.0 +ARDB_VERSION=0.8.1 uname_S := $(shell sh -c 'uname -s 2>/dev/null || echo not') @@ -59,7 +59,7 @@ LEVELDB_PATH=${LIB_PATH}/${LEVELDB_VER} LEVELDB_FILE=${LEVELDB_PATH}.tar.gz LEVELDB_LIBA=${LEVELDB_PATH}/libleveldb.a -ROCKSDB_VER=rocksdb-rocksdb-3.5 +ROCKSDB_VER=rocksdb-rocksdb-3.4 ROCKSDB_PATH=${LIB_PATH}/${ROCKSDB_VER} ROCKSDB_FILE=${ROCKSDB_PATH}.tar.gz ROCKSDB_LIBA=${ROCKSDB_PATH}/librocksdb.a @@ -268,6 +268,7 @@ rocksdb: $(ROCKSDB_LIBA) $(ROCKSDB_LIBA): $(SNAPPY_LIBA) $(ROCKSDB_PATH) echo ">>>>> Building ROCKSDB" && \ cd ${ROCKSDB_PATH} && \ + patch -p0 < ../../patch/rocksdb-3.4.patch && \ CXXFLAGS="-I${SNAPPY_PATH}" CFLAGS="-I${SNAPPY_PATH}" LDFLAGS="${SNAPPY_PATH}/.libs" $(MAKE) librocksdb.a && \ echo "<<<<< Done building ROCKSDB" @@ -299,9 +300,9 @@ $(ZOOKEEPER_PATH): cd ${LIB_PATH} && \ tar zxf ${ZOOKEEPER_FILE} && \ cd ${ZOOKEEPER_PATH}/src && \ - patch < ../../../zookeeper.mt_adaptor.patch && \ + patch < ../patch/zookeeper.mt_adaptor.patch && \ cd ${ZOOKEEPER_PATH} && \ - patch < ../../zookeeper.configure.ac.patch && \ + patch < ../patch/zookeeper.configure.ac.patch && \ autoconf configure.ac > configure 2> autoconf.out && \ echo "<<<<<< Done unpacking ZOOKEEPER" diff --git a/src/ardb.hpp b/src/ardb.hpp index 26540a00..bc90f3e8 100755 --- a/src/ardb.hpp +++ b/src/ardb.hpp @@ -543,6 +543,8 @@ OP_NAMESPACE_BEGIN int PFCount(Context& ctx, RedisCommandFrame& cmd); int PFMerge(Context& ctx, RedisCommandFrame& cmd); + int Cluster(Context& ctx, RedisCommandFrame& cmd); + int DoCall(Context& ctx, RedisCommandHandlerSetting& setting, RedisCommandFrame& cmd); RedisCommandHandlerSetting* FindRedisCommandHandlerSetting(RedisCommandFrame& cmd); bool ParseConfig(const Properties& props); diff --git a/src/command/t_hash.cpp b/src/command/t_hash.cpp index 219424c3..ee302c64 100755 --- a/src/command/t_hash.cpp +++ b/src/command/t_hash.cpp @@ -75,7 +75,7 @@ OP_NAMESPACE_BEGIN it++; if (!meta.attach.force_zipsave && (field.StringLength() > m_cfg.hash_max_ziplist_value - || value.StringLength() > m_cfg.hash_max_ziplist_value)) + || value.StringLength() > m_cfg.hash_max_ziplist_value) && meta.meta.zipmap.size() > 1) { zipsave = false; } @@ -400,7 +400,8 @@ OP_NAMESPACE_BEGIN { break; } - if ((pattern.empty() || stringmatchlen(pattern.c_str(), pattern.size(), tmpelement.c_str(), tmpelement.size(), 0) == 1)) + if ((pattern.empty() + || stringmatchlen(pattern.c_str(), pattern.size(), tmpelement.c_str(), tmpelement.size(), 0) == 1)) { RedisReply& rr1 = r2.AddMember(); RedisReply& rr2 = r2.AddMember(); @@ -411,10 +412,11 @@ OP_NAMESPACE_BEGIN } if (iter.Valid()) { - if(m_cfg.scan_redis_compatible) + if (m_cfg.scan_redis_compatible) { fill_str_reply(r1, stringfromll(GetNewRedisCursor(tmpelement))); - }else + } + else { fill_str_reply(r1, tmpelement); } diff --git a/src/command/t_list.cpp b/src/command/t_list.cpp index 82f9dba4..772f914f 100755 --- a/src/command/t_list.cpp +++ b/src/command/t_list.cpp @@ -337,8 +337,9 @@ OP_NAMESPACE_BEGIN meta.meta.ziplist.push_back(element); } } - if (meta.meta.Length() >= m_cfg.list_max_ziplist_entries - || element.StringLength() >= m_cfg.list_max_ziplist_value) + if (meta.meta.Length() > 1 + && (meta.meta.Length() >= m_cfg.list_max_ziplist_entries + || element.StringLength() >= m_cfg.list_max_ziplist_value)) { //convert to non ziplist ZipListConvert(ctx, meta); diff --git a/src/command/t_set.cpp b/src/command/t_set.cpp index 648eca49..6b0dcc7a 100755 --- a/src/command/t_set.cpp +++ b/src/command/t_set.cpp @@ -65,7 +65,7 @@ OP_NAMESPACE_BEGIN count++; if (!meta.attach.force_zipsave && (meta.meta.zipset.size() > m_cfg.set_max_ziplist_entries - || element.StringLength() >= m_cfg.set_max_ziplist_value)) + || element.StringLength() >= m_cfg.set_max_ziplist_value) && meta.meta.zipset.size() > 1) { meta.meta.SetEncoding(COLLECTION_ECODING_RAW); DataSet::iterator it = meta.meta.zipset.begin(); @@ -993,12 +993,7 @@ OP_NAMESPACE_BEGIN SetIter(ctx, meta, meta.meta.min_index, iter, false); while (iter.Valid()) { - KeyObject fk; - fk.db = ctx.currentDB; - fk.key = meta.key.key; - fk.type = SET_ELEMENT; - fk.element = *(iter.Element()); - DelKeyValue((ctx), fk); + DelRaw(ctx, iter.CurrentRawKey()); iter.Next(); } } diff --git a/src/command/t_zset.cpp b/src/command/t_zset.cpp index 38134e60..dc5089f0 100755 --- a/src/command/t_zset.cpp +++ b/src/command/t_zset.cpp @@ -154,8 +154,9 @@ OP_NAMESPACE_BEGIN if (meta.meta.Encoding() == COLLECTION_ECODING_ZIPZSET) { meta.meta.zipmap[element] = score; - if (meta.meta.zipmap.size() > m_cfg.zset_max_ziplist_entries - || element.StringLength() >= m_cfg.zset_max_ziplist_value) + if (meta.meta.zipmap.size() > 1 + && (meta.meta.zipmap.size() > m_cfg.zset_max_ziplist_entries + || element.StringLength() >= m_cfg.zset_max_ziplist_value)) { meta.meta.SetEncoding(COLLECTION_ECODING_RAW); DataMap::iterator it = meta.meta.zipmap.begin(); diff --git a/src/common/channel/codec/redis_command.hpp b/src/common/channel/codec/redis_command.hpp index b14bbed4..816bf9d1 100755 --- a/src/common/channel/codec/redis_command.hpp +++ b/src/common/channel/codec/redis_command.hpp @@ -214,6 +214,8 @@ namespace ardb REDIS_CMD_HREPLACE = 175, REDIS_CMD_SREPLACE = 176, + + REDIS_CMD_CLUSTER = 177, //used in cluster mode }; class RedisCommandDecoder; diff --git a/src/config.hpp b/src/config.hpp index 18a64ac7..4e10066d 100755 --- a/src/config.hpp +++ b/src/config.hpp @@ -147,9 +147,9 @@ OP_NAMESPACE_BEGIN false), repl_ping_slave_period(10), repl_timeout(60), repl_backlog_size(100 * 1024 * 1024), repl_state_persist_period( 1), repl_backlog_time_limit(3600), slave_cleardb_before_fullresync(true), slave_readonly( true), slave_serve_stale_data(true), slave_priority(100), lua_time_limit(0), master_port(0), loglevel( - "INFO"), hash_max_ziplist_entries(128), hash_max_ziplist_value(64), list_max_ziplist_entries( - 128), list_max_ziplist_value(64), zset_max_ziplist_entries(128), zset_max_ziplist_value(64), set_max_ziplist_entries( - 128), set_max_ziplist_value(64), L1_zset_max_cache_size(0), L1_set_max_cache_size(0), L1_list_max_cache_size( + "INFO"), hash_max_ziplist_entries(128), hash_max_ziplist_value(256), list_max_ziplist_entries( + 128), list_max_ziplist_value(256), zset_max_ziplist_entries(128), zset_max_ziplist_value(256), set_max_ziplist_entries( + 128), set_max_ziplist_value(256), L1_zset_max_cache_size(0), L1_set_max_cache_size(0), L1_list_max_cache_size( 0), L1_hash_max_cache_size(0), L1_string_max_cache_size(0), L1_zset_read_fill_cache(false), L1_zset_seek_load_cache( false), L1_set_read_fill_cache(false), L1_set_seek_load_cache(false), L1_hash_read_fill_cache( false), L1_hash_seek_load_cache(false), L1_list_read_fill_cache(false), L1_list_seek_load_cache( diff --git a/src/engine/rocksdb_engine.cpp b/src/engine/rocksdb_engine.cpp index 973b3b78..07d4d541 100755 --- a/src/engine/rocksdb_engine.cpp +++ b/src/engine/rocksdb_engine.cpp @@ -170,28 +170,32 @@ namespace ardb m_cfg = cfg; m_options.create_if_missing = true; m_options.comparator = &m_comparator; - rocksdb::BlockBasedTableOptions block_options; + //rocksdb::BlockBasedTableOptions block_options; if (cfg.block_cache_size > 0) { - block_options.block_cache = rocksdb::NewLRUCache(cfg.block_cache_size); - //m_options.block_cache_compressed = rocksdb::NewLRUCache(cfg.block_cache_compressed_size); + //block_options.block_cache = rocksdb::NewLRUCache(cfg.block_cache_size); + m_options.block_cache_compressed = rocksdb::NewLRUCache(cfg.block_cache_compressed_size); }else if(cfg.block_cache_size < 0) { - block_options.no_block_cache = true; + //block_options.no_block_cache = true; + m_options.no_block_cache = true; } if (cfg.block_size > 0) { - block_options.block_size = cfg.block_size; + //block_options.block_size = cfg.block_size; + m_options.block_size = cfg.block_size; } if (cfg.block_restart_interval > 0) { - block_options.block_restart_interval = cfg.block_restart_interval; + //block_options.block_restart_interval = cfg.block_restart_interval; + m_options.block_restart_interval = cfg.block_restart_interval; } if (cfg.bloom_bits > 0) { - block_options.filter_policy.reset(rocksdb::NewBloomFilterPolicy(cfg.bloom_bits)); + //block_options.filter_policy.reset(rocksdb::NewBloomFilterPolicy(cfg.bloom_bits)); + m_options.filter_policy = rocksdb::NewBloomFilterPolicy(cfg.bloom_bits); } - m_options.table_factory.reset(rocksdb::NewBlockBasedTableFactory(block_options)); + //m_options.table_factory.reset(rocksdb::NewBlockBasedTableFactory(block_options)); if (cfg.write_buffer_size > 0) {