v0.8.1 downgrade rocksdb to 3.4 since 3.5 have some critical bugs

yinqiwen · Sep 18, 2014 · 2615cc4 · 2615cc4
1 parent f886601
commit 2615cc4
Show file tree

Hide file tree

Showing 14 changed files with 87 additions and 30 deletions.
diff --git a/deps/rocksdb-rocksdb-3.4.tar.gz b/deps/rocksdb-rocksdb-3.4.tar.gz
diff --git a/deps/rocksdb-rocksdb-3.5.tar.gz b/deps/rocksdb-rocksdb-3.5.tar.gz
diff --git a/patch/rocksdb-3.4.patch b/patch/rocksdb-3.4.patch
@@ -0,0 +1,11 @@
+--- util/rate_limiter.cc	2014-09-18 19:43:00.509681500 +0800
++++ util/rate_limiter.cc	2014-09-18 19:36:31.071741600 +0800
+@@ -60,7 +60,7 @@
+ }
+
+ void GenericRateLimiter::Request(int64_t bytes, const Env::IOPriority pri) {
+-  assert(bytes < refill_bytes_per_period_);
++  assert(bytes <= refill_bytes_per_period_);
+
+   MutexLock g(&request_mutex_);
+   if (stop_) {
diff --git a/patch/zookeeper.configure.ac.patch b/patch/zookeeper.configure.ac.patch
@@ -0,0 +1,18 @@
+--- configure.ac	2012-09-30 10:53:32.000000000 -0700
++++ configure.ac	2014-07-08 18:02:20.285143803 -0700
+@@ -41,9 +41,12 @@
+ CPPUNIT_CFLAGS="$CPPUNIT_CFLAGS -DZKSERVER_CMD=\"\\\"${base_dir}/src/c/tests/zkServer.sh\\\"\""
+ else
+ CPPUNIT_CFLAGS="$CPPUNIT_CFLAGS -DZKSERVER_CMD=\"\\\"./tests/zkServer.sh\\\"\""
+-AC_CHECK_FILES([generated/zookeeper.jute.c generated/zookeeper.jute.h],[],
+-    [AC_MSG_ERROR([jute files are missing! Please run "ant compile_jute" while in the zookeeper top level directory.])
+-])
++if test -f "generated/zookeeper.jute.c"; then :; else
++   AC_MSG_ERROR([jute files are missing! Please run "ant compile_jute" while in the zookeeper top level directory.])
++fi
++if test -f "generated/zookeeper.jute.h"; then :; else
++   AC_MSG_ERROR([jute files are missing! Please run "ant compile_jute" while in the zookeeper top level directory.])
++fi
+ fi
+ AC_SUBST(CPPUNIT_CFLAGS)
+
diff --git a/patch/zookeeper.mt_adaptor.patch b/patch/zookeeper.mt_adaptor.patch
@@ -0,0 +1,20 @@
+--- mt_adaptor.c	2012-09-30 10:53:32.000000000 -0700
++++ mt_adaptor.c	2014-07-01 16:05:07.387967525 -0700
+@@ -482,6 +482,9 @@
+
+ int32_t fetch_and_add(volatile int32_t* operand, int incr)
+ {
++#if defined(__GNUC__)
++  return __sync_fetch_and_add(operand, incr);
++#else
+ #ifndef WIN32
+     int32_t result;
+     asm __volatile__(
+@@ -503,6 +506,7 @@
+      }
+      return result;
+ #endif
++#endif
+ }
+
+ // make sure the static xid is initialized before any threads started
diff --git a/src/Makefile b/src/Makefile
@@ -34,7 +34,7 @@
 CXX=g++
 CC=gcc
 
-ARDB_VERSION=0.8.0
+ARDB_VERSION=0.8.1
 
 uname_S := $(shell sh -c 'uname -s 2>/dev/null || echo not')
 
@@ -59,7 +59,7 @@ LEVELDB_PATH=${LIB_PATH}/${LEVELDB_VER}
 LEVELDB_FILE=${LEVELDB_PATH}.tar.gz
 LEVELDB_LIBA=${LEVELDB_PATH}/libleveldb.a
 
-ROCKSDB_VER=rocksdb-rocksdb-3.5
+ROCKSDB_VER=rocksdb-rocksdb-3.4
 ROCKSDB_PATH=${LIB_PATH}/${ROCKSDB_VER}
 ROCKSDB_FILE=${ROCKSDB_PATH}.tar.gz
 ROCKSDB_LIBA=${ROCKSDB_PATH}/librocksdb.a
@@ -268,6 +268,7 @@ rocksdb: $(ROCKSDB_LIBA)
 $(ROCKSDB_LIBA): $(SNAPPY_LIBA) $(ROCKSDB_PATH)
 	echo ">>>>> Building ROCKSDB" && \
 	cd ${ROCKSDB_PATH} && \
+	patch -p0 < ../../patch/rocksdb-3.4.patch && \
 	CXXFLAGS="-I${SNAPPY_PATH}" CFLAGS="-I${SNAPPY_PATH}" LDFLAGS="${SNAPPY_PATH}/.libs" $(MAKE) librocksdb.a && \
 	echo "<<<<< Done building ROCKSDB"
 
@@ -299,9 +300,9 @@ $(ZOOKEEPER_PATH):
 	cd ${LIB_PATH} && \
 	tar zxf ${ZOOKEEPER_FILE} && \
 	cd ${ZOOKEEPER_PATH}/src && \
-	patch < ../../../zookeeper.mt_adaptor.patch && \
+	patch < ../patch/zookeeper.mt_adaptor.patch && \
 	cd ${ZOOKEEPER_PATH} && \
-	patch < ../../zookeeper.configure.ac.patch && \
+	patch < ../patch/zookeeper.configure.ac.patch && \
 	autoconf configure.ac > configure 2> autoconf.out && \
 	echo "<<<<<< Done unpacking ZOOKEEPER"
 

diff --git a/src/ardb.hpp b/src/ardb.hpp
@@ -543,6 +543,8 @@ OP_NAMESPACE_BEGIN
             int PFCount(Context& ctx, RedisCommandFrame& cmd);
             int PFMerge(Context& ctx, RedisCommandFrame& cmd);
 
+            int Cluster(Context& ctx, RedisCommandFrame& cmd);
+
             int DoCall(Context& ctx, RedisCommandHandlerSetting& setting, RedisCommandFrame& cmd);
             RedisCommandHandlerSetting* FindRedisCommandHandlerSetting(RedisCommandFrame& cmd);
             bool ParseConfig(const Properties& props);

diff --git a/src/command/t_hash.cpp b/src/command/t_hash.cpp
@@ -75,7 +75,7 @@ OP_NAMESPACE_BEGIN
                 it++;
                 if (!meta.attach.force_zipsave
                         && (field.StringLength() > m_cfg.hash_max_ziplist_value
-                                || value.StringLength() > m_cfg.hash_max_ziplist_value))
+                                || value.StringLength() > m_cfg.hash_max_ziplist_value) && meta.meta.zipmap.size() > 1)
                 {
                     zipsave = false;
                 }
@@ -400,7 +400,8 @@ OP_NAMESPACE_BEGIN
             {
                 break;
             }
-            if ((pattern.empty() || stringmatchlen(pattern.c_str(), pattern.size(), tmpelement.c_str(), tmpelement.size(), 0) == 1))
+            if ((pattern.empty()
+                    || stringmatchlen(pattern.c_str(), pattern.size(), tmpelement.c_str(), tmpelement.size(), 0) == 1))
             {
                 RedisReply& rr1 = r2.AddMember();
                 RedisReply& rr2 = r2.AddMember();
@@ -411,10 +412,11 @@ OP_NAMESPACE_BEGIN
         }
         if (iter.Valid())
         {
-            if(m_cfg.scan_redis_compatible)
+            if (m_cfg.scan_redis_compatible)
             {
                 fill_str_reply(r1, stringfromll(GetNewRedisCursor(tmpelement)));
-            }else
+            }
+            else
             {
                 fill_str_reply(r1, tmpelement);
             }

diff --git a/src/command/t_list.cpp b/src/command/t_list.cpp
@@ -337,8 +337,9 @@ OP_NAMESPACE_BEGIN
                         meta.meta.ziplist.push_back(element);
                     }
                 }
-                if (meta.meta.Length() >= m_cfg.list_max_ziplist_entries
-                        || element.StringLength() >= m_cfg.list_max_ziplist_value)
+                if (meta.meta.Length() > 1
+                        && (meta.meta.Length() >= m_cfg.list_max_ziplist_entries
+                                || element.StringLength() >= m_cfg.list_max_ziplist_value))
                 {
                     //convert to non ziplist
                     ZipListConvert(ctx, meta);

diff --git a/src/command/t_set.cpp b/src/command/t_set.cpp
@@ -65,7 +65,7 @@ OP_NAMESPACE_BEGIN
             count++;
             if (!meta.attach.force_zipsave
                     && (meta.meta.zipset.size() > m_cfg.set_max_ziplist_entries
-                            || element.StringLength() >= m_cfg.set_max_ziplist_value))
+                            || element.StringLength() >= m_cfg.set_max_ziplist_value) && meta.meta.zipset.size() > 1)
             {
                 meta.meta.SetEncoding(COLLECTION_ECODING_RAW);
                 DataSet::iterator it = meta.meta.zipset.begin();
@@ -993,12 +993,7 @@ OP_NAMESPACE_BEGIN
             SetIter(ctx, meta, meta.meta.min_index, iter, false);
             while (iter.Valid())
             {
-                KeyObject fk;
-                fk.db = ctx.currentDB;
-                fk.key = meta.key.key;
-                fk.type = SET_ELEMENT;
-                fk.element = *(iter.Element());
-                DelKeyValue((ctx), fk);
+                DelRaw(ctx, iter.CurrentRawKey());
                 iter.Next();
             }
         }

diff --git a/src/command/t_zset.cpp b/src/command/t_zset.cpp
@@ -154,8 +154,9 @@ OP_NAMESPACE_BEGIN
         if (meta.meta.Encoding() == COLLECTION_ECODING_ZIPZSET)
         {
             meta.meta.zipmap[element] = score;
-            if (meta.meta.zipmap.size() > m_cfg.zset_max_ziplist_entries
-                    || element.StringLength() >= m_cfg.zset_max_ziplist_value)
+            if (meta.meta.zipmap.size() > 1
+                    && (meta.meta.zipmap.size() > m_cfg.zset_max_ziplist_entries
+                            || element.StringLength() >= m_cfg.zset_max_ziplist_value))
             {
                 meta.meta.SetEncoding(COLLECTION_ECODING_RAW);
                 DataMap::iterator it = meta.meta.zipmap.begin();

diff --git a/src/common/channel/codec/redis_command.hpp b/src/common/channel/codec/redis_command.hpp
@@ -214,6 +214,8 @@ namespace ardb
 
             REDIS_CMD_HREPLACE = 175,
             REDIS_CMD_SREPLACE = 176,
+
+            REDIS_CMD_CLUSTER = 177,  //used in cluster mode
         };
 
         class RedisCommandDecoder;

diff --git a/src/config.hpp b/src/config.hpp
@@ -147,9 +147,9 @@ OP_NAMESPACE_BEGIN
                             false), repl_ping_slave_period(10), repl_timeout(60), repl_backlog_size(100 * 1024 * 1024), repl_state_persist_period(
                             1), repl_backlog_time_limit(3600), slave_cleardb_before_fullresync(true), slave_readonly(
                             true), slave_serve_stale_data(true), slave_priority(100), lua_time_limit(0), master_port(0), loglevel(
-                            "INFO"), hash_max_ziplist_entries(128), hash_max_ziplist_value(64), list_max_ziplist_entries(
-                            128), list_max_ziplist_value(64), zset_max_ziplist_entries(128), zset_max_ziplist_value(64), set_max_ziplist_entries(
-                            128), set_max_ziplist_value(64), L1_zset_max_cache_size(0), L1_set_max_cache_size(0), L1_list_max_cache_size(
+                            "INFO"), hash_max_ziplist_entries(128), hash_max_ziplist_value(256), list_max_ziplist_entries(
+                            128), list_max_ziplist_value(256), zset_max_ziplist_entries(128), zset_max_ziplist_value(256), set_max_ziplist_entries(
+                            128), set_max_ziplist_value(256), L1_zset_max_cache_size(0), L1_set_max_cache_size(0), L1_list_max_cache_size(
                             0), L1_hash_max_cache_size(0), L1_string_max_cache_size(0), L1_zset_read_fill_cache(false), L1_zset_seek_load_cache(
                             false), L1_set_read_fill_cache(false), L1_set_seek_load_cache(false), L1_hash_read_fill_cache(
                             false), L1_hash_seek_load_cache(false), L1_list_read_fill_cache(false), L1_list_seek_load_cache(

diff --git a/src/engine/rocksdb_engine.cpp b/src/engine/rocksdb_engine.cpp
@@ -170,28 +170,32 @@ namespace ardb
         m_cfg = cfg;
         m_options.create_if_missing = true;
         m_options.comparator = &m_comparator;
-        rocksdb::BlockBasedTableOptions block_options;
+        //rocksdb::BlockBasedTableOptions block_options;
         if (cfg.block_cache_size > 0)
         {
-            block_options.block_cache = rocksdb::NewLRUCache(cfg.block_cache_size);
-            //m_options.block_cache_compressed = rocksdb::NewLRUCache(cfg.block_cache_compressed_size);
+            //block_options.block_cache = rocksdb::NewLRUCache(cfg.block_cache_size);
+            m_options.block_cache_compressed = rocksdb::NewLRUCache(cfg.block_cache_compressed_size);
         }else if(cfg.block_cache_size < 0)
         {
-            block_options.no_block_cache = true;
+            //block_options.no_block_cache = true;
+            m_options.no_block_cache = true;
         }
         if (cfg.block_size > 0)
         {
-            block_options.block_size = cfg.block_size;
+            //block_options.block_size = cfg.block_size;
+            m_options.block_size = cfg.block_size;
         }
         if (cfg.block_restart_interval > 0)
         {
-            block_options.block_restart_interval = cfg.block_restart_interval;
+            //block_options.block_restart_interval = cfg.block_restart_interval;
+            m_options.block_restart_interval = cfg.block_restart_interval;
         }
         if (cfg.bloom_bits > 0)
         {
-            block_options.filter_policy.reset(rocksdb::NewBloomFilterPolicy(cfg.bloom_bits));
+            //block_options.filter_policy.reset(rocksdb::NewBloomFilterPolicy(cfg.bloom_bits));
+            m_options.filter_policy = rocksdb::NewBloomFilterPolicy(cfg.bloom_bits);
         }
-        m_options.table_factory.reset(rocksdb::NewBlockBasedTableFactory(block_options));
+        //m_options.table_factory.reset(rocksdb::NewBlockBasedTableFactory(block_options));
 
         if (cfg.write_buffer_size > 0)
         {