Add test_follower option to crash test

db_stress_tool/db_stress_common.h

@@ -176,7 +176,9 @@ DECLARE_int32(index_type);
 DECLARE_int32(data_block_index_type);
 DECLARE_string(db);
 DECLARE_string(secondaries_base);
+DECLARE_string(followers_base);
 DECLARE_bool(test_secondary);
+DECLARE_bool(test_follower);
 DECLARE_string(expected_values_dir);
 DECLARE_bool(verify_checksum);
 DECLARE_bool(mmap_read);

db_stress_tool/db_stress_gflags.cc

@@ -618,10 +618,17 @@ DEFINE_string(db, "", "Use the db with the following name.");
 DEFINE_string(secondaries_base, "",
               "Use this path as the base path for secondary instances.");
 
+DEFINE_string(followers_base, "",
+              "Use this path as the base path for follower instances.");
+
 DEFINE_bool(test_secondary, false,
             "If true, start an additional secondary instance which can be used "
             "for verification.");
 
+DEFINE_bool(test_follower, false,
+            "If true, start an additional follower instance which can be used "
+            "for verification.");
+
 DEFINE_string(
     expected_values_dir, "",
     "Dir where files containing info about the latest/historical values will "

db_stress_tool/db_stress_stat.cc

@@ -11,6 +11,7 @@ namespace ROCKSDB_NAMESPACE {
 
 std::shared_ptr<ROCKSDB_NAMESPACE::Statistics> dbstats;
 std::shared_ptr<ROCKSDB_NAMESPACE::Statistics> dbstats_secondaries;
+std::shared_ptr<ROCKSDB_NAMESPACE::Statistics> dbstats_followers;
 
 }  // namespace ROCKSDB_NAMESPACE
 

db_stress_tool/db_stress_stat.h

@@ -25,6 +25,7 @@ namespace ROCKSDB_NAMESPACE {
 // Database statistics
 extern std::shared_ptr<ROCKSDB_NAMESPACE::Statistics> dbstats;
 extern std::shared_ptr<ROCKSDB_NAMESPACE::Statistics> dbstats_secondaries;
+extern std::shared_ptr<ROCKSDB_NAMESPACE::Statistics> dbstats_followers;
 
 class Stats {
  private:

db_stress_tool/db_stress_test_base.cc

@@ -645,6 +645,10 @@ void StressTest::PrintStatistics() {
     fprintf(stdout, "Secondary instances STATISTICS:\n%s\n",
             dbstats_secondaries->ToString().c_str());
   }
+  if (dbstats_followers) {
+    fprintf(stdout, "Follower instances STATISTICS:\n%s\n",
+            dbstats_followers->ToString().c_str());
+  }
 }
 
 // Currently PreloadDb has to be single-threaded.
@@ -3686,9 +3690,9 @@ void StressTest::Open(SharedState* shared, bool reopen) {
     assert(column_families_.size() ==
            static_cast<size_t>(FLAGS_column_families));
 
-    // Secondary instance does not support write-prepared/write-unprepared
-    // transactions, thus just disable secondary instance if we use
-    // transaction.
+    // Secondary and follower instances do not support
+    // write-prepared/write-unprepared transactions, thus just disable secondary
+    // instance if we use transaction.
     if (s.ok() && FLAGS_test_secondary && !FLAGS_use_txn) {
       Options tmp_opts;
       // TODO(yanqin) support max_open_files != -1 for secondary instance.
@@ -3700,6 +3704,24 @@ void StressTest::Open(SharedState* shared, bool reopen) {
       assert(s.ok());
       assert(cmp_cfhs_.size() == static_cast<size_t>(FLAGS_column_families));
     }
+
+    if (s.ok() && FLAGS_test_follower && !FLAGS_use_txn) {
+      Options tmp_opts;
+      tmp_opts.max_open_files = -1;
+      tmp_opts.env = db_stress_env;
+      // Equivalent to "name" argument in OpenAsSecondary
+      const std::string& leader_path = FLAGS_db;
+      // Equivalent to "secondary_path" in OpenAsSecondary
+      const std::string& name = FLAGS_followers_base;
+      s = DB::OpenAsFollower(tmp_opts, name, leader_path, cf_descriptors,
+                             &follower_cfhs_, &follower_db_);
+      if (!s.ok()) {
+        fprintf(stderr, "Error opening follower: %s\n", s.ToString().c_str());
+      }
+      assert(s.ok());
+      assert(follower_cfhs_.size() ==
+             static_cast<size_t>(FLAGS_column_families));
+    }
   } else {
     DBWithTTL* db_with_ttl;
     s = DBWithTTL::Open(options_, FLAGS_db, &db_with_ttl, FLAGS_ttl);

db_stress_tool/db_stress_test_base.h

@@ -415,6 +415,8 @@ class StressTest {
   // Fields used for continuous verification from another thread
   DB* cmp_db_;
   std::vector<ColumnFamilyHandle*> cmp_cfhs_;
+  std::unique_ptr<DB> follower_db_;
+  std::vector<ColumnFamilyHandle*> follower_cfhs_;
   bool is_db_stopped_;
 };
 

db_stress_tool/db_stress_tool.cc

@@ -58,6 +58,9 @@ int db_stress_tool(int argc, char** argv) {
     if (FLAGS_test_secondary) {
       dbstats_secondaries = ROCKSDB_NAMESPACE::CreateDBStatistics();
     }
+    if (FLAGS_test_follower) {
+      dbstats_followers = ROCKSDB_NAMESPACE::CreateDBStatistics();
+    }
   }
   compression_type_e = StringToCompressionType(FLAGS_compression_type.c_str());
   bottommost_compression_type_e =
@@ -231,6 +234,19 @@ int db_stress_tool(int argc, char** argv) {
     FLAGS_secondaries_base = default_secondaries_path;
   }
 
+  if (FLAGS_test_follower && FLAGS_followers_base.empty()) {
+    std::string default_followers_path;
+    db_stress_env->GetTestDirectory(&default_followers_path);
+    default_followers_path += "/dbstress_followers";
+    s = db_stress_env->CreateDirIfMissing(default_followers_path);
+    if (!s.ok()) {
+      fprintf(stderr, "Failed to create directory %s: %s\n",
+              default_followers_path.c_str(), s.ToString().c_str());
+      exit(1);
+    }
+    FLAGS_followers_base = default_followers_path;
+  }
+
   if (FLAGS_best_efforts_recovery &&
       !(FLAGS_skip_verifydb && FLAGS_disable_wal)) {
     fprintf(stderr,

db_stress_tool/multi_ops_txns_stress.cc

@@ -1759,11 +1759,11 @@ void CheckAndSetOptionsForMultiOpsTxnStressTest() {
   if (!FLAGS_use_txn) {
     fprintf(stderr, "-use_txn must be true if -test_multi_ops_txns\n");
     exit(1);
-  } else if (FLAGS_test_secondary > 0) {
-    fprintf(
-        stderr,
-        "secondary instance does not support replaying logs (MANIFEST + WAL) "
-        "of TransactionDB with write-prepared/write-unprepared policy\n");
+  } else if (FLAGS_test_secondary > 0 || FLAGS_test_follower > 0) {
+    fprintf(stderr,
+            "secondary and follower instances do not support replaying logs "
+            "(MANIFEST + WAL) "
+            "of TransactionDB with write-prepared/write-unprepared policy\n");
     exit(1);
   }
   if (FLAGS_clear_column_family_one_in > 0) {

tools/db_crashtest.py

@@ -485,6 +485,7 @@ def is_direct_io_supported(dbname):
     "write_buffer_size": 32 * 1024 * 1024,
     "level_compaction_dynamic_level_bytes": lambda: random.randint(0, 1),
     "paranoid_file_checks": lambda: random.choice([0, 1, 1, 1]),
+    "test_follower": lambda: random.choice([0, 1]),
 }
 
 blackbox_simple_default_params = {