Replace script with build script, use features

.gitignore

@@ -28,5 +28,6 @@ tmp
 harness/target
 
 Cargo.lock
+rust-toolchain
 *.rs.bk
 *.rs.fmt

.travis.yml

@@ -27,7 +27,7 @@ matrix:
       - rustup component add clippy-preview
     before_script:
       - cargo fmt --all -- --check
-      - cargo clippy --all -- -D clippy
+      - cargo clippy --all -- -D clippy::all
   # Since many users will use nightlies, also test that.
   - rust: nightly
 

Cargo.toml

@@ -10,16 +10,25 @@ homepage = "https://github.com/pingcap/raft-rs"
 documentation = "https://docs.rs/raft"
 description = "The rust language implementation of Raft algorithm."
 categories = ["algorithms", "database-implementations"]
+edition = "2018"
+build = "build.rs"
+
+[build-dependencies]
+protoc-rust = "~2.0-2.2"
+protoc-grpcio = "0.3.1"
+regex = "1.1"
 
 [features]
-default = []
+default = ["lib-rust-protobuf"]
+lib-rust-protobuf = ["protobuf"]
+regenerate = []
 # Enable failpoints
 failpoint = ["fail"]
 
 # Make sure to synchronize updates with Harness.
 [dependencies]
 log = ">0.2"
-protobuf = "~2.0-2.2"
+protobuf = { version = "~2.0-2.2", optional = true }
 quick-error = "1.2.2"
 rand = "0.5.4"
 hashbrown = "0.1"

benches/benches.rs

@@ -1,10 +1,6 @@
 #![allow(dead_code)] // Due to criterion we need this to avoid warnings.
 #![cfg_attr(feature = "cargo-clippy", allow(clippy::let_and_return))] // Benches often artificially return values. Allow it.
 
-extern crate criterion;
-extern crate env_logger;
-extern crate raft;
-
 use criterion::Criterion;
 use std::time::Duration;
 

benches/suites/progress_set.rs

@@ -1,6 +1,6 @@
+use crate::DEFAULT_RAFT_SETS;
 use criterion::{Bencher, Criterion};
 use raft::{Progress, ProgressSet};
-use DEFAULT_RAFT_SETS;
 
 pub fn bench_progress_set(c: &mut Criterion) {
     bench_progress_set_new(c);

benches/suites/raft.rs

@@ -1,6 +1,6 @@
+use crate::DEFAULT_RAFT_SETS;
 use criterion::{Bencher, Criterion};
 use raft::{storage::MemStorage, Config, Raft};
-use DEFAULT_RAFT_SETS;
 
 pub fn bench_raft(c: &mut Criterion) {
     bench_raft_new(c);

build.rs

@@ -0,0 +1,170 @@
+// Copyright 2019 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+extern crate regex;
+
+use regex::Regex;
+use std::fs::read_dir;
+use std::fs::File;
+use std::io::{Read, Write};
+use std::process::Command;
+use std::{env, str};
+
+fn main() {
+    // This build script creates files in the `src` directory. Since that is
+    // outside Cargo's OUT_DIR it will cause an error when this crate is used
+    // as a dependency. Therefore, the user must opt-in to regenerating the
+    // Rust files.
+    if env::var_os("CARGO_FEATURE_REGENERATE").is_none() {
+        println!("cargo:rerun-if-changed=build.rs");
+        return;
+    }
+
+    let file_names: Vec<_> = read_dir("proto")
+        .expect("Couldn't read proto directory")
+        .map(|e| {
+            format!(
+                "proto/{}",
+                e.expect("Couldn't list file").file_name().to_string_lossy()
+            )
+        })
+        .collect();
+    let file_names: Vec<_> = file_names.iter().map(|s| &**s).collect();
+
+    for f in &file_names {
+        println!("cargo:rerun-if-changed={}", f);
+    }
+
+    match BufferLib::from_env_vars() {
+        BufferLib::Prost => {
+            unimplemented!("Prost support is not yet implemented");
+        }
+
+        BufferLib::Protobuf => {
+            check_protoc_version();
+
+            generate_protobuf_files(file_names);
+            let mod_names: Vec<_> = read_dir("src/rsprotobuf")
+                .expect("Couldn't read src directory")
+                .filter_map(|e| {
+                    let file_name = e.expect("Couldn't list file").file_name();
+                    file_name
+                        .to_string_lossy()
+                        .split(".rs")
+                        .next()
+                        .map(|n| n.to_owned())
+                })
+                .collect();
+            replace_read_unknown_fields(&mod_names);
+            generate_protobuf_rs(&mod_names);
+        }
+    }
+}
+
+#[derive(Eq, PartialEq)]
+enum BufferLib {
+    Prost,
+    Protobuf,
+}
+
+impl BufferLib {
+    fn from_env_vars() -> BufferLib {
+        match (
+            env::var_os("CARGO_FEATURE_LIB_PROST"),
+            env::var_os("CARGO_FEATURE_LIB_RUST_PROTOBUF"),
+        ) {
+            (Some(_), Some(_)) | (None, None) => {
+                panic!("You must use exactly one of `lib-rust-protobuf` and `lib-prost` features")
+            }
+            (Some(_), _) => BufferLib::Prost,
+            (_, Some(_)) => BufferLib::Protobuf,
+        }
+    }
+}
+
+fn check_protoc_version() {
+    let ver_re = Regex::new(r"([0-9]+)\.([0-9]+)\.[0-9]").unwrap();
+    let ver = Command::new("protoc")
+        .arg("--version")
+        .output()
+        .expect("Program `protoc` not installed (is it in PATH?).");
+    let caps = ver_re
+        .captures(str::from_utf8(&ver.stdout).unwrap())
+        .unwrap();
+    let major = caps.get(1).unwrap().as_str().parse::<i16>().unwrap();
+    let minor = caps.get(2).unwrap().as_str().parse::<i16>().unwrap();
+    if major == 3 && minor < 1 || major < 3 {
+        panic!(
+            "Invalid version of protoc (required 3.1.x, get {}.{}.x).",
+            major, minor,
+        );
+    }
+}
+
+fn generate_protobuf_files(file_names: Vec<&str>) {
+    protoc_rust::run(protoc_rust::Args {
+        out_dir: "src/rsprotobuf",
+        input: &file_names,
+        includes: &["proto", "include"],
+        customize: protoc_rust::Customize {
+            ..Default::default()
+        },
+    })
+    .unwrap();
+
+    protoc_grpcio::compile_grpc_protos(file_names, &["proto", "include"], "src/rsprotobuf")
+        .unwrap();
+}
+
+// Use the old way to read protobuf enums.
+// FIXME: Remove this once stepancheg/rust-protobuf#233 is resolved.
+fn replace_read_unknown_fields(mod_names: &[String]) {
+    let regex =
+        Regex::new(r"::protobuf::rt::read_proto3_enum_with_unknown_fields_into\(([^,]+), ([^,]+), &mut ([^,]+), [^\)]+\)\?").unwrap();
+    for mod_name in mod_names {
+        let file_name = &format!("src/rsprotobuf/{}.rs", mod_name);
+
+        let mut text = String::new();
+        {
+            let mut f = File::open(file_name).unwrap();
+            f.read_to_string(&mut text)
+                .expect("Couldn't read source file");
+        }
+
+        let text = regex.replace_all(
+            &text,
+            "if $1 == ::protobuf::wire_format::WireTypeVarint {\
+             $3 = $2.read_enum()?;\
+             } else {\
+             return ::std::result::Result::Err(::protobuf::rt::unexpected_wire_type(wire_type));\
+             }",
+        );
+        let mut out = File::create(file_name).unwrap();
+        out.write_all(text.as_bytes())
+            .expect("Could not write source file");
+    }
+}
+
+fn generate_protobuf_rs(mod_names: &[String]) {
+    let mut text = "".to_owned();
+
+    for mod_name in mod_names {
+        text.push_str("pub mod ");
+        text.push_str(mod_name);
+        text.push_str(";\n");
+    }
+
+    let mut lib = File::create("src/rsprotobuf.rs").expect("Could not create rsprotobuf.rs");
+    lib.write_all(text.as_bytes())
+        .expect("Could not write rsprotobuf.rs");
+}

examples/five_mem_node/main.rs

@@ -12,10 +12,7 @@
 // limitations under the License.
 #[macro_use]
 extern crate log;
-extern crate env_logger;
-extern crate protobuf;
-extern crate raft;
-extern crate regex;
+use env_logger;
 
 use std::collections::{HashMap, VecDeque};
 use std::sync::mpsc::{self, Receiver, Sender, SyncSender, TryRecvError};

examples/single_mem_node/main.rs

@@ -11,7 +11,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-extern crate raft;
+use raft;
 
 use std::collections::HashMap;
 use std::sync::mpsc::{self, RecvTimeoutError};
@@ -21,7 +21,7 @@ use std::time::{Duration, Instant};
 use raft::prelude::*;
 use raft::storage::MemStorage;
 
-type ProposeCallback = Box<Fn() + Send>;
+type ProposeCallback = Box<dyn Fn() + Send>;
 
 enum Msg {
     Propose {

src/errors.rs

@@ -11,10 +11,11 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+use crate::StateRole;
 use std::error;
 use std::{cmp, io, result};
-use StateRole;
 
+#[cfg(feature = "lib-rust-protobuf")]
 use protobuf::ProtobufError;
 
 quick_error! {
@@ -50,6 +51,7 @@ quick_error! {
             description(desc)
         }
         /// A Protobuf message failed in some manner.
+        #[cfg(feature = "lib-rust-protobuf")]
         Codec(err: ProtobufError) {
             from()
             cause(err)
@@ -115,7 +117,7 @@ quick_error! {
             description("snapshot is temporarily unavailable")
         }
         /// Some other error occurred.
-        Other(err: Box<error::Error + Sync + Send>) {
+        Other(err: Box<dyn error::Error + Sync + Send>) {
             from()
             cause(err.as_ref())
             description(err.description())