WIP: Codegen option to create module hierarchy

protobuf-codegen/Cargo.toml

@@ -22,6 +22,10 @@ regex     = "1.5.5"
 once_cell = "1.10.0"
 tempfile  = "3"
 
+tracing = "0.1.40"
+tracing-appender = "0.2.2"
+tracing-subscriber = { version = "0.3.17", features = ["env-filter", "fmt", "json"] }
+
 protobuf = { path = "../protobuf", version = "=4.0.0-alpha.0" }
 protobuf-parse = { path = "../protobuf-parse", version = "=4.0.0-alpha.0" }
 

protobuf-codegen/src/customize/mod.rs

@@ -106,9 +106,15 @@ pub struct Customize {
     ///
     /// This option will likely be on by default in rust-protobuf version 3.
     pub(crate) gen_mod_rs: Option<bool>,
+    /// Generate `mod.rs` in a hierarchy in the output directory assuming that
+    /// this name is the crate-relative Rust path to the output directory.
+    pub(crate) gen_mod_rs_hierarchy_out_dir_mod_name: Option<String>,
     /// Used internally to generate protos bundled in protobuf crate
     /// like `descriptor.proto`
     pub(crate) inside_protobuf: Option<bool>,
+    /// Enable logging to the specified `log_file` for debugging. The logging
+    /// verbosity is controlled with the `RUST_LOG` environment variable.
+    pub(crate) log_file: Option<String>,
 }
 
 #[derive(Debug, thiserror::Error)]
@@ -165,12 +171,29 @@ impl Customize {
         self
     }
 
+    /// Generate `mod.rs` with all the generated modules in a Rust mod
+    /// hierarchy.
+    ///
+    /// This option is on by default in rust-protobuf version 3.
+    pub fn gen_mod_rs_hierarchy_out_dir_mod_name(
+        mut self,
+        gen_mod_rs_hierarchy_out_dir_mod_name: String,
+    ) -> Self {
+        self.gen_mod_rs_hierarchy_out_dir_mod_name = Some(gen_mod_rs_hierarchy_out_dir_mod_name);
+        self
+    }
+
     /// Generate code bundled in protobuf crate. Regular users don't need this option.
     pub fn inside_protobuf(mut self, inside_protobuf: bool) -> Self {
         self.inside_protobuf = Some(inside_protobuf);
         self
     }
 
+    pub fn log_file(mut self, log_file: String) -> Self {
+        self.log_file = Some(log_file);
+        self
+    }
+
     /// Update fields of self with fields defined in other customize
     pub fn update_with(&mut self, that: &Customize) {
         if let Some(v) = &that.before {
@@ -194,9 +217,15 @@ impl Customize {
         if let Some(v) = that.gen_mod_rs {
             self.gen_mod_rs = Some(v);
         }
+        if let Some(v) = &that.gen_mod_rs_hierarchy_out_dir_mod_name {
+            self.gen_mod_rs_hierarchy_out_dir_mod_name = Some(v.to_owned());
+        }
         if let Some(v) = that.inside_protobuf {
             self.inside_protobuf = Some(v);
         }
+        if let Some(v) = &that.log_file {
+            self.log_file = Some(v.to_owned());
+        }
     }
 
     /// Update unset fields of self with fields from other customize
@@ -236,12 +265,16 @@ impl Customize {
                 r.lite_runtime = Some(parse_bool(v)?);
             } else if n == "gen_mod_rs" {
                 r.gen_mod_rs = Some(parse_bool(v)?);
+            } else if n == "gen_mod_rs_hierarchy_out_dir_mod_name" {
+                r.gen_mod_rs_hierarchy_out_dir_mod_name = Some(v.to_owned());
             } else if n == "inside_protobuf" {
                 r.inside_protobuf = Some(parse_bool(v)?);
             } else if n == "lite" {
                 // Support Java and C++ protoc plugin syntax:
                 // https://github.com/protocolbuffers/protobuf/issues/6489
                 r.lite_runtime = Some(parse_bool(v)?);
+            } else if n == "log_file" {
+                r.log_file = Some(v.to_owned());
             } else {
                 return Err(CustomizeParseParameterError::UnknownOptionName(n.to_owned()).into());
             }

protobuf-codegen/src/customize/rustproto_proto.rs

@@ -14,7 +14,9 @@ pub(crate) fn customize_from_rustproto_for_message(source: &MessageOptions) -> C
     let tokio_bytes_for_string = rustproto::exts::tokio_bytes_for_string.get(source);
     let lite_runtime = None;
     let gen_mod_rs = None;
+    let gen_mod_rs_hierarchy_out_dir_mod_name = None;
     let inside_protobuf = None;
+    let log_file = None;
     Customize {
         before,
         generate_accessors,
@@ -23,7 +25,9 @@ pub(crate) fn customize_from_rustproto_for_message(source: &MessageOptions) -> C
         tokio_bytes_for_string,
         lite_runtime,
         gen_mod_rs,
+        gen_mod_rs_hierarchy_out_dir_mod_name,
         inside_protobuf,
+        log_file,
     }
 }
 
@@ -39,7 +43,9 @@ pub(crate) fn customize_from_rustproto_for_field(source: &FieldOptions) -> Custo
     let tokio_bytes_for_string = rustproto::exts::tokio_bytes_for_string_field.get(source);
     let lite_runtime = None;
     let gen_mod_rs = None;
+    let gen_mod_rs_hierarchy_out_dir_mod_name = None;
     let inside_protobuf = None;
+    let log_file = None;
     Customize {
         before,
         generate_accessors,
@@ -48,7 +54,9 @@ pub(crate) fn customize_from_rustproto_for_field(source: &FieldOptions) -> Custo
         tokio_bytes_for_string,
         lite_runtime,
         gen_mod_rs,
+        gen_mod_rs_hierarchy_out_dir_mod_name,
         inside_protobuf,
+        log_file,
     }
 }
 
@@ -60,7 +68,9 @@ pub(crate) fn customize_from_rustproto_for_file(source: &FileOptions) -> Customi
     let tokio_bytes_for_string = rustproto::exts::tokio_bytes_for_string_all.get(source);
     let lite_runtime = rustproto::exts::lite_runtime_all.get(source);
     let gen_mod_rs = None;
+    let gen_mod_rs_hierarchy_out_dir_mod_name = None;
     let inside_protobuf = None;
+    let log_file = None;
     Customize {
         before,
         generate_accessors,
@@ -70,5 +80,7 @@ pub(crate) fn customize_from_rustproto_for_file(source: &FileOptions) -> Customi
         lite_runtime,
         inside_protobuf,
         gen_mod_rs,
+        gen_mod_rs_hierarchy_out_dir_mod_name,
+        log_file,
     }
 }

protobuf-codegen/src/gen/all.rs

@@ -1,9 +1,10 @@
-use std::collections::HashMap;
-
 use protobuf::descriptor::FileDescriptorProto;
 use protobuf::reflect::FileDescriptor;
 use protobuf_parse::ProtoPath;
 use protobuf_parse::ProtoPathBuf;
+use std::collections::HashMap;
+use tracing::instrument;
+use tracing::Level;
 
 use crate::compiler_plugin;
 use crate::customize::ctx::CustomizeElemCtx;
@@ -14,6 +15,7 @@ use crate::gen::scope::RootScope;
 use crate::gen::well_known_types::gen_well_known_types_mod;
 use crate::Customize;
 
+#[instrument(level = Level::INFO, skip_all)]
 pub(crate) fn gen_all(
     file_descriptors: &[FileDescriptorProto],
     parser: &str,

protobuf-codegen/src/gen/file.rs

@@ -4,6 +4,9 @@ use protobuf::descriptor::file_options;
 use protobuf::descriptor::FileDescriptorProto;
 use protobuf::reflect::FileDescriptor;
 use protobuf_parse::ProtoPath;
+use tracing::event;
+use tracing::instrument;
+use tracing::Level;
 
 use crate::compiler_plugin;
 use crate::customize::ctx::CustomizeElemCtx;
@@ -14,6 +17,7 @@ use crate::gen::extensions::write_extensions;
 use crate::gen::file_descriptor::write_file_descriptor_data;
 use crate::gen::inside::protobuf_crate_path;
 use crate::gen::message::MessageGen;
+use crate::gen::paths::file_descriptor_to_hierarchical_rs;
 use crate::gen::paths::proto_path_to_rust_mod;
 use crate::gen::scope::FileScope;
 use crate::gen::scope::RootScope;
@@ -24,6 +28,7 @@ pub(crate) struct GenFileResult {
     pub(crate) mod_name: String,
 }
 
+#[instrument(level = Level::INFO, skip_all)]
 pub(crate) fn gen_file(
     file_descriptor: &FileDescriptor,
     _files_map: &HashMap<&ProtoPath, &FileDescriptor>,
@@ -53,6 +58,17 @@ pub(crate) fn gen_file(
 
     let lite_runtime = customize.for_elem.lite_runtime.unwrap_or(false);
 
+    let mod_name = proto_path_to_rust_mod(file_descriptor.proto().name()).into_string();
+
+    event!(
+        Level::INFO,
+        file_descriptor.name = file_descriptor.proto().name(),
+        file_descriptor.package = file_descriptor.proto().package(),
+        lite_runtime,
+        mod_name,
+        "Generating file"
+    );
+
     let v = CodeWriter::with(|w| {
         w.write_generated_by("rust-protobuf", env!("CARGO_PKG_VERSION"), parser);
 
@@ -141,9 +157,17 @@ pub(crate) fn gen_file(
 
     Ok(GenFileResult {
         compiler_plugin_result: compiler_plugin::GenResult {
-            name: proto_name_to_rs(file_descriptor.proto().name()),
+            name: if customize
+                .for_elem
+                .gen_mod_rs_hierarchy_out_dir_mod_name
+                .is_some()
+            {
+                file_descriptor_to_hierarchical_rs(file_descriptor.proto())
+            } else {
+                proto_name_to_rs(file_descriptor.proto().name())
+            },
             content: v.into_bytes(),
         },
-        mod_name: proto_path_to_rust_mod(file_descriptor.proto().name()).into_string(),
+        mod_name,
     })
 }

protobuf-codegen/src/gen/message.rs

@@ -4,6 +4,8 @@ use protobuf::descriptor::*;
 use protobuf::reflect::FileDescriptor;
 use protobuf::reflect::MessageDescriptor;
 use protobuf_parse::snake_case;
+use tracing::event;
+use tracing::Level;
 
 use crate::customize::ctx::CustomizeElemCtx;
 use crate::customize::ctx::SpecialFieldPseudoDescriptor;
@@ -657,6 +659,14 @@ impl<'a> MessageGen<'a> {
     }
 
     pub fn write(&self, w: &mut CodeWriter) -> anyhow::Result<()> {
+        event!(
+            Level::INFO,
+            proto_file.name = self.file_descriptor.proto().name(),
+            proto_file.package = self.file_descriptor.proto().package(),
+            message.name = self.message_descriptor.proto().name(),
+            "Generating message"
+        );
+
         w.all_documentation(self.info, self.path);
         self.write_struct(w);
 

protobuf-codegen/src/gen/paths.rs

@@ -1,3 +1,6 @@
+use protobuf::descriptor::FileDescriptorProto;
+use tracing::{instrument, Level};
+
 use crate::gen::inside::protobuf_crate_path;
 use crate::gen::rust::ident::RustIdent;
 use crate::gen::rust::path::RustPath;
@@ -40,10 +43,24 @@ pub(crate) fn proto_path_to_rust_mod(path: &str) -> RustIdent {
 }
 
 /// Used in protobuf-codegen-identical-test
-pub fn proto_name_to_rs(proto_file_path: &str) -> String {
-    format!("{}.rs", proto_path_to_rust_mod(proto_file_path))
+pub fn proto_name_to_rs(proto_name: &str) -> String {
+    format!("{}.rs", proto_path_to_rust_mod(proto_name))
 }
 
+/// Determines the one-for-one relative path in which a Rust source file
+/// should be found for the provided `file_descriptor`.
+pub fn file_descriptor_to_hierarchical_rs(file_descriptor: &FileDescriptorProto) -> String {
+    match &file_descriptor.package {
+        Some(package) => format!(
+            "{}/{}",
+            package.replace('.', "/"),
+            proto_name_to_rs(file_descriptor.name())
+        ),
+        None => proto_name_to_rs(file_descriptor.name()),
+    }
+}
+
+#[instrument(level = Level::DEBUG, skip(customize), ret(Display))]
 pub(crate) fn proto_path_to_fn_file_descriptor(
     proto_path: &str,
     customize: &Customize,
@@ -58,9 +75,22 @@ pub(crate) fn proto_path_to_fn_file_descriptor(
             .append_ident("well_known_types".into())
             .append_ident(proto_path_to_rust_mod(s))
             .append_ident("file_descriptor".into()),
-        s => RustPath::super_path()
-            .append_ident(proto_path_to_rust_mod(s))
-            .append_ident("file_descriptor".into()),
+        s => {
+            if let Some(mod_path) = &customize.gen_mod_rs_hierarchy_out_dir_mod_name {
+                let mut rust_path = RustPath::from("crate");
+                for mod_part in mod_path.split("::") {
+                    rust_path = rust_path.append_ident(RustIdent::from(mod_part));
+                }
+                for component in proto_path.split("/").filter(|p| !p.ends_with(".proto")) {
+                    rust_path = rust_path.append_ident(RustIdent::from(component));
+                }
+                rust_path.append_ident("file_descriptor".into())
+            } else {
+                RustPath::super_path()
+                    .append_ident(proto_path_to_rust_mod(s))
+                    .append_ident("file_descriptor".into())
+            }
+        }
     }
 }