diff --git a/protobuf-codegen/Cargo.toml b/protobuf-codegen/Cargo.toml index 00dabb3ac..8576f5246 100644 --- a/protobuf-codegen/Cargo.toml +++ b/protobuf-codegen/Cargo.toml @@ -22,6 +22,10 @@ regex = "1.5.5" once_cell = "1.10.0" tempfile = "3" +tracing = "0.1.40" +tracing-appender = "0.2.2" +tracing-subscriber = { version = "0.3.17", features = ["env-filter", "fmt", "json"] } + protobuf = { path = "../protobuf", version = "=4.0.0-alpha.0" } protobuf-parse = { path = "../protobuf-parse", version = "=4.0.0-alpha.0" } diff --git a/protobuf-codegen/src/customize/mod.rs b/protobuf-codegen/src/customize/mod.rs index 826bd3f6d..277a4bcca 100644 --- a/protobuf-codegen/src/customize/mod.rs +++ b/protobuf-codegen/src/customize/mod.rs @@ -106,9 +106,15 @@ pub struct Customize { /// /// This option will likely be on by default in rust-protobuf version 3. pub(crate) gen_mod_rs: Option, + /// Generate `mod.rs` in a hierarchy in the output directory assuming that + /// this name is the crate-relative Rust path to the output directory. + pub(crate) gen_mod_rs_hierarchy_out_dir_mod_name: Option, /// Used internally to generate protos bundled in protobuf crate /// like `descriptor.proto` pub(crate) inside_protobuf: Option, + /// Enable logging to the specified `log_file` for debugging. The logging + /// verbosity is controlled with the `RUST_LOG` environment variable. + pub(crate) log_file: Option, } #[derive(Debug, thiserror::Error)] @@ -165,12 +171,29 @@ impl Customize { self } + /// Generate `mod.rs` with all the generated modules in a Rust mod + /// hierarchy. + /// + /// This option is on by default in rust-protobuf version 3. + pub fn gen_mod_rs_hierarchy_out_dir_mod_name( + mut self, + gen_mod_rs_hierarchy_out_dir_mod_name: String, + ) -> Self { + self.gen_mod_rs_hierarchy_out_dir_mod_name = Some(gen_mod_rs_hierarchy_out_dir_mod_name); + self + } + /// Generate code bundled in protobuf crate. Regular users don't need this option. pub fn inside_protobuf(mut self, inside_protobuf: bool) -> Self { self.inside_protobuf = Some(inside_protobuf); self } + pub fn log_file(mut self, log_file: String) -> Self { + self.log_file = Some(log_file); + self + } + /// Update fields of self with fields defined in other customize pub fn update_with(&mut self, that: &Customize) { if let Some(v) = &that.before { @@ -194,9 +217,15 @@ impl Customize { if let Some(v) = that.gen_mod_rs { self.gen_mod_rs = Some(v); } + if let Some(v) = &that.gen_mod_rs_hierarchy_out_dir_mod_name { + self.gen_mod_rs_hierarchy_out_dir_mod_name = Some(v.to_owned()); + } if let Some(v) = that.inside_protobuf { self.inside_protobuf = Some(v); } + if let Some(v) = &that.log_file { + self.log_file = Some(v.to_owned()); + } } /// Update unset fields of self with fields from other customize @@ -236,12 +265,16 @@ impl Customize { r.lite_runtime = Some(parse_bool(v)?); } else if n == "gen_mod_rs" { r.gen_mod_rs = Some(parse_bool(v)?); + } else if n == "gen_mod_rs_hierarchy_out_dir_mod_name" { + r.gen_mod_rs_hierarchy_out_dir_mod_name = Some(v.to_owned()); } else if n == "inside_protobuf" { r.inside_protobuf = Some(parse_bool(v)?); } else if n == "lite" { // Support Java and C++ protoc plugin syntax: // https://github.com/protocolbuffers/protobuf/issues/6489 r.lite_runtime = Some(parse_bool(v)?); + } else if n == "log_file" { + r.log_file = Some(v.to_owned()); } else { return Err(CustomizeParseParameterError::UnknownOptionName(n.to_owned()).into()); } diff --git a/protobuf-codegen/src/customize/rustproto_proto.rs b/protobuf-codegen/src/customize/rustproto_proto.rs index 3d9a77b2f..2be3d4914 100644 --- a/protobuf-codegen/src/customize/rustproto_proto.rs +++ b/protobuf-codegen/src/customize/rustproto_proto.rs @@ -14,7 +14,9 @@ pub(crate) fn customize_from_rustproto_for_message(source: &MessageOptions) -> C let tokio_bytes_for_string = rustproto::exts::tokio_bytes_for_string.get(source); let lite_runtime = None; let gen_mod_rs = None; + let gen_mod_rs_hierarchy_out_dir_mod_name = None; let inside_protobuf = None; + let log_file = None; Customize { before, generate_accessors, @@ -23,7 +25,9 @@ pub(crate) fn customize_from_rustproto_for_message(source: &MessageOptions) -> C tokio_bytes_for_string, lite_runtime, gen_mod_rs, + gen_mod_rs_hierarchy_out_dir_mod_name, inside_protobuf, + log_file, } } @@ -39,7 +43,9 @@ pub(crate) fn customize_from_rustproto_for_field(source: &FieldOptions) -> Custo let tokio_bytes_for_string = rustproto::exts::tokio_bytes_for_string_field.get(source); let lite_runtime = None; let gen_mod_rs = None; + let gen_mod_rs_hierarchy_out_dir_mod_name = None; let inside_protobuf = None; + let log_file = None; Customize { before, generate_accessors, @@ -48,7 +54,9 @@ pub(crate) fn customize_from_rustproto_for_field(source: &FieldOptions) -> Custo tokio_bytes_for_string, lite_runtime, gen_mod_rs, + gen_mod_rs_hierarchy_out_dir_mod_name, inside_protobuf, + log_file, } } @@ -60,7 +68,9 @@ pub(crate) fn customize_from_rustproto_for_file(source: &FileOptions) -> Customi let tokio_bytes_for_string = rustproto::exts::tokio_bytes_for_string_all.get(source); let lite_runtime = rustproto::exts::lite_runtime_all.get(source); let gen_mod_rs = None; + let gen_mod_rs_hierarchy_out_dir_mod_name = None; let inside_protobuf = None; + let log_file = None; Customize { before, generate_accessors, @@ -70,5 +80,7 @@ pub(crate) fn customize_from_rustproto_for_file(source: &FileOptions) -> Customi lite_runtime, inside_protobuf, gen_mod_rs, + gen_mod_rs_hierarchy_out_dir_mod_name, + log_file, } } diff --git a/protobuf-codegen/src/gen/all.rs b/protobuf-codegen/src/gen/all.rs index 312b8dbe6..86fc545a7 100644 --- a/protobuf-codegen/src/gen/all.rs +++ b/protobuf-codegen/src/gen/all.rs @@ -1,9 +1,10 @@ -use std::collections::HashMap; - use protobuf::descriptor::FileDescriptorProto; use protobuf::reflect::FileDescriptor; use protobuf_parse::ProtoPath; use protobuf_parse::ProtoPathBuf; +use std::collections::HashMap; +use tracing::instrument; +use tracing::Level; use crate::compiler_plugin; use crate::customize::ctx::CustomizeElemCtx; @@ -14,6 +15,7 @@ use crate::gen::scope::RootScope; use crate::gen::well_known_types::gen_well_known_types_mod; use crate::Customize; +#[instrument(level = Level::INFO, skip_all)] pub(crate) fn gen_all( file_descriptors: &[FileDescriptorProto], parser: &str, diff --git a/protobuf-codegen/src/gen/file.rs b/protobuf-codegen/src/gen/file.rs index 119a8f1eb..79af21fda 100644 --- a/protobuf-codegen/src/gen/file.rs +++ b/protobuf-codegen/src/gen/file.rs @@ -4,6 +4,9 @@ use protobuf::descriptor::file_options; use protobuf::descriptor::FileDescriptorProto; use protobuf::reflect::FileDescriptor; use protobuf_parse::ProtoPath; +use tracing::event; +use tracing::instrument; +use tracing::Level; use crate::compiler_plugin; use crate::customize::ctx::CustomizeElemCtx; @@ -14,6 +17,7 @@ use crate::gen::extensions::write_extensions; use crate::gen::file_descriptor::write_file_descriptor_data; use crate::gen::inside::protobuf_crate_path; use crate::gen::message::MessageGen; +use crate::gen::paths::file_descriptor_to_hierarchical_rs; use crate::gen::paths::proto_path_to_rust_mod; use crate::gen::scope::FileScope; use crate::gen::scope::RootScope; @@ -24,6 +28,7 @@ pub(crate) struct GenFileResult { pub(crate) mod_name: String, } +#[instrument(level = Level::INFO, skip_all)] pub(crate) fn gen_file( file_descriptor: &FileDescriptor, _files_map: &HashMap<&ProtoPath, &FileDescriptor>, @@ -53,6 +58,17 @@ pub(crate) fn gen_file( let lite_runtime = customize.for_elem.lite_runtime.unwrap_or(false); + let mod_name = proto_path_to_rust_mod(file_descriptor.proto().name()).into_string(); + + event!( + Level::INFO, + file_descriptor.name = file_descriptor.proto().name(), + file_descriptor.package = file_descriptor.proto().package(), + lite_runtime, + mod_name, + "Generating file" + ); + let v = CodeWriter::with(|w| { w.write_generated_by("rust-protobuf", env!("CARGO_PKG_VERSION"), parser); @@ -141,9 +157,17 @@ pub(crate) fn gen_file( Ok(GenFileResult { compiler_plugin_result: compiler_plugin::GenResult { - name: proto_name_to_rs(file_descriptor.proto().name()), + name: if customize + .for_elem + .gen_mod_rs_hierarchy_out_dir_mod_name + .is_some() + { + file_descriptor_to_hierarchical_rs(file_descriptor.proto()) + } else { + proto_name_to_rs(file_descriptor.proto().name()) + }, content: v.into_bytes(), }, - mod_name: proto_path_to_rust_mod(file_descriptor.proto().name()).into_string(), + mod_name, }) } diff --git a/protobuf-codegen/src/gen/message.rs b/protobuf-codegen/src/gen/message.rs index bd8a1098a..aa72157a9 100644 --- a/protobuf-codegen/src/gen/message.rs +++ b/protobuf-codegen/src/gen/message.rs @@ -4,6 +4,8 @@ use protobuf::descriptor::*; use protobuf::reflect::FileDescriptor; use protobuf::reflect::MessageDescriptor; use protobuf_parse::snake_case; +use tracing::event; +use tracing::Level; use crate::customize::ctx::CustomizeElemCtx; use crate::customize::ctx::SpecialFieldPseudoDescriptor; @@ -657,6 +659,14 @@ impl<'a> MessageGen<'a> { } pub fn write(&self, w: &mut CodeWriter) -> anyhow::Result<()> { + event!( + Level::INFO, + proto_file.name = self.file_descriptor.proto().name(), + proto_file.package = self.file_descriptor.proto().package(), + message.name = self.message_descriptor.proto().name(), + "Generating message" + ); + w.all_documentation(self.info, self.path); self.write_struct(w); diff --git a/protobuf-codegen/src/gen/paths.rs b/protobuf-codegen/src/gen/paths.rs index 2bc958d0f..8910f96e5 100644 --- a/protobuf-codegen/src/gen/paths.rs +++ b/protobuf-codegen/src/gen/paths.rs @@ -1,3 +1,6 @@ +use protobuf::descriptor::FileDescriptorProto; +use tracing::{instrument, Level}; + use crate::gen::inside::protobuf_crate_path; use crate::gen::rust::ident::RustIdent; use crate::gen::rust::path::RustPath; @@ -40,10 +43,24 @@ pub(crate) fn proto_path_to_rust_mod(path: &str) -> RustIdent { } /// Used in protobuf-codegen-identical-test -pub fn proto_name_to_rs(proto_file_path: &str) -> String { - format!("{}.rs", proto_path_to_rust_mod(proto_file_path)) +pub fn proto_name_to_rs(proto_name: &str) -> String { + format!("{}.rs", proto_path_to_rust_mod(proto_name)) } +/// Determines the one-for-one relative path in which a Rust source file +/// should be found for the provided `file_descriptor`. +pub fn file_descriptor_to_hierarchical_rs(file_descriptor: &FileDescriptorProto) -> String { + match &file_descriptor.package { + Some(package) => format!( + "{}/{}", + package.replace('.', "/"), + proto_name_to_rs(file_descriptor.name()) + ), + None => proto_name_to_rs(file_descriptor.name()), + } +} + +#[instrument(level = Level::DEBUG, skip(customize), ret(Display))] pub(crate) fn proto_path_to_fn_file_descriptor( proto_path: &str, customize: &Customize, @@ -58,9 +75,22 @@ pub(crate) fn proto_path_to_fn_file_descriptor( .append_ident("well_known_types".into()) .append_ident(proto_path_to_rust_mod(s)) .append_ident("file_descriptor".into()), - s => RustPath::super_path() - .append_ident(proto_path_to_rust_mod(s)) - .append_ident("file_descriptor".into()), + s => { + if let Some(mod_path) = &customize.gen_mod_rs_hierarchy_out_dir_mod_name { + let mut rust_path = RustPath::from("crate"); + for mod_part in mod_path.split("::") { + rust_path = rust_path.append_ident(RustIdent::from(mod_part)); + } + for component in proto_path.split("/").filter(|p| !p.ends_with(".proto")) { + rust_path = rust_path.append_ident(RustIdent::from(component)); + } + rust_path.append_ident("file_descriptor".into()) + } else { + RustPath::super_path() + .append_ident(proto_path_to_rust_mod(s)) + .append_ident("file_descriptor".into()) + } + } } } diff --git a/protobuf-codegen/src/gen/rust_types_values.rs b/protobuf-codegen/src/gen/rust_types_values.rs index cd67f2b35..1608cb8b5 100644 --- a/protobuf-codegen/src/gen/rust_types_values.rs +++ b/protobuf-codegen/src/gen/rust_types_values.rs @@ -5,6 +5,8 @@ use protobuf::descriptor::*; use protobuf::reflect::FileDescriptor; use protobuf_parse::ProtobufAbsPath; use regex::Regex; +use tracing::event; +use tracing::{instrument, Level}; use crate::customize::Customize; use crate::gen::field::type_ext::TypeExt; @@ -81,14 +83,15 @@ impl RustType { RustType::Option(ref param) => { format!("::std::option::Option<{}>", param.to_code(customize)) } - RustType::MessageField(ref param) => format!( - "{}::MessageField<{}>", - protobuf_crate_path(customize), - param.to_code(customize) - ), + RustType::MessageField(ref param) => { + let crate_path = protobuf_crate_path(customize); + format!("{}::MessageField<{}>", crate_path, param.to_code(customize)) + } RustType::Uniq(ref param) => format!("::std::boxed::Box<{}>", param.to_code(customize)), RustType::Ref(ref param) => format!("&{}", param.to_code(customize)), - RustType::Message(ref name) => format!("{}", name), + RustType::Message(ref name) => { + format!("{}", name) + } RustType::Enum(ref name, ..) | RustType::Oneof(ref name) => format!("{}", name), RustType::EnumOrUnknown(ref name, ..) => format!( "{}::EnumOrUnknown<{}>", @@ -482,6 +485,7 @@ pub(crate) fn make_path(source: &RustRelativePath, dest: &RustIdentWithPath) -> make_path_to_path(source, &dest.path).with_ident(dest.ident.clone()) } +#[instrument(level = Level::DEBUG, skip_all, fields(current.file = current.file, current.relative_mod = current.relative_mod.to_string()), ret(Display))] pub(crate) fn message_or_enum_to_rust_relative( message_or_enum: &dyn WithScope, current: &FileAndMod, @@ -489,8 +493,20 @@ pub(crate) fn message_or_enum_to_rust_relative( let same_file = message_or_enum.file_descriptor().name() == current.file; if same_file { // field type is a message or enum declared in the same file + event!( + Level::DEBUG, + current.file = current.file, + current.relative_mod = current.relative_mod.to_string(), + "Field type is message or enum in same file" + ); make_path(¤t.relative_mod, &message_or_enum.rust_name_to_file()) } else if let Some(name) = is_well_known_type_full(&message_or_enum.name_absolute()) { + event!( + Level::DEBUG, + current.file = current.file, + current.relative_mod = current.relative_mod.to_string(), + "Field type is a well-known type included in the rust-protobuf library" + ); // Well-known types are included in rust-protobuf library // https://developers.google.com/protocol-buffers/docs/reference/google.protobuf let file_descriptor = message_or_enum.file_descriptor(); @@ -506,13 +522,43 @@ pub(crate) fn message_or_enum_to_rust_relative( protobuf_crate = protobuf_crate_path(¤t.customize), )) } else if is_descriptor_proto(&message_or_enum.file_descriptor()) { + event!( + Level::DEBUG, + current.file = current.file, + current.relative_mod = current.relative_mod.to_string(), + "Field type is a descriptor proto" + ); // Messages defined in descriptor.proto RustIdentWithPath::from(format!( "{}::descriptor::{}", protobuf_crate_path(¤t.customize), message_or_enum.rust_name_to_file() )) + } else if let Some(mod_name) = ¤t.customize.gen_mod_rs_hierarchy_out_dir_mod_name { + event!( + Level::DEBUG, + current.file = current.file, + current.relative_mod = current.relative_mod.to_string(), + root_mod_name = mod_name, + "Field type identifier needs to be resolved in the output dir hierarchy" + ); + let mut rust_name = format!("crate::{}", mod_name.to_owned()); + for component in message_or_enum + .name_absolute() + .trim_start_matches(':') + .split('.') + .filter(|s| !s.is_empty()) + { + rust_name.push_str(&format!("::{}", component.replace('-', "_"))); + } + RustIdentWithPath::from(rust_name) } else { + event!( + Level::DEBUG, + current.file = current.file, + current.relative_mod = current.relative_mod.to_string(), + "Field type identifier is relative to current mod" + ); current .relative_mod .to_reverse() diff --git a/protobuf-codegen/src/gen_and_write.rs b/protobuf-codegen/src/gen_and_write.rs index 95d621eab..f482b77af 100644 --- a/protobuf-codegen/src/gen_and_write.rs +++ b/protobuf-codegen/src/gen_and_write.rs @@ -1,6 +1,7 @@ #![doc(hidden)] use std::fs; +use std::fs::create_dir_all; use std::io; use std::path::Path; @@ -9,6 +10,7 @@ use protobuf_parse::ProtoPathBuf; use crate::customize::CustomizeCallback; use crate::gen::all::gen_all; +use crate::logging::init_logging; use crate::Customize; #[derive(Debug, thiserror::Error)] @@ -19,6 +21,8 @@ enum Error { OutputDoesNotExistOrNotAccssible(String, #[source] io::Error), #[error("failed to create file `{0}`: {1}")] FailedToWriteFile(String, #[source] io::Error), + #[error("failed to create directory hierarchy for path `{0}`")] + FailedToCreateDirectoryHierarchy(String, #[source] io::Error), } #[doc(hidden)] @@ -43,6 +47,8 @@ pub fn gen_and_write( } } + init_logging(&customize)?; + let results = gen_all( file_descriptors, parser, @@ -54,6 +60,13 @@ pub fn gen_and_write( for r in &results { let mut file_path = out_dir.to_owned(); file_path.push(&r.name); + if customize.gen_mod_rs_hierarchy_out_dir_mod_name.is_some() { + if let Some(parent) = file_path.parent() { + create_dir_all(parent).map_err(|error| { + Error::FailedToCreateDirectoryHierarchy(parent.display().to_string(), error) + })?; + } + } fs::write(&file_path, r.content.as_slice()) .map_err(|e| Error::FailedToWriteFile(file_path.display().to_string(), e))?; } diff --git a/protobuf-codegen/src/lib.rs b/protobuf-codegen/src/lib.rs index 0471d7c2d..0470f93e1 100644 --- a/protobuf-codegen/src/lib.rs +++ b/protobuf-codegen/src/lib.rs @@ -136,6 +136,7 @@ mod compiler_plugin; mod customize; mod gen; pub mod gen_and_write; +mod logging; pub mod protoc_gen_rust; pub use codegen::Codegen; diff --git a/protobuf-codegen/src/logging.rs b/protobuf-codegen/src/logging.rs new file mode 100644 index 000000000..409a433e8 --- /dev/null +++ b/protobuf-codegen/src/logging.rs @@ -0,0 +1,52 @@ +use std::path::PathBuf; + +use anyhow::anyhow; + +use tracing::{event, Level}; +use tracing_subscriber::filter::LevelFilter; +use tracing_subscriber::fmt::format::FmtSpan; +use tracing_subscriber::{prelude::*, EnvFilter}; + +use crate::Customize; + +pub(crate) fn init_logging(customize: &Customize) -> anyhow::Result<()> { + if let Some(log_file) = &customize.log_file { + let file_name = PathBuf::from(log_file); + let file_appender = tracing_appender::rolling::never( + file_name.parent().ok_or_else(|| { + anyhow!( + "Specified log_file did not have a parent directory: {}", + file_name.display().to_string() + ) + })?, + file_name.file_name().ok_or_else(|| { + anyhow!( + "Specified log_file was invalid: {}", + file_name.display().to_string() + ) + })?, + ); + let fmt = tracing_subscriber::fmt::layer() + .with_level(true) + .with_target(true) + .with_ansi(false) + .with_writer(file_appender) + .with_span_events(FmtSpan::ENTER) + .with_span_events(FmtSpan::EXIT) + .json(); + tracing_subscriber::registry() + .with(fmt) + .with( + EnvFilter::builder() + .with_default_directive(LevelFilter::INFO.into()) + .from_env_lossy(), + ) + .init(); + event!( + Level::INFO, + log_file = file_name.display().to_string(), + "Logging initialized" + ); + } + Ok(()) +} diff --git a/protobuf-codegen/src/protoc_gen_rust.rs b/protobuf-codegen/src/protoc_gen_rust.rs index 014db6a14..f4db1c1b2 100644 --- a/protobuf-codegen/src/protoc_gen_rust.rs +++ b/protobuf-codegen/src/protoc_gen_rust.rs @@ -3,12 +3,14 @@ use crate::compiler_plugin; use crate::customize::CustomizeCallbackDefault; use crate::gen::all::gen_all; +use crate::logging::init_logging; use crate::Customize; #[doc(hidden)] pub fn protoc_gen_rust_main() { compiler_plugin::plugin_main(|r| { let customize = Customize::parse_from_parameter(r.parameter).expect("parse options"); + init_logging(&customize)?; gen_all( r.file_descriptors, "protoc --rust-out=...",