From 92366e9599a09f8cafab1816688df342877a069d Mon Sep 17 00:00:00 2001 From: Kristopher Wuollett Date: Fri, 27 Oct 2023 12:34:44 -0500 Subject: [PATCH 1/5] style: harmonize proto_name_to_rs param naming --- protobuf-codegen/src/gen/paths.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/protobuf-codegen/src/gen/paths.rs b/protobuf-codegen/src/gen/paths.rs index 2bc958d0f..040dcc40b 100644 --- a/protobuf-codegen/src/gen/paths.rs +++ b/protobuf-codegen/src/gen/paths.rs @@ -40,8 +40,8 @@ pub(crate) fn proto_path_to_rust_mod(path: &str) -> RustIdent { } /// Used in protobuf-codegen-identical-test -pub fn proto_name_to_rs(proto_file_path: &str) -> String { - format!("{}.rs", proto_path_to_rust_mod(proto_file_path)) +pub fn proto_name_to_rs(proto_name: &str) -> String { + format!("{}.rs", proto_path_to_rust_mod(proto_name)) } pub(crate) fn proto_path_to_fn_file_descriptor( From be36e913c177653c901da1f63b13f4cb78638b3d Mon Sep 17 00:00:00 2001 From: Kristopher Wuollett Date: Fri, 27 Oct 2023 15:15:48 -0500 Subject: [PATCH 2/5] feat: add gen_mod_rs_hierarchy_out_dir_mod_name option --- protobuf-codegen/src/customize/mod.rs | 17 +++++++++++ .../src/customize/rustproto_proto.rs | 6 ++++ protobuf-codegen/src/gen/file.rs | 7 ++++- protobuf-codegen/src/gen/paths.rs | 30 +++++++++++++++++-- protobuf-codegen/src/gen/rust_types_values.rs | 11 +++++++ protobuf-codegen/src/gen_and_write.rs | 10 +++++++ 6 files changed, 78 insertions(+), 3 deletions(-) diff --git a/protobuf-codegen/src/customize/mod.rs b/protobuf-codegen/src/customize/mod.rs index 826bd3f6d..ad75d0f7f 100644 --- a/protobuf-codegen/src/customize/mod.rs +++ b/protobuf-codegen/src/customize/mod.rs @@ -106,6 +106,9 @@ pub struct Customize { /// /// This option will likely be on by default in rust-protobuf version 3. pub(crate) gen_mod_rs: Option, + /// Generate `mod.rs` in a hierarchy in the output directory assuming that + /// this name is the crate-relative Rust path to the output directory. + pub(crate) gen_mod_rs_hierarchy_out_dir_mod_name: Option, /// Used internally to generate protos bundled in protobuf crate /// like `descriptor.proto` pub(crate) inside_protobuf: Option, @@ -165,6 +168,15 @@ impl Customize { self } + /// Generate `mod.rs` with all the generated modules in a Rust mod + /// hierarchy. + /// + /// This option is on by default in rust-protobuf version 3. + pub fn gen_mod_rs_hierarchy_out_dir_mod_name(mut self, gen_mod_rs_hierarchy_out_dir_mod_name: String) -> Self { + self.gen_mod_rs_hierarchy_out_dir_mod_name = Some(gen_mod_rs_hierarchy_out_dir_mod_name); + self + } + /// Generate code bundled in protobuf crate. Regular users don't need this option. pub fn inside_protobuf(mut self, inside_protobuf: bool) -> Self { self.inside_protobuf = Some(inside_protobuf); @@ -194,6 +206,9 @@ impl Customize { if let Some(v) = that.gen_mod_rs { self.gen_mod_rs = Some(v); } + if let Some(v) = &that.gen_mod_rs_hierarchy_out_dir_mod_name { + self.gen_mod_rs_hierarchy_out_dir_mod_name = Some(v.to_owned()); + } if let Some(v) = that.inside_protobuf { self.inside_protobuf = Some(v); } @@ -236,6 +251,8 @@ impl Customize { r.lite_runtime = Some(parse_bool(v)?); } else if n == "gen_mod_rs" { r.gen_mod_rs = Some(parse_bool(v)?); + } else if n == "gen_mod_rs_hierarchy_out_dir_mod_name" { + r.gen_mod_rs_hierarchy_out_dir_mod_name = Some(v.to_owned()); } else if n == "inside_protobuf" { r.inside_protobuf = Some(parse_bool(v)?); } else if n == "lite" { diff --git a/protobuf-codegen/src/customize/rustproto_proto.rs b/protobuf-codegen/src/customize/rustproto_proto.rs index 3d9a77b2f..a615857d3 100644 --- a/protobuf-codegen/src/customize/rustproto_proto.rs +++ b/protobuf-codegen/src/customize/rustproto_proto.rs @@ -14,6 +14,7 @@ pub(crate) fn customize_from_rustproto_for_message(source: &MessageOptions) -> C let tokio_bytes_for_string = rustproto::exts::tokio_bytes_for_string.get(source); let lite_runtime = None; let gen_mod_rs = None; + let gen_mod_rs_hierarchy_out_dir_mod_name = None; let inside_protobuf = None; Customize { before, @@ -23,6 +24,7 @@ pub(crate) fn customize_from_rustproto_for_message(source: &MessageOptions) -> C tokio_bytes_for_string, lite_runtime, gen_mod_rs, + gen_mod_rs_hierarchy_out_dir_mod_name, inside_protobuf, } } @@ -39,6 +41,7 @@ pub(crate) fn customize_from_rustproto_for_field(source: &FieldOptions) -> Custo let tokio_bytes_for_string = rustproto::exts::tokio_bytes_for_string_field.get(source); let lite_runtime = None; let gen_mod_rs = None; + let gen_mod_rs_hierarchy_out_dir_mod_name = None; let inside_protobuf = None; Customize { before, @@ -48,6 +51,7 @@ pub(crate) fn customize_from_rustproto_for_field(source: &FieldOptions) -> Custo tokio_bytes_for_string, lite_runtime, gen_mod_rs, + gen_mod_rs_hierarchy_out_dir_mod_name, inside_protobuf, } } @@ -60,6 +64,7 @@ pub(crate) fn customize_from_rustproto_for_file(source: &FileOptions) -> Customi let tokio_bytes_for_string = rustproto::exts::tokio_bytes_for_string_all.get(source); let lite_runtime = rustproto::exts::lite_runtime_all.get(source); let gen_mod_rs = None; + let gen_mod_rs_hierarchy_out_dir_mod_name = None; let inside_protobuf = None; Customize { before, @@ -70,5 +75,6 @@ pub(crate) fn customize_from_rustproto_for_file(source: &FileOptions) -> Customi lite_runtime, inside_protobuf, gen_mod_rs, + gen_mod_rs_hierarchy_out_dir_mod_name, } } diff --git a/protobuf-codegen/src/gen/file.rs b/protobuf-codegen/src/gen/file.rs index 119a8f1eb..71570cb77 100644 --- a/protobuf-codegen/src/gen/file.rs +++ b/protobuf-codegen/src/gen/file.rs @@ -14,6 +14,7 @@ use crate::gen::extensions::write_extensions; use crate::gen::file_descriptor::write_file_descriptor_data; use crate::gen::inside::protobuf_crate_path; use crate::gen::message::MessageGen; +use crate::gen::paths::file_descriptor_to_hierarchical_rs; use crate::gen::paths::proto_path_to_rust_mod; use crate::gen::scope::FileScope; use crate::gen::scope::RootScope; @@ -141,7 +142,11 @@ pub(crate) fn gen_file( Ok(GenFileResult { compiler_plugin_result: compiler_plugin::GenResult { - name: proto_name_to_rs(file_descriptor.proto().name()), + name: if customize.for_elem.gen_mod_rs_hierarchy_out_dir_mod_name.is_some() { + file_descriptor_to_hierarchical_rs(file_descriptor.proto()) + } else { + proto_name_to_rs(file_descriptor.proto().name()) + }, content: v.into_bytes(), }, mod_name: proto_path_to_rust_mod(file_descriptor.proto().name()).into_string(), diff --git a/protobuf-codegen/src/gen/paths.rs b/protobuf-codegen/src/gen/paths.rs index 040dcc40b..d0914bc28 100644 --- a/protobuf-codegen/src/gen/paths.rs +++ b/protobuf-codegen/src/gen/paths.rs @@ -44,6 +44,19 @@ pub fn proto_name_to_rs(proto_name: &str) -> String { format!("{}.rs", proto_path_to_rust_mod(proto_name)) } +/// Determines the one-for-one relative path in which a Rust source file +/// should be found for the provided `file_descriptor`. +pub fn file_descriptor_to_hierarchical_rs(file_descriptor: &FileDescriptorProto) -> String { + match &file_descriptor.package { + Some(package) => format!( + "{}/{}", + package.replace('.', "/"), + proto_name_to_rs(file_descriptor.name()) + ), + None => proto_name_to_rs(file_descriptor.name()), + } +} + pub(crate) fn proto_path_to_fn_file_descriptor( proto_path: &str, customize: &Customize, @@ -58,9 +71,22 @@ pub(crate) fn proto_path_to_fn_file_descriptor( .append_ident("well_known_types".into()) .append_ident(proto_path_to_rust_mod(s)) .append_ident("file_descriptor".into()), - s => RustPath::super_path() + s => { + if let Some(mod_path) = &customize.gen_mod_rs_hierarchy_out_dir_mod_name { + let mut rust_path = RustPath::from("crate"); + for mod_part in mod_path.split("::") { + rust_path = rust_path.append_ident(RustIdent::from(mod_part)); + } + for component in proto_path.split("/").filter(|p| !p.ends_with(".proto")) { + rust_path = rust_path.append_ident(RustIdent::from(component)); + } + rust_path.append_ident("file_descriptor".into()) + } else { + RustPath::super_path() .append_ident(proto_path_to_rust_mod(s)) - .append_ident("file_descriptor".into()), + .append_ident("file_descriptor".into()) + } + } } } diff --git a/protobuf-codegen/src/gen/rust_types_values.rs b/protobuf-codegen/src/gen/rust_types_values.rs index cd67f2b35..d5e6de731 100644 --- a/protobuf-codegen/src/gen/rust_types_values.rs +++ b/protobuf-codegen/src/gen/rust_types_values.rs @@ -512,6 +512,17 @@ pub(crate) fn message_or_enum_to_rust_relative( protobuf_crate_path(¤t.customize), message_or_enum.rust_name_to_file() )) + } else if let Some(mod_name) = ¤t.customize.gen_mod_rs_hierarchy_out_dir_mod_name { + let mut rust_name = format!("crate::{}", mod_name.to_owned()); + for component in message_or_enum + .name_absolute() + .trim_start_matches(':') + .split('.') + .filter(|s| !s.is_empty()) + { + rust_name.push_str(&format!("::{}", component.replace('-', "_"))); + } + RustIdentWithPath::from(rust_name) } else { current .relative_mod diff --git a/protobuf-codegen/src/gen_and_write.rs b/protobuf-codegen/src/gen_and_write.rs index 95d621eab..1d0c7299d 100644 --- a/protobuf-codegen/src/gen_and_write.rs +++ b/protobuf-codegen/src/gen_and_write.rs @@ -1,6 +1,7 @@ #![doc(hidden)] use std::fs; +use std::fs::create_dir_all; use std::io; use std::path::Path; @@ -19,6 +20,8 @@ enum Error { OutputDoesNotExistOrNotAccssible(String, #[source] io::Error), #[error("failed to create file `{0}`: {1}")] FailedToWriteFile(String, #[source] io::Error), + #[error("failed to create directory hierarchy for path `{0}`")] + FailedToCreateDirectoryHierarchy(String, #[source] io::Error), } #[doc(hidden)] @@ -54,6 +57,13 @@ pub fn gen_and_write( for r in &results { let mut file_path = out_dir.to_owned(); file_path.push(&r.name); + if customize.gen_mod_rs_hierarchy_out_dir_mod_name.is_some() { + if let Some(parent) = file_path.parent() { + create_dir_all(parent).map_err(|error| { + Error::FailedToCreateDirectoryHierarchy(parent.display().to_string(), error) + })?; + } + } fs::write(&file_path, r.content.as_slice()) .map_err(|e| Error::FailedToWriteFile(file_path.display().to_string(), e))?; } From 81fff73d250e68f54001cefb24631c709267f5ac Mon Sep 17 00:00:00 2001 From: Kristopher Wuollett Date: Fri, 27 Oct 2023 15:16:18 -0500 Subject: [PATCH 3/5] DO NOT SUBMIT: temporary logging --- protobuf-codegen/src/gen/enums.rs | 3 + protobuf-codegen/src/gen/field/elem.rs | 2 + protobuf-codegen/src/gen/field/mod.rs | 8 +++ protobuf-codegen/src/gen/oneof.rs | 2 + protobuf-codegen/src/gen/paths.rs | 7 ++- protobuf-codegen/src/gen/rust_types_values.rs | 62 +++++++++++++++++-- protobuf-codegen/src/gen/scope.rs | 5 ++ 7 files changed, 82 insertions(+), 7 deletions(-) diff --git a/protobuf-codegen/src/gen/enums.rs b/protobuf-codegen/src/gen/enums.rs index e6d891d25..a17af9b6b 100644 --- a/protobuf-codegen/src/gen/enums.rs +++ b/protobuf-codegen/src/gen/enums.rs @@ -70,6 +70,9 @@ impl<'a> EnumGen<'a> { path: &'a [i32], info: Option<&'a SourceCodeInfo>, ) -> EnumGen<'a> { + let enum_proto = enum_with_scope.en.proto(); + eprintln!("EnumGen::parse enum.name: {}", enum_proto.name()); + let customize = customize.child( &customize_from_rustproto_for_enum(enum_with_scope.en.proto().options.get_or_default()), &enum_with_scope.en, diff --git a/protobuf-codegen/src/gen/field/elem.rs b/protobuf-codegen/src/gen/field/elem.rs index fd06b4ced..ab6d899d9 100644 --- a/protobuf-codegen/src/gen/field/elem.rs +++ b/protobuf-codegen/src/gen/field/elem.rs @@ -65,6 +65,8 @@ impl<'a> FieldElemMessage<'a> { } fn rust_type(&self, reference: &FileAndMod) -> RustType { + eprintln!("FieldElemMessage::rust_type message.mod_name: {}", self.message.mod_name().to_string()); + eprintln!("FieldElemMessage::rust_type message.name: {}", self.message.message.name()); RustType::Message(self.rust_name_relative(reference)) } } diff --git a/protobuf-codegen/src/gen/field/mod.rs b/protobuf-codegen/src/gen/field/mod.rs index d6a5e44a4..4fc908409 100644 --- a/protobuf-codegen/src/gen/field/mod.rs +++ b/protobuf-codegen/src/gen/field/mod.rs @@ -145,6 +145,12 @@ impl<'a> FieldGen<'a> { path: Vec, info: Option<&'a SourceCodeInfo>, ) -> anyhow::Result> { + let message_proto = field.message.message.proto(); + eprintln!("FieldGen::parse message.name: {}", message_proto.name()); + eprintln!("FieldGen::parse field.name: {}", field.field.proto().name()); + eprintln!("FieldGen::parse field.type.protobuf_name: {}", field.field.proto().type_().protobuf_name()); + eprintln!("FieldGen::parse field.type_name: {}", field.field.proto().type_name()); + let customize = parent_customize .child( &customize_from_rustproto_for_field(field.field.proto().options.get_or_default()), @@ -336,6 +342,8 @@ impl<'a> FieldGen<'a> { // for field `foo`, return type of `fn foo(..)` fn getter_return_type(&self) -> RustType { + eprintln!("FieldGen::getter_return_type"); + let reference = self .proto_field .message diff --git a/protobuf-codegen/src/gen/oneof.rs b/protobuf-codegen/src/gen/oneof.rs index 3f34cdccf..9f5046bf6 100644 --- a/protobuf-codegen/src/gen/oneof.rs +++ b/protobuf-codegen/src/gen/oneof.rs @@ -176,6 +176,8 @@ impl<'a> OneofGen<'a> { oneof: OneofWithContext<'a>, parent_customize: &CustomizeElemCtx<'a>, ) -> OneofGen<'a> { + let message_proto = message.message.message.proto(); + eprintln!("OneofGen::parse message.name: {}", message_proto.name()); let customize = parent_customize.child(&Customize::default(), &oneof.oneof); let lite_runtime = customize.for_elem.lite_runtime.unwrap_or_else(|| { oneof diff --git a/protobuf-codegen/src/gen/paths.rs b/protobuf-codegen/src/gen/paths.rs index d0914bc28..f562cbaf7 100644 --- a/protobuf-codegen/src/gen/paths.rs +++ b/protobuf-codegen/src/gen/paths.rs @@ -1,3 +1,5 @@ +use protobuf::descriptor::FileDescriptorProto; + use crate::gen::inside::protobuf_crate_path; use crate::gen::rust::ident::RustIdent; use crate::gen::rust::path::RustPath; @@ -74,16 +76,19 @@ pub(crate) fn proto_path_to_fn_file_descriptor( s => { if let Some(mod_path) = &customize.gen_mod_rs_hierarchy_out_dir_mod_name { let mut rust_path = RustPath::from("crate"); + eprintln!("Forming path wth rust_path: {}", rust_path); for mod_part in mod_path.split("::") { rust_path = rust_path.append_ident(RustIdent::from(mod_part)); + eprintln!("Forming path wth rust_path: {}", rust_path); } for component in proto_path.split("/").filter(|p| !p.ends_with(".proto")) { rust_path = rust_path.append_ident(RustIdent::from(component)); + eprintln!("Forming path wth rust_path: {}", rust_path); } rust_path.append_ident("file_descriptor".into()) } else { RustPath::super_path() - .append_ident(proto_path_to_rust_mod(s)) + .append_ident(proto_path_to_rust_mod(s)) .append_ident("file_descriptor".into()) } } diff --git a/protobuf-codegen/src/gen/rust_types_values.rs b/protobuf-codegen/src/gen/rust_types_values.rs index d5e6de731..cc903a3a9 100644 --- a/protobuf-codegen/src/gen/rust_types_values.rs +++ b/protobuf-codegen/src/gen/rust_types_values.rs @@ -81,14 +81,19 @@ impl RustType { RustType::Option(ref param) => { format!("::std::option::Option<{}>", param.to_code(customize)) } - RustType::MessageField(ref param) => format!( - "{}::MessageField<{}>", - protobuf_crate_path(customize), - param.to_code(customize) - ), + RustType::MessageField(ref param) => { + let crate_path = protobuf_crate_path(customize); + eprintln!("Writing MessageField"); + format!("{}::MessageField<{}>", crate_path, param.to_code(customize)) + } RustType::Uniq(ref param) => format!("::std::boxed::Box<{}>", param.to_code(customize)), RustType::Ref(ref param) => format!("&{}", param.to_code(customize)), - RustType::Message(ref name) => format!("{}", name), + RustType::Message(ref name) => { + eprintln!("Writing Message name: {}", name); + eprintln!("Writing Message rust.ident: {}", name.0.ident); + eprintln!("Writing Message rust.path: {}", name.0.path); + format!("{}", name) + } RustType::Enum(ref name, ..) | RustType::Oneof(ref name) => format!("{}", name), RustType::EnumOrUnknown(ref name, ..) => format!( "{}::EnumOrUnknown<{}>", @@ -486,6 +491,7 @@ pub(crate) fn message_or_enum_to_rust_relative( message_or_enum: &dyn WithScope, current: &FileAndMod, ) -> RustIdentWithPath { + eprintln!("message_or_enum_to_rust_relative"); let same_file = message_or_enum.file_descriptor().name() == current.file; if same_file { // field type is a message or enum declared in the same file @@ -514,16 +520,60 @@ pub(crate) fn message_or_enum_to_rust_relative( )) } else if let Some(mod_name) = ¤t.customize.gen_mod_rs_hierarchy_out_dir_mod_name { let mut rust_name = format!("crate::{}", mod_name.to_owned()); + eprintln!( + "message_or_enum_to_rust_relative MY_WAY mod_name: {}", + rust_name + ); for component in message_or_enum .name_absolute() .trim_start_matches(':') .split('.') .filter(|s| !s.is_empty()) { + eprintln!( + "message_or_enum_to_rust_relative MY_WAY component: {}", + component + ); rust_name.push_str(&format!("::{}", component.replace('-', "_"))); } + eprintln!( + "message_or_enum_to_rust_relative MY_WAY rust_name: {}", + rust_name + ); RustIdentWithPath::from(rust_name) } else { + eprintln!( + "message_or_enum_to_rust_relative current.file: {}", + current.file + ); + eprintln!( + "message_or_enum_to_rust_relative current.relative_mod: {}", + current.relative_mod + ); + eprintln!( + "message_or_enum_to_rust_relative message_or_enum.name_absolute: {}", + message_or_enum.name_absolute() + ); + eprintln!( + "message_or_enum_to_rust_relative message_or_enum.name_to_package: {}", + message_or_enum.name_to_package() + ); + eprintln!( + "message_or_enum_to_rust_relative message_or_enum.protobuf_name_to_package: {}", + message_or_enum.protobuf_name_to_package() + ); + eprintln!( + "message_or_enum_to_rust_relative message_or_enum.rust_name: {}", + message_or_enum.rust_name() + ); + eprintln!( + "message_or_enum_to_rust_relative message_or_enum.rust_name_to_file: {}", + message_or_enum.rust_name_to_file() + ); + eprintln!( + "message_or_enum_to_rust_relative message_or_enum.rust_name_with_file: {}", + message_or_enum.rust_name_with_file() + ); current .relative_mod .to_reverse() diff --git a/protobuf-codegen/src/gen/scope.rs b/protobuf-codegen/src/gen/scope.rs index f70d233f1..58d46b9f3 100644 --- a/protobuf-codegen/src/gen/scope.rs +++ b/protobuf-codegen/src/gen/scope.rs @@ -277,6 +277,11 @@ impl<'a> Scope<'a> { } pub fn file_and_mod(&self, customize: Customize) -> FileAndMod { + let d = self.file_scope.file_descriptor.proto(); + eprintln!("Scope::file_and_mod package: package: {}", d.package()); + eprintln!("Scope::file_and_mod package: name: {}", d.name()); + eprintln!("Scope::file_and_mod package: rust_path_to_file: {}", self.rust_path_to_file()); + eprintln!("Scope::file_and_mod package: path_str: {}", self.path_str()); FileAndMod { file: self.file_scope.file_descriptor.proto().name().to_owned(), relative_mod: self.rust_path_to_file(), From 225a6badfe6d62b86aa6a68d0422009d203a4892 Mon Sep 17 00:00:00 2001 From: Kristopher Wuollett Date: Sat, 28 Oct 2023 07:31:16 -0500 Subject: [PATCH 4/5] fixup! feat: add gen_mod_rs_hierarchy_out_dir_mod_name option --- protobuf-codegen/src/customize/mod.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/protobuf-codegen/src/customize/mod.rs b/protobuf-codegen/src/customize/mod.rs index ad75d0f7f..0c182ad22 100644 --- a/protobuf-codegen/src/customize/mod.rs +++ b/protobuf-codegen/src/customize/mod.rs @@ -172,7 +172,10 @@ impl Customize { /// hierarchy. /// /// This option is on by default in rust-protobuf version 3. - pub fn gen_mod_rs_hierarchy_out_dir_mod_name(mut self, gen_mod_rs_hierarchy_out_dir_mod_name: String) -> Self { + pub fn gen_mod_rs_hierarchy_out_dir_mod_name( + mut self, + gen_mod_rs_hierarchy_out_dir_mod_name: String, + ) -> Self { self.gen_mod_rs_hierarchy_out_dir_mod_name = Some(gen_mod_rs_hierarchy_out_dir_mod_name); self } From 0d45c02008484180af767d01c7c8e6fb99790606 Mon Sep 17 00:00:00 2001 From: Kristopher Wuollett Date: Sat, 28 Oct 2023 07:35:38 -0500 Subject: [PATCH 5/5] fixup! DO NOT SUBMIT: temporary logging --- protobuf-codegen/Cargo.toml | 4 + protobuf-codegen/src/customize/mod.rs | 13 +++ .../src/customize/rustproto_proto.rs | 6 ++ protobuf-codegen/src/gen/all.rs | 6 +- protobuf-codegen/src/gen/enums.rs | 3 - protobuf-codegen/src/gen/field/elem.rs | 2 - protobuf-codegen/src/gen/field/mod.rs | 8 -- protobuf-codegen/src/gen/file.rs | 23 +++++- protobuf-codegen/src/gen/message.rs | 10 +++ protobuf-codegen/src/gen/oneof.rs | 2 - protobuf-codegen/src/gen/paths.rs | 5 +- protobuf-codegen/src/gen/rust_types_values.rs | 81 ++++++++----------- protobuf-codegen/src/gen/scope.rs | 5 -- protobuf-codegen/src/gen_and_write.rs | 3 + protobuf-codegen/src/lib.rs | 1 + protobuf-codegen/src/logging.rs | 52 ++++++++++++ protobuf-codegen/src/protoc_gen_rust.rs | 2 + 17 files changed, 151 insertions(+), 75 deletions(-) create mode 100644 protobuf-codegen/src/logging.rs diff --git a/protobuf-codegen/Cargo.toml b/protobuf-codegen/Cargo.toml index 00dabb3ac..8576f5246 100644 --- a/protobuf-codegen/Cargo.toml +++ b/protobuf-codegen/Cargo.toml @@ -22,6 +22,10 @@ regex = "1.5.5" once_cell = "1.10.0" tempfile = "3" +tracing = "0.1.40" +tracing-appender = "0.2.2" +tracing-subscriber = { version = "0.3.17", features = ["env-filter", "fmt", "json"] } + protobuf = { path = "../protobuf", version = "=4.0.0-alpha.0" } protobuf-parse = { path = "../protobuf-parse", version = "=4.0.0-alpha.0" } diff --git a/protobuf-codegen/src/customize/mod.rs b/protobuf-codegen/src/customize/mod.rs index 0c182ad22..277a4bcca 100644 --- a/protobuf-codegen/src/customize/mod.rs +++ b/protobuf-codegen/src/customize/mod.rs @@ -112,6 +112,9 @@ pub struct Customize { /// Used internally to generate protos bundled in protobuf crate /// like `descriptor.proto` pub(crate) inside_protobuf: Option, + /// Enable logging to the specified `log_file` for debugging. The logging + /// verbosity is controlled with the `RUST_LOG` environment variable. + pub(crate) log_file: Option, } #[derive(Debug, thiserror::Error)] @@ -186,6 +189,11 @@ impl Customize { self } + pub fn log_file(mut self, log_file: String) -> Self { + self.log_file = Some(log_file); + self + } + /// Update fields of self with fields defined in other customize pub fn update_with(&mut self, that: &Customize) { if let Some(v) = &that.before { @@ -215,6 +223,9 @@ impl Customize { if let Some(v) = that.inside_protobuf { self.inside_protobuf = Some(v); } + if let Some(v) = &that.log_file { + self.log_file = Some(v.to_owned()); + } } /// Update unset fields of self with fields from other customize @@ -262,6 +273,8 @@ impl Customize { // Support Java and C++ protoc plugin syntax: // https://github.com/protocolbuffers/protobuf/issues/6489 r.lite_runtime = Some(parse_bool(v)?); + } else if n == "log_file" { + r.log_file = Some(v.to_owned()); } else { return Err(CustomizeParseParameterError::UnknownOptionName(n.to_owned()).into()); } diff --git a/protobuf-codegen/src/customize/rustproto_proto.rs b/protobuf-codegen/src/customize/rustproto_proto.rs index a615857d3..2be3d4914 100644 --- a/protobuf-codegen/src/customize/rustproto_proto.rs +++ b/protobuf-codegen/src/customize/rustproto_proto.rs @@ -16,6 +16,7 @@ pub(crate) fn customize_from_rustproto_for_message(source: &MessageOptions) -> C let gen_mod_rs = None; let gen_mod_rs_hierarchy_out_dir_mod_name = None; let inside_protobuf = None; + let log_file = None; Customize { before, generate_accessors, @@ -26,6 +27,7 @@ pub(crate) fn customize_from_rustproto_for_message(source: &MessageOptions) -> C gen_mod_rs, gen_mod_rs_hierarchy_out_dir_mod_name, inside_protobuf, + log_file, } } @@ -43,6 +45,7 @@ pub(crate) fn customize_from_rustproto_for_field(source: &FieldOptions) -> Custo let gen_mod_rs = None; let gen_mod_rs_hierarchy_out_dir_mod_name = None; let inside_protobuf = None; + let log_file = None; Customize { before, generate_accessors, @@ -53,6 +56,7 @@ pub(crate) fn customize_from_rustproto_for_field(source: &FieldOptions) -> Custo gen_mod_rs, gen_mod_rs_hierarchy_out_dir_mod_name, inside_protobuf, + log_file, } } @@ -66,6 +70,7 @@ pub(crate) fn customize_from_rustproto_for_file(source: &FileOptions) -> Customi let gen_mod_rs = None; let gen_mod_rs_hierarchy_out_dir_mod_name = None; let inside_protobuf = None; + let log_file = None; Customize { before, generate_accessors, @@ -76,5 +81,6 @@ pub(crate) fn customize_from_rustproto_for_file(source: &FileOptions) -> Customi inside_protobuf, gen_mod_rs, gen_mod_rs_hierarchy_out_dir_mod_name, + log_file, } } diff --git a/protobuf-codegen/src/gen/all.rs b/protobuf-codegen/src/gen/all.rs index 312b8dbe6..86fc545a7 100644 --- a/protobuf-codegen/src/gen/all.rs +++ b/protobuf-codegen/src/gen/all.rs @@ -1,9 +1,10 @@ -use std::collections::HashMap; - use protobuf::descriptor::FileDescriptorProto; use protobuf::reflect::FileDescriptor; use protobuf_parse::ProtoPath; use protobuf_parse::ProtoPathBuf; +use std::collections::HashMap; +use tracing::instrument; +use tracing::Level; use crate::compiler_plugin; use crate::customize::ctx::CustomizeElemCtx; @@ -14,6 +15,7 @@ use crate::gen::scope::RootScope; use crate::gen::well_known_types::gen_well_known_types_mod; use crate::Customize; +#[instrument(level = Level::INFO, skip_all)] pub(crate) fn gen_all( file_descriptors: &[FileDescriptorProto], parser: &str, diff --git a/protobuf-codegen/src/gen/enums.rs b/protobuf-codegen/src/gen/enums.rs index a17af9b6b..e6d891d25 100644 --- a/protobuf-codegen/src/gen/enums.rs +++ b/protobuf-codegen/src/gen/enums.rs @@ -70,9 +70,6 @@ impl<'a> EnumGen<'a> { path: &'a [i32], info: Option<&'a SourceCodeInfo>, ) -> EnumGen<'a> { - let enum_proto = enum_with_scope.en.proto(); - eprintln!("EnumGen::parse enum.name: {}", enum_proto.name()); - let customize = customize.child( &customize_from_rustproto_for_enum(enum_with_scope.en.proto().options.get_or_default()), &enum_with_scope.en, diff --git a/protobuf-codegen/src/gen/field/elem.rs b/protobuf-codegen/src/gen/field/elem.rs index ab6d899d9..fd06b4ced 100644 --- a/protobuf-codegen/src/gen/field/elem.rs +++ b/protobuf-codegen/src/gen/field/elem.rs @@ -65,8 +65,6 @@ impl<'a> FieldElemMessage<'a> { } fn rust_type(&self, reference: &FileAndMod) -> RustType { - eprintln!("FieldElemMessage::rust_type message.mod_name: {}", self.message.mod_name().to_string()); - eprintln!("FieldElemMessage::rust_type message.name: {}", self.message.message.name()); RustType::Message(self.rust_name_relative(reference)) } } diff --git a/protobuf-codegen/src/gen/field/mod.rs b/protobuf-codegen/src/gen/field/mod.rs index 4fc908409..d6a5e44a4 100644 --- a/protobuf-codegen/src/gen/field/mod.rs +++ b/protobuf-codegen/src/gen/field/mod.rs @@ -145,12 +145,6 @@ impl<'a> FieldGen<'a> { path: Vec, info: Option<&'a SourceCodeInfo>, ) -> anyhow::Result> { - let message_proto = field.message.message.proto(); - eprintln!("FieldGen::parse message.name: {}", message_proto.name()); - eprintln!("FieldGen::parse field.name: {}", field.field.proto().name()); - eprintln!("FieldGen::parse field.type.protobuf_name: {}", field.field.proto().type_().protobuf_name()); - eprintln!("FieldGen::parse field.type_name: {}", field.field.proto().type_name()); - let customize = parent_customize .child( &customize_from_rustproto_for_field(field.field.proto().options.get_or_default()), @@ -342,8 +336,6 @@ impl<'a> FieldGen<'a> { // for field `foo`, return type of `fn foo(..)` fn getter_return_type(&self) -> RustType { - eprintln!("FieldGen::getter_return_type"); - let reference = self .proto_field .message diff --git a/protobuf-codegen/src/gen/file.rs b/protobuf-codegen/src/gen/file.rs index 71570cb77..79af21fda 100644 --- a/protobuf-codegen/src/gen/file.rs +++ b/protobuf-codegen/src/gen/file.rs @@ -4,6 +4,9 @@ use protobuf::descriptor::file_options; use protobuf::descriptor::FileDescriptorProto; use protobuf::reflect::FileDescriptor; use protobuf_parse::ProtoPath; +use tracing::event; +use tracing::instrument; +use tracing::Level; use crate::compiler_plugin; use crate::customize::ctx::CustomizeElemCtx; @@ -25,6 +28,7 @@ pub(crate) struct GenFileResult { pub(crate) mod_name: String, } +#[instrument(level = Level::INFO, skip_all)] pub(crate) fn gen_file( file_descriptor: &FileDescriptor, _files_map: &HashMap<&ProtoPath, &FileDescriptor>, @@ -54,6 +58,17 @@ pub(crate) fn gen_file( let lite_runtime = customize.for_elem.lite_runtime.unwrap_or(false); + let mod_name = proto_path_to_rust_mod(file_descriptor.proto().name()).into_string(); + + event!( + Level::INFO, + file_descriptor.name = file_descriptor.proto().name(), + file_descriptor.package = file_descriptor.proto().package(), + lite_runtime, + mod_name, + "Generating file" + ); + let v = CodeWriter::with(|w| { w.write_generated_by("rust-protobuf", env!("CARGO_PKG_VERSION"), parser); @@ -142,13 +157,17 @@ pub(crate) fn gen_file( Ok(GenFileResult { compiler_plugin_result: compiler_plugin::GenResult { - name: if customize.for_elem.gen_mod_rs_hierarchy_out_dir_mod_name.is_some() { + name: if customize + .for_elem + .gen_mod_rs_hierarchy_out_dir_mod_name + .is_some() + { file_descriptor_to_hierarchical_rs(file_descriptor.proto()) } else { proto_name_to_rs(file_descriptor.proto().name()) }, content: v.into_bytes(), }, - mod_name: proto_path_to_rust_mod(file_descriptor.proto().name()).into_string(), + mod_name, }) } diff --git a/protobuf-codegen/src/gen/message.rs b/protobuf-codegen/src/gen/message.rs index bd8a1098a..aa72157a9 100644 --- a/protobuf-codegen/src/gen/message.rs +++ b/protobuf-codegen/src/gen/message.rs @@ -4,6 +4,8 @@ use protobuf::descriptor::*; use protobuf::reflect::FileDescriptor; use protobuf::reflect::MessageDescriptor; use protobuf_parse::snake_case; +use tracing::event; +use tracing::Level; use crate::customize::ctx::CustomizeElemCtx; use crate::customize::ctx::SpecialFieldPseudoDescriptor; @@ -657,6 +659,14 @@ impl<'a> MessageGen<'a> { } pub fn write(&self, w: &mut CodeWriter) -> anyhow::Result<()> { + event!( + Level::INFO, + proto_file.name = self.file_descriptor.proto().name(), + proto_file.package = self.file_descriptor.proto().package(), + message.name = self.message_descriptor.proto().name(), + "Generating message" + ); + w.all_documentation(self.info, self.path); self.write_struct(w); diff --git a/protobuf-codegen/src/gen/oneof.rs b/protobuf-codegen/src/gen/oneof.rs index 9f5046bf6..3f34cdccf 100644 --- a/protobuf-codegen/src/gen/oneof.rs +++ b/protobuf-codegen/src/gen/oneof.rs @@ -176,8 +176,6 @@ impl<'a> OneofGen<'a> { oneof: OneofWithContext<'a>, parent_customize: &CustomizeElemCtx<'a>, ) -> OneofGen<'a> { - let message_proto = message.message.message.proto(); - eprintln!("OneofGen::parse message.name: {}", message_proto.name()); let customize = parent_customize.child(&Customize::default(), &oneof.oneof); let lite_runtime = customize.for_elem.lite_runtime.unwrap_or_else(|| { oneof diff --git a/protobuf-codegen/src/gen/paths.rs b/protobuf-codegen/src/gen/paths.rs index f562cbaf7..8910f96e5 100644 --- a/protobuf-codegen/src/gen/paths.rs +++ b/protobuf-codegen/src/gen/paths.rs @@ -1,4 +1,5 @@ use protobuf::descriptor::FileDescriptorProto; +use tracing::{instrument, Level}; use crate::gen::inside::protobuf_crate_path; use crate::gen::rust::ident::RustIdent; @@ -59,6 +60,7 @@ pub fn file_descriptor_to_hierarchical_rs(file_descriptor: &FileDescriptorProto) } } +#[instrument(level = Level::DEBUG, skip(customize), ret(Display))] pub(crate) fn proto_path_to_fn_file_descriptor( proto_path: &str, customize: &Customize, @@ -76,14 +78,11 @@ pub(crate) fn proto_path_to_fn_file_descriptor( s => { if let Some(mod_path) = &customize.gen_mod_rs_hierarchy_out_dir_mod_name { let mut rust_path = RustPath::from("crate"); - eprintln!("Forming path wth rust_path: {}", rust_path); for mod_part in mod_path.split("::") { rust_path = rust_path.append_ident(RustIdent::from(mod_part)); - eprintln!("Forming path wth rust_path: {}", rust_path); } for component in proto_path.split("/").filter(|p| !p.ends_with(".proto")) { rust_path = rust_path.append_ident(RustIdent::from(component)); - eprintln!("Forming path wth rust_path: {}", rust_path); } rust_path.append_ident("file_descriptor".into()) } else { diff --git a/protobuf-codegen/src/gen/rust_types_values.rs b/protobuf-codegen/src/gen/rust_types_values.rs index cc903a3a9..1608cb8b5 100644 --- a/protobuf-codegen/src/gen/rust_types_values.rs +++ b/protobuf-codegen/src/gen/rust_types_values.rs @@ -5,6 +5,8 @@ use protobuf::descriptor::*; use protobuf::reflect::FileDescriptor; use protobuf_parse::ProtobufAbsPath; use regex::Regex; +use tracing::event; +use tracing::{instrument, Level}; use crate::customize::Customize; use crate::gen::field::type_ext::TypeExt; @@ -83,15 +85,11 @@ impl RustType { } RustType::MessageField(ref param) => { let crate_path = protobuf_crate_path(customize); - eprintln!("Writing MessageField"); format!("{}::MessageField<{}>", crate_path, param.to_code(customize)) } RustType::Uniq(ref param) => format!("::std::boxed::Box<{}>", param.to_code(customize)), RustType::Ref(ref param) => format!("&{}", param.to_code(customize)), RustType::Message(ref name) => { - eprintln!("Writing Message name: {}", name); - eprintln!("Writing Message rust.ident: {}", name.0.ident); - eprintln!("Writing Message rust.path: {}", name.0.path); format!("{}", name) } RustType::Enum(ref name, ..) | RustType::Oneof(ref name) => format!("{}", name), @@ -487,16 +485,28 @@ pub(crate) fn make_path(source: &RustRelativePath, dest: &RustIdentWithPath) -> make_path_to_path(source, &dest.path).with_ident(dest.ident.clone()) } +#[instrument(level = Level::DEBUG, skip_all, fields(current.file = current.file, current.relative_mod = current.relative_mod.to_string()), ret(Display))] pub(crate) fn message_or_enum_to_rust_relative( message_or_enum: &dyn WithScope, current: &FileAndMod, ) -> RustIdentWithPath { - eprintln!("message_or_enum_to_rust_relative"); let same_file = message_or_enum.file_descriptor().name() == current.file; if same_file { // field type is a message or enum declared in the same file + event!( + Level::DEBUG, + current.file = current.file, + current.relative_mod = current.relative_mod.to_string(), + "Field type is message or enum in same file" + ); make_path(¤t.relative_mod, &message_or_enum.rust_name_to_file()) } else if let Some(name) = is_well_known_type_full(&message_or_enum.name_absolute()) { + event!( + Level::DEBUG, + current.file = current.file, + current.relative_mod = current.relative_mod.to_string(), + "Field type is a well-known type included in the rust-protobuf library" + ); // Well-known types are included in rust-protobuf library // https://developers.google.com/protocol-buffers/docs/reference/google.protobuf let file_descriptor = message_or_enum.file_descriptor(); @@ -512,6 +522,12 @@ pub(crate) fn message_or_enum_to_rust_relative( protobuf_crate = protobuf_crate_path(¤t.customize), )) } else if is_descriptor_proto(&message_or_enum.file_descriptor()) { + event!( + Level::DEBUG, + current.file = current.file, + current.relative_mod = current.relative_mod.to_string(), + "Field type is a descriptor proto" + ); // Messages defined in descriptor.proto RustIdentWithPath::from(format!( "{}::descriptor::{}", @@ -519,60 +535,29 @@ pub(crate) fn message_or_enum_to_rust_relative( message_or_enum.rust_name_to_file() )) } else if let Some(mod_name) = ¤t.customize.gen_mod_rs_hierarchy_out_dir_mod_name { - let mut rust_name = format!("crate::{}", mod_name.to_owned()); - eprintln!( - "message_or_enum_to_rust_relative MY_WAY mod_name: {}", - rust_name + event!( + Level::DEBUG, + current.file = current.file, + current.relative_mod = current.relative_mod.to_string(), + root_mod_name = mod_name, + "Field type identifier needs to be resolved in the output dir hierarchy" ); + let mut rust_name = format!("crate::{}", mod_name.to_owned()); for component in message_or_enum .name_absolute() .trim_start_matches(':') .split('.') .filter(|s| !s.is_empty()) { - eprintln!( - "message_or_enum_to_rust_relative MY_WAY component: {}", - component - ); rust_name.push_str(&format!("::{}", component.replace('-', "_"))); } - eprintln!( - "message_or_enum_to_rust_relative MY_WAY rust_name: {}", - rust_name - ); RustIdentWithPath::from(rust_name) } else { - eprintln!( - "message_or_enum_to_rust_relative current.file: {}", - current.file - ); - eprintln!( - "message_or_enum_to_rust_relative current.relative_mod: {}", - current.relative_mod - ); - eprintln!( - "message_or_enum_to_rust_relative message_or_enum.name_absolute: {}", - message_or_enum.name_absolute() - ); - eprintln!( - "message_or_enum_to_rust_relative message_or_enum.name_to_package: {}", - message_or_enum.name_to_package() - ); - eprintln!( - "message_or_enum_to_rust_relative message_or_enum.protobuf_name_to_package: {}", - message_or_enum.protobuf_name_to_package() - ); - eprintln!( - "message_or_enum_to_rust_relative message_or_enum.rust_name: {}", - message_or_enum.rust_name() - ); - eprintln!( - "message_or_enum_to_rust_relative message_or_enum.rust_name_to_file: {}", - message_or_enum.rust_name_to_file() - ); - eprintln!( - "message_or_enum_to_rust_relative message_or_enum.rust_name_with_file: {}", - message_or_enum.rust_name_with_file() + event!( + Level::DEBUG, + current.file = current.file, + current.relative_mod = current.relative_mod.to_string(), + "Field type identifier is relative to current mod" ); current .relative_mod diff --git a/protobuf-codegen/src/gen/scope.rs b/protobuf-codegen/src/gen/scope.rs index 58d46b9f3..f70d233f1 100644 --- a/protobuf-codegen/src/gen/scope.rs +++ b/protobuf-codegen/src/gen/scope.rs @@ -277,11 +277,6 @@ impl<'a> Scope<'a> { } pub fn file_and_mod(&self, customize: Customize) -> FileAndMod { - let d = self.file_scope.file_descriptor.proto(); - eprintln!("Scope::file_and_mod package: package: {}", d.package()); - eprintln!("Scope::file_and_mod package: name: {}", d.name()); - eprintln!("Scope::file_and_mod package: rust_path_to_file: {}", self.rust_path_to_file()); - eprintln!("Scope::file_and_mod package: path_str: {}", self.path_str()); FileAndMod { file: self.file_scope.file_descriptor.proto().name().to_owned(), relative_mod: self.rust_path_to_file(), diff --git a/protobuf-codegen/src/gen_and_write.rs b/protobuf-codegen/src/gen_and_write.rs index 1d0c7299d..f482b77af 100644 --- a/protobuf-codegen/src/gen_and_write.rs +++ b/protobuf-codegen/src/gen_and_write.rs @@ -10,6 +10,7 @@ use protobuf_parse::ProtoPathBuf; use crate::customize::CustomizeCallback; use crate::gen::all::gen_all; +use crate::logging::init_logging; use crate::Customize; #[derive(Debug, thiserror::Error)] @@ -46,6 +47,8 @@ pub fn gen_and_write( } } + init_logging(&customize)?; + let results = gen_all( file_descriptors, parser, diff --git a/protobuf-codegen/src/lib.rs b/protobuf-codegen/src/lib.rs index 0471d7c2d..0470f93e1 100644 --- a/protobuf-codegen/src/lib.rs +++ b/protobuf-codegen/src/lib.rs @@ -136,6 +136,7 @@ mod compiler_plugin; mod customize; mod gen; pub mod gen_and_write; +mod logging; pub mod protoc_gen_rust; pub use codegen::Codegen; diff --git a/protobuf-codegen/src/logging.rs b/protobuf-codegen/src/logging.rs new file mode 100644 index 000000000..409a433e8 --- /dev/null +++ b/protobuf-codegen/src/logging.rs @@ -0,0 +1,52 @@ +use std::path::PathBuf; + +use anyhow::anyhow; + +use tracing::{event, Level}; +use tracing_subscriber::filter::LevelFilter; +use tracing_subscriber::fmt::format::FmtSpan; +use tracing_subscriber::{prelude::*, EnvFilter}; + +use crate::Customize; + +pub(crate) fn init_logging(customize: &Customize) -> anyhow::Result<()> { + if let Some(log_file) = &customize.log_file { + let file_name = PathBuf::from(log_file); + let file_appender = tracing_appender::rolling::never( + file_name.parent().ok_or_else(|| { + anyhow!( + "Specified log_file did not have a parent directory: {}", + file_name.display().to_string() + ) + })?, + file_name.file_name().ok_or_else(|| { + anyhow!( + "Specified log_file was invalid: {}", + file_name.display().to_string() + ) + })?, + ); + let fmt = tracing_subscriber::fmt::layer() + .with_level(true) + .with_target(true) + .with_ansi(false) + .with_writer(file_appender) + .with_span_events(FmtSpan::ENTER) + .with_span_events(FmtSpan::EXIT) + .json(); + tracing_subscriber::registry() + .with(fmt) + .with( + EnvFilter::builder() + .with_default_directive(LevelFilter::INFO.into()) + .from_env_lossy(), + ) + .init(); + event!( + Level::INFO, + log_file = file_name.display().to_string(), + "Logging initialized" + ); + } + Ok(()) +} diff --git a/protobuf-codegen/src/protoc_gen_rust.rs b/protobuf-codegen/src/protoc_gen_rust.rs index 014db6a14..f4db1c1b2 100644 --- a/protobuf-codegen/src/protoc_gen_rust.rs +++ b/protobuf-codegen/src/protoc_gen_rust.rs @@ -3,12 +3,14 @@ use crate::compiler_plugin; use crate::customize::CustomizeCallbackDefault; use crate::gen::all::gen_all; +use crate::logging::init_logging; use crate::Customize; #[doc(hidden)] pub fn protoc_gen_rust_main() { compiler_plugin::plugin_main(|r| { let customize = Customize::parse_from_parameter(r.parameter).expect("parse options"); + init_logging(&customize)?; gen_all( r.file_descriptors, "protoc --rust-out=...",