diff --git a/Project.toml b/Project.toml index 39dcd72..c357e90 100644 --- a/Project.toml +++ b/Project.toml @@ -17,7 +17,7 @@ Aqua = "0.8" BufferedStreams = "1.2" Dates = "1" EnumX = "1" -JET = "0.4, 0.5, 0.6, 0.7, 0.8" +JET = "0.4, 0.5, 0.6, 0.7, 0.8, 0.9" TOML = "1" Test = "1" julia = "1.6" diff --git a/src/codegen/modules.jl b/src/codegen/modules.jl index ac355b5..f72e00c 100644 --- a/src/codegen/modules.jl +++ b/src/codegen/modules.jl @@ -50,7 +50,7 @@ struct ProtoModule nonpkg_imports::Set{String} end namespace(m::ProtoModule) = m.namespace -empty_module(name::AbstractString, namespace_vec) = ProtoModule(name, namespace_vec, [], Dict(), Set(), Set(), Set()) +empty_module(name::AbstractString, namespace_vec) = ProtoModule(name, namespace_vec, [], Dict(), Set(), Set(), Set()) struct Namespaces non_namespaced_protos::Vector{ResolvedProtoFile} @@ -64,7 +64,7 @@ function Namespaces(files_in_order::Vector{ResolvedProtoFile}, root_path::String push!(t.non_namespaced_protos, file) else top_namespace = first(namespace(file)) - p = get!(()->empty_module(top_namespace, [top_namespace]), t.packages, top_namespace) + p = get!(() -> empty_module(top_namespace, [top_namespace]), t.packages, top_namespace) _add_file_to_package!(p, file, proto_files, root_path) end end @@ -78,7 +78,7 @@ function _add_file_to_package!(root::ProtoModule, file::ResolvedProtoFile, proto depth += 1 name == node.name && continue _ns = namespace(file)[1:depth] - node = get!(()->empty_module(name, _ns), node.submodules, _ns) + node = get!(() -> empty_module(name, _ns), node.submodules, _ns) end for ipath in import_paths(file) imported_file = proto_files[ipath] @@ -87,7 +87,7 @@ function _add_file_to_package!(root::ProtoModule, file::ResolvedProtoFile, proto # Sometimes they are forced to be namespaced with `always_use_modules` # but it is the responsibility of the root module to make sure there # is a importable module in to topmost scope - depth != 1 && push!(node.external_imports, string("." ^ depth, replace(proto_script_name(imported_file), ".jl" => ""))) + depth != 1 && push!(node.external_imports, string("."^depth, replace(proto_script_name(imported_file), ".jl" => ""))) push!(root.nonpkg_imports, replace(proto_script_name(imported_file), ".jl" => "")) else file_pkg = proto_package_name(imported_file) @@ -96,7 +96,7 @@ function _add_file_to_package!(root::ProtoModule, file::ResolvedProtoFile, proto elseif file_pkg == root.name # same package, different submodule push!(node.internal_imports, namespace(imported_file)) else - depth != 1 && push!(node.external_imports, string("." ^ depth, proto_package_name(imported_file))) + depth != 1 && push!(node.external_imports, string("."^depth, proto_package_name(imported_file))) push!(root.external_imports, rel_import_path(imported_file, root_path)) end end @@ -108,6 +108,9 @@ end function generate_module_file(io::IO, m::ProtoModule, output_directory::AbstractString, parsed_files::Dict, options::Options, depth::Int) println(io, "module $(m.name)") println(io) + + included_files = Set{String}() + has_deps = !isempty(m.internal_imports) || !isempty(m.external_imports) || !isempty(m.nonpkg_imports) if depth == 1 # This is where we include external packages so they are available downstream @@ -118,9 +121,12 @@ function generate_module_file(io::IO, m::ProtoModule, output_directory::Abstract # We wrap them in a module to make sure that multiple downstream dependencies # can import them safely. for nonpkg_import in m.nonpkg_imports - !options.always_use_modules && print(io, "module $(nonpkg_import)\n ") - println(io, "include(", repr(joinpath("..", string(nonpkg_import, ".jl"))), ')') - !options.always_use_modules && println(io, "end") + if !(nonpkg_import in included_files) + !options.always_use_modules && print(io, "module $(nonpkg_import)\n ") + println(io, "include(", repr(joinpath("..", string(nonpkg_import, ".jl"))), ')') + !options.always_use_modules && println(io, "end") + push!(included_files, nonpkg_import) + end end else # depth > 1 # We're not a top package module, we can import external dependencies @@ -145,11 +151,11 @@ function generate_module_file(io::IO, m::ProtoModule, output_directory::Abstract # This is where we import internal dependencies. We only need to import the toplevel package # and then use fully qualified names for the imported definitions. if !isempty(m.internal_imports) - print(io, "import ", string("." ^ length(namespace(m)), first(namespace(m)))) + print(io, "import ", string("."^length(namespace(m)), first(namespace(m)))) # We can't import the toplevel module to a leaf module if their names collide # we also need to tweak the `package_namespace` field of each imported ReferencedType if first(namespace(m)) == last(namespace(m)) - println(io, " as var\"#", first(namespace(m)), '"') + println(io, " as var\"#", first(namespace(m)), '"') else println(io) end @@ -158,6 +164,7 @@ function generate_module_file(io::IO, m::ProtoModule, output_directory::Abstract # Load in imported proto files that are defined in this package (the files ending with `_pb.jl`) # In case there is a dependency of some of these files on a submodule, we include that submodule # first. + seen = Set{String}() for file in m.proto_files for i in import_paths(file) @@ -169,7 +176,10 @@ function generate_module_file(io::IO, m::ProtoModule, output_directory::Abstract end end end - println(io, "include(", repr(proto_script_name(file)), ")") + if !(proto_script_name(file) in included_files) + println(io, "include(", repr(proto_script_name(file)), ")") + push!(included_files, proto_script_name(file)) + end end # Load in submodules nested in this namespace (the modules ending with `PB`), # that is, if we didn't include them above. @@ -194,14 +204,14 @@ function generate_package(node::ProtoModule, output_directory::AbstractString, p CodeGenerators.translate(dst_path, file, parsed_files, options) end for submodule in values(node.submodules) - generate_package(submodule, path, parsed_files, options, depth+1) + generate_package(submodule, path, parsed_files, options, depth + 1) end return nothing end function validate_search_directories!(search_directories::Vector{String}, include_vendored_wellknown_types::Bool) include_vendored_wellknown_types && push!(search_directories, VENDORED_WELLKNOWN_TYPES_PARENT_PATH) - unique!(map!(x->joinpath(abspath(x), ""), search_directories, search_directories)) + unique!(map!(x -> joinpath(abspath(x), ""), search_directories, search_directories)) bad_dirs = filter(!isdir, search_directories) !isempty(bad_dirs) && error("`search_directories` $bad_dirs don't exist") return nothing diff --git a/src/codegen/toplevel_definitions.jl b/src/codegen/toplevel_definitions.jl index af1799f..338475a 100644 --- a/src/codegen/toplevel_definitions.jl +++ b/src/codegen/toplevel_definitions.jl @@ -18,13 +18,13 @@ function generate_struct_field(io, field::FieldType{ReferencedType}, ctx::Contex # is already a vector if the field was repeated. type_param = get(type_params, field.name, nothing) if struct_name == type_name - type_name = string("Union{Nothing,", type_name,"}") + type_name = string("Union{Nothing,", type_name, "}") elseif !isnothing(type_param) type_name = _is_repeated_field(field) ? string("Vector{", type_param.param, '}') : type_param.param elseif field.label == Parsers.OPTIONAL || field.label == Parsers.DEFAULT should_force_required = _should_force_required(string(struct_name, ".", field.name), ctx) if !should_force_required && _is_message(field.type, ctx) - type_name = string("Union{Nothing,", type_name,"}") + type_name = string("Union{Nothing,", type_name, "}") end end println(io, " ", field_name, "::", type_name) @@ -40,13 +40,13 @@ function generate_struct_field(io, field::GroupType, ctx::Context, type_params) # is already a vector if the field was repeated. type_param = get(type_params, field.name, nothing) if struct_name == type_name - type_name = string("Union{Nothing,", type_name,"}") + type_name = string("Union{Nothing,", type_name, "}") elseif !isnothing(type_param) type_name = type_param.param elseif field.label == Parsers.OPTIONAL || field.label == Parsers.DEFAULT should_force_required = _should_force_required(string(struct_name, ".", field.name), ctx) if !should_force_required - type_name = string("Union{Nothing,", type_name,"}") + type_name = string("Union{Nothing,", type_name, "}") end end println(io, " ", field_name, "::", type_name) @@ -101,7 +101,7 @@ function codegen(io, t::MessageType, ctx::Context) generate_struct(io, t, ctx) maybe_generate_kwarg_constructor_method(io, t, ctx) maybe_generate_deprecation(io, t) - maybe_generate_reserved_fields_method(io, t ) + maybe_generate_reserved_fields_method(io, t) maybe_generate_extendable_field_numbers_method(io, t) maybe_generate_oneof_field_types_method(io, t, ctx) maybe_generate_default_values_method(io, t, ctx) @@ -149,10 +149,12 @@ function translate(io, rp::ResolvedProtoFile, file_map::Dict{String,ResolvedProt p = rp.proto_file Parsers.check_name_collisions(p, file_map) println(io, "# Autogenerated using ProtoBuf.jl v$(PACKAGE_VERSION) on $(Dates.now())") - println(io, "# original file: ", p.filepath," (proto", p.preamble.isproto3 ? '3' : '2', " syntax)") + println(io, "# original file: ", p.filepath, " (proto", p.preamble.isproto3 ? '3' : '2', " syntax)") println(io) # TODO: cleanup here, we probably don't need a reference to rp.transitive_imports in ctx? + included_files = Set{String}() + ctx = Context(p, rp.import_path, file_map, copy(p.cyclic_definitions), Ref{String}(), rp.transitive_imports, options) if !is_namespaced(p) options.always_use_modules && println(io, "module $(replace(proto_script_name(p), ".jl" => ""))") @@ -162,14 +164,19 @@ function translate(io, rp::ResolvedProtoFile, file_map::Dict{String,ResolvedProt for path in import_paths(p) dependency = file_map[path] if !is_namespaced(dependency) - # if the dependency is also not namespaced, we can just include it - println(io, "include(", repr(proto_script_name(dependency)), ")") - options.always_use_modules && println(io, "import $(replace(proto_script_name(dependency), ".jl" => ""))") + if !(proto_script_name(dependency) in included_files) + println(io, "include(", repr(proto_script_name(dependency)), ")") + options.always_use_modules && println(io, "import $(replace(proto_script_name(dependency), ".jl" => ""))") + push!(included_files, proto_script_name(dependency)) + end else # otherwise we need to import it trough a module import_pkg_name = namespaced_top_import(dependency) - println(io, "include(", repr(namespaced_top_include(dependency)), ")") - println(io, "import $(import_pkg_name)") + if !(import_pkg_name in included_files) + println(io, "include(", repr(namespaced_top_include(dependency)), ")") + println(io, "import $(import_pkg_name)") + push!(included_files, import_pkg_name) + end end end end # Otherwise all includes will happen in the enclosing module