Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

build(proto): improve protobuf generation #3636

Merged
merged 1 commit into from
Dec 8, 2023
Merged

build(proto): improve protobuf generation #3636

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
117 changes: 40 additions & 77 deletions hybridse/src/proto/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,80 +12,43 @@
# See the License for the specific language governing permissions and
# limitations under the License.

include_directories(${INCLUDE_DIRECTORIES}
${PROJECT_SOURCE_DIR}/src
${PROJECT_BINARY_DIR}/src)

set(TYPE_PB_CC
${CMAKE_CURRENT_BINARY_DIR}/fe_type.pb.cc)

set(COMMON_PB_CC
${CMAKE_CURRENT_BINARY_DIR}/fe_common.pb.cc
)

set(DBMS_PB_CC
${CMAKE_CURRENT_BINARY_DIR}/dbms.pb.cc)

set(TABLET_PB_CC
${CMAKE_CURRENT_BINARY_DIR}/fe_tablet.pb.cc)

set(BATCH_PB_CC
${CMAKE_CURRENT_BINARY_DIR}/batch.pb.cc)

set(PLAN_PB_CC
${CMAKE_CURRENT_BINARY_DIR}/plan.pb.cc)

add_custom_command(OUTPUT ${COMMON_PB_CC}
COMMAND ${Protobuf_PROTOC_EXECUTABLE} -I ${PROJECT_SOURCE_DIR}/src/proto
--cpp_out=${CMAKE_CURRENT_BINARY_DIR} ${PROJECT_SOURCE_DIR}/src/proto/fe_common.proto
DEPENDS ${PROJECT_SOURCE_DIR}/src/proto/fe_common.proto
)

add_custom_command(OUTPUT ${TYPE_PB_CC}
COMMAND ${Protobuf_PROTOC_EXECUTABLE} -I ${PROJECT_SOURCE_DIR}/src/proto --cpp_out=${CMAKE_CURRENT_BINARY_DIR} ${PROJECT_SOURCE_DIR}/src/proto/fe_type.proto
DEPENDS ${PROJECT_SOURCE_DIR}/src/proto/fe_type.proto
)

add_custom_command(OUTPUT ${DBMS_PB_CC}
COMMAND ${Protobuf_PROTOC_EXECUTABLE} -I ${PROJECT_SOURCE_DIR}/src/proto --cpp_out=${CMAKE_CURRENT_BINARY_DIR} ${PROJECT_SOURCE_DIR}/src/proto/dbms.proto
DEPENDS ${PROJECT_SOURCE_DIR}/src/proto/dbms.proto
)

add_custom_command(OUTPUT ${TABLET_PB_CC}
COMMAND ${Protobuf_PROTOC_EXECUTABLE} -I ${PROJECT_SOURCE_DIR}/src/proto --cpp_out=${CMAKE_CURRENT_BINARY_DIR} ${PROJECT_SOURCE_DIR}/src/proto/fe_tablet.proto
DEPENDS ${PROJECT_SOURCE_DIR}/src/proto/fe_tablet.proto
)

add_custom_command(OUTPUT ${BATCH_PB_CC}
COMMAND ${Protobuf_PROTOC_EXECUTABLE} -I ${PROJECT_SOURCE_DIR}/src/proto
--cpp_out=${CMAKE_CURRENT_BINARY_DIR} ${PROJECT_SOURCE_DIR}/src/proto/batch.proto
DEPENDS ${PROJECT_SOURCE_DIR}/src/proto/batch.proto
)

add_custom_command(OUTPUT ${PLAN_PB_CC}
COMMAND ${Protobuf_PROTOC_EXECUTABLE} -I ${PROJECT_SOURCE_DIR}/src/proto --cpp_out=${CMAKE_CURRENT_BINARY_DIR} ${PROJECT_SOURCE_DIR}/src/proto/plan.proto
DEPENDS ${PROJECT_SOURCE_DIR}/src/proto/plan.proto
)

set(HYBRIDSE_PROTO_SRC_FILES
${TABLET_PB_CC}
${DBMS_PB_CC}
${COMMON_PB_CC}
${TYPE_PB_CC}
${BATCH_PB_CC}
${PLAN_PB_CC})

ADD_CUSTOM_TARGET(run_gen_proto DEPENDS ${HYBRIDSE_PROTO_SRC_FILES})

add_library(hybridse_proto OBJECT ${HYBRIDSE_PROTO_SRC_FILES})
add_dependencies(hybridse_proto run_gen_proto)

if (SQL_JAVASDK_ENABLE)
add_custom_target(hybridse_proto_java ALL
COMMAND ${Protobuf_PROTOC_EXECUTABLE}
-I ${PROJECT_SOURCE_DIR}/src/proto
--java_out=${CMAKE_SOURCE_DIR}/java/hybridse-proto/src/main/java
${PROJECT_SOURCE_DIR}/src/proto/*.proto
COMMENT "Generating protobuf java class into ${CMAKE_SOURCE_DIR}/java/hybridse-proto/src/main/java"
)
endif()
include_directories(${INCLUDE_DIRECTORIES} ${PROJECT_SOURCE_DIR}/src
${PROJECT_BINARY_DIR}/src)

set(PROTO_CPP_FILES "")
set(PROTO_FILES "")
function(compile_proto proto_name java_file_suffix_path)
add_custom_command(
OUTPUT
${CMAKE_CURRENT_BINARY_DIR}/${proto_name}.pb.cc
${CMAKE_CURRENT_BINARY_DIR}/${proto_name}.pb.h
${CMAKE_SOURCE_DIR}/java/hybridse-proto/src/main/java/com/_4paradigm/hybridse/${java_file_suffix_path}
COMMAND ${Protobuf_PROTOC_EXECUTABLE}
-I ${PROJECT_SOURCE_DIR}/src/proto
--cpp_out=${CMAKE_CURRENT_BINARY_DIR}
--java_out=${CMAKE_SOURCE_DIR}/java/hybridse-proto/src/main/java
${PROJECT_SOURCE_DIR}/src/proto/${proto_name}.proto
DEPENDS ${PROJECT_SOURCE_DIR}/src/proto/${proto_name}.proto)
list(APPEND PROTO_CPP_FILES ${CMAKE_CURRENT_BINARY_DIR}/${proto_name}.pb.cc)
list(APPEND PROTO_FILES
${CMAKE_CURRENT_BINARY_DIR}/${proto_name}.pb.cc
${CMAKE_CURRENT_BINARY_DIR}/${proto_name}.pb.h
${CMAKE_SOURCE_DIR}/java/hybridse-proto/src/main/java/com/_4paradigm/hybridse/${java_file_suffix_path}
)
set(PROTO_CPP_FILES ${PROTO_CPP_FILES} PARENT_SCOPE)
set(PROTO_FILES ${PROTO_FILES} PARENT_SCOPE)
endfunction(compile_proto)

compile_proto(fe_type type/TypeOuterClass.java)
compile_proto(fe_common common/Common.java)
compile_proto(dbms dbms/DBMS.java)
compile_proto(fe_tablet tablet/Tablet.java)
compile_proto(batch batch/Batch.java)
compile_proto(plan batch/Plan.java)

add_library(hybridse_proto OBJECT ${PROTO_CPP_FILES})
set_property(
TARGET hybridse_proto
APPEND
PROPERTY ADDITIONAL_CLEAN_FILES ${PROTO_FILES}
)
2 changes: 1 addition & 1 deletion hybridse/src/sdk/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,5 +72,5 @@ if(SQL_JAVASDK_ENABLE)
COMMAND ${CMAKE_COMMAND} -E make_directory ${CMAKE_SOURCE_DIR}/java/hybridse-native/src/main/resources/
COMMAND ${CMAKE_COMMAND} -E copy $<TARGET_FILE:hybridse_jsdk_core> ${CMAKE_SOURCE_DIR}/java/hybridse-native/src/main/resources/
COMMENT "copy hybridse native so")
add_dependencies(cp_hybridse_native_so hybridse_proto_java)
add_dependencies(cp_hybridse_native_so hybridse_proto)
endif()
6 changes: 0 additions & 6 deletions java/openmldb-batch/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -210,12 +210,6 @@
</exclusions>
</dependency>

<dependency>
<groupId>com.4paradigm.openmldb</groupId>
<artifactId>openmldb-common</artifactId>
<version>${project.parent.version}</version>
</dependency>

<dependency>
<groupId>com.4paradigm.openmldb</groupId>
<artifactId>openmldb-spark-connector</artifactId>
Expand Down
5 changes: 0 additions & 5 deletions java/openmldb-jdbc/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,11 +38,6 @@
<artifactId>openmldb-native</artifactId>
<version>${variant.native.version}</version>
</dependency>
<dependency>
<groupId>com.4paradigm.openmldb</groupId>
<artifactId>openmldb-common</artifactId>
<version>${project.parent.version}</version>
</dependency>

<!-- https://mvnrepository.com/artifact/org.slf4j/slf4j-log4j12 -->
<dependency>
Expand Down
7 changes: 6 additions & 1 deletion java/openmldb-native/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,12 @@
</properties>

<dependencies>
<dependency>
<dependency>
<groupId>com.4paradigm.openmldb</groupId>
<artifactId>openmldb-common</artifactId>
<version>${project.parent.version}</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
<version>1.7.21</version>
Expand Down
5 changes: 0 additions & 5 deletions java/openmldb-synctool/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,11 +28,6 @@
<scope>test</scope>
</dependency>

<dependency>
<groupId>com.4paradigm.openmldb</groupId>
<artifactId>openmldb-common</artifactId>
<version>${project.parent.version}</version>
</dependency>
<dependency>
<groupId>com.4paradigm.openmldb</groupId>
<artifactId>openmldb-jdbc</artifactId>
Expand Down
6 changes: 0 additions & 6 deletions java/openmldb-taskmanager/pom.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -159,12 +159,6 @@
<version>${project.parent.version}</version>
</dependency>

<dependency>
<groupId>com.4paradigm.openmldb</groupId>
<artifactId>openmldb-common</artifactId>
<version>${project.parent.version}</version>
</dependency>

<dependency>
<groupId>com.4paradigm.openmldb</groupId>
<artifactId>openmldb-batchjob</artifactId>
Expand Down
58 changes: 36 additions & 22 deletions src/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,19 +23,26 @@ add_subdirectory(statistics)
file(MAKE_DIRECTORY ${CMAKE_BINARY_DIR}/src/proto)

set(PROTO_FILES "")
function(compile_proto proto_name project_dir)
add_custom_command(OUTPUT ${CMAKE_BINARY_DIR}/src/proto/${proto_name}.pb.cc
COMMAND ${Protobuf_PROTOC_EXECUTABLE} -I ${PROJECT_SOURCE_DIR}/src/proto
--cpp_out=${CMAKE_BINARY_DIR}/src/proto
--java_out=${project_dir}/java/openmldb-native/src/main/java
--java_out=${project_dir}/java/openmldb-import/src/main/java
--java_out=${project_dir}/java/openmldb-taskmanager/src/main/java
--java_out=${project_dir}/java/openmldb-common/src/main/java
--java_out=${project_dir}/java/openmldb-synctool/src/main/java
${project_dir}/src/proto/${proto_name}.proto
DEPENDS ${project_dir}/src/proto/${proto_name}.proto)
list(APPEND PROTO_FILES ${CMAKE_BINARY_DIR}/src/proto/${proto_name}.pb.cc)
set(PROTO_FILES ${PROTO_FILES} PARENT_SCOPE)
set(PROTO_CPP_FILES "")
function(compile_proto proto_name java_out_file_name)
# will rerun if DEPENDS changed or OUTPUT not exists
add_custom_command(
OUTPUT ${CMAKE_BINARY_DIR}/src/proto/${proto_name}.pb.cc
${CMAKE_BINARY_DIR}/src/proto/${proto_name}.pb.h
${CMAKE_SOURCE_DIR}/java/openmldb-common/src/main/java/com/_4paradigm/openmldb/proto/${java_out_file_name}.java
COMMAND ${Protobuf_PROTOC_EXECUTABLE} -I ${PROJECT_SOURCE_DIR}/src/proto
--cpp_out=${CMAKE_BINARY_DIR}/src/proto
--java_out=${CMAKE_SOURCE_DIR}/java/openmldb-common/src/main/java
${CMAKE_SOURCE_DIR}/src/proto/${proto_name}.proto
DEPENDS ${CMAKE_SOURCE_DIR}/src/proto/${proto_name}.proto)
list(APPEND PROTO_CPP_FILES ${CMAKE_BINARY_DIR}/src/proto/${proto_name}.pb.cc)
list(APPEND PROTO_FILES
${CMAKE_BINARY_DIR}/src/proto/${proto_name}.pb.cc
${CMAKE_BINARY_DIR}/src/proto/${proto_name}.pb.h
${CMAKE_SOURCE_DIR}/java/openmldb-common/src/main/java/com/_4paradigm/openmldb/proto/${java_out_file_name}.java
)
set(PROTO_CPP_FILES ${PROTO_CPP_FILES} PARENT_SCOPE)
set(PROTO_FILES ${PROTO_FILES} PARENT_SCOPE)
endfunction(compile_proto)

function(compile_lib LIB_NAME DIR DEPEND_FILE_LIST)
Expand Down Expand Up @@ -98,16 +105,23 @@ function(compile_test DIR)
set(test_list ${test_list} PARENT_SCOPE)
endfunction(compile_test)

compile_proto(type ${PROJECT_SOURCE_DIR})
compile_proto(common ${PROJECT_SOURCE_DIR})
compile_proto(tablet ${PROJECT_SOURCE_DIR})
compile_proto(name_server ${PROJECT_SOURCE_DIR})
compile_proto(sql_procedure ${PROJECT_SOURCE_DIR})
compile_proto(api_server ${PROJECT_SOURCE_DIR})
compile_proto(taskmanager ${PROJECT_SOURCE_DIR})
compile_proto(data_sync ${PROJECT_SOURCE_DIR})
compile_proto(type Type)
compile_proto(common Common)
compile_proto(tablet Tablet)
compile_proto(name_server NS)
compile_proto(sql_procedure SQLProcedure)
compile_proto(api_server ApiServer)
compile_proto(taskmanager TaskManager)
compile_proto(data_sync DataSync)

add_library(openmldb_proto STATIC ${PROTO_CPP_FILES})
# clean generated proto file, including '{cmake_source}/java/openmldb-common'
set_property(
TARGET openmldb_proto
APPEND
PROPERTY ADDITIONAL_CLEAN_FILES ${PROTO_FILES}
)

add_library(openmldb_proto STATIC ${PROTO_FILES})
add_library(openmldb_flags STATIC flags.cc)

compile_lib(openmldb_codec codec "")
Expand Down
Loading