build: upgrade thirdparty to 0.6.0 (#3557)

* build: upgrade thirdparty to 0.6.0 * fix: dockerfile * fix(single_tablet_test): pure virtual method call called
4paradigm · Oct 27, 2023 · 5c6b40c · 5c6b40c
1 parent 671897e
commit 5c6b40c
Show file tree

Hide file tree

Showing 5 changed files with 45 additions and 30 deletions.
diff --git a/Makefile b/Makefile
@@ -139,29 +139,39 @@ THIRD_PARTY_BUILD_DIR ?= $(MAKEFILE_DIR)/.deps
 THIRD_PARTY_SRC_DIR ?= $(MAKEFILE_DIR)/thirdsrc
 THIRD_PARTY_DIR ?= $(THIRD_PARTY_BUILD_DIR)/usr
 
-# trick: for those compile inside hybridsql docker image, thirdparty is pre-installed in /deps/usr.
-#  we check this by asserting if the environment variable '$THIRD_PARTY_DIR' is defined to '/deps/usr',
-#  if true, thirdparty download is skipped
-# zetasql check separately since it update more frequently:
-#  it will updated if the variable '$ZETASQL_VERSION' (defined in docker) not equal to that defined in current code
-override GREP_PATTERN = "set(ZETASQL_VERSION"
+override ZETASQL_PATTERN = "set(ZETASQL_VERSION"
+override THIRD_PATTERN = "set(HYBRIDSQL_ASSERTS_VERSION"
+new_zetasql_version := $(shell grep $(ZETASQL_PATTERN) third-party/cmake/FetchZetasql.cmake | sed 's/[^0-9.]*\([0-9.]*\).*/\1/')
+new_third_version := $(shell grep $(THIRD_PATTERN) third-party/CMakeLists.txt | sed 's/[^0-9.]*\([0-9.]*\).*/\1/')
+
 thirdparty-fast:
 	@if [ $(THIRD_PARTY_DIR) != "/deps/usr" ] ; then \
 	    echo "[deps]: install thirdparty and zetasql"; \
 	    $(MAKE) thirdparty; \
-	elif [ -n "$(ZETASQL_VERSION)" ]; then \
-	    new_zetasql_version=$(shell grep $(GREP_PATTERN) third-party/cmake/FetchZetasql.cmake | sed 's/[^0-9.]*\([0-9.]*\).*/\1/'); \
-	    if [ "$$new_zetasql_version" != "$(ZETASQL_VERSION)" ] ; then \
-		echo "[deps]: thirdparty up-to-date. reinstall zetasql from $(ZETASQL_VERSION) to $$new_zetasql_version"; \
-		$(MAKE) thirdparty-configure; \
-		$(CMAKE_PRG) --build $(THIRD_PARTY_BUILD_DIR) --target zetasql; \
-	    else \
-		echo "[deps]: all up-to-date. zetasql already installed with version: $(ZETASQL_VERSION)"; \
-	    fi; \
 	else \
-	    echo "[deps]: install zetasql only"; \
 	    $(MAKE) thirdparty-configure; \
-	    $(CMAKE_PRG) --build $(THIRD_PARTY_BUILD_DIR) --target zetasql; \
+	    if [ -n "$(ZETASQL_VERSION)" ] ; then \
+		if [ "$(new_zetasql_version)" != "$(ZETASQL_VERSION)" ] ; then \
+		    echo "[deps]: installing zetasql from $(ZETASQL_VERSION) to $(new_zetasql_version)"; \
+		    $(CMAKE_PRG) --build $(THIRD_PARTY_BUILD_DIR) --target zetasql; \
+		else \
+		    echo "[deps]: zetasql up-to-date with version: $(ZETASQL_VERSION)"; \
+		fi; \
+	    else \
+		echo "[deps]: installing latest zetasql"; \
+		$(CMAKE_PRG) --build $(THIRD_PARTY_BUILD_DIR) --target zetasql; \
+	    fi;  \
+	    if [ -n "$(THIRDPARTY_VERSION)" ]; then \
+		if [ "$(new_third_version)" != "$(THIRDPARTY_VERSION)" ] ; then \
+		    echo "[deps]: installing thirdparty from $(THIRDPARTY_VERSION) to $(new_third_version)"; \
+		    $(CMAKE_PRG) --build $(THIRD_PARTY_BUILD_DIR) --target hybridsql-asserts; \
+		else \
+		    echo "[deps]: thirdparty up-to-date: $(THIRDPARTY_VERSION)"; \
+		fi ; \
+	    else \
+		echo "[deps]: installing latest thirdparty"; \
+		$(CMAKE_PRG) --build $(THIRD_PARTY_BUILD_DIR) --target hybridsql-asserts; \
+	    fi ; \
 	fi
 
 # third party compiled code install to 'OpenMLDB/.deps/usr', source code install to 'OpenMLDB/thirdsrc'

diff --git a/docker/Dockerfile b/docker/Dockerfile
@@ -15,8 +15,8 @@
 
 FROM centos:7
 
-ARG ZETASQL_VERSION=0.3.0
-ARG THIRDPARTY_VERSION=0.5.2
+ARG ZETASQL_VERSION=0.3.1
+ARG THIRDPARTY_VERSION=0.6.0
 ARG TARGETARCH
 
 LABEL org.opencontainers.image.source https://github.com/4paradigm/OpenMLDB
@@ -28,8 +28,6 @@ RUN yum update -y && yum install -y centos-release-scl epel-release && \
     curl -Lo lcov-1.15-1.noarch.rpm https://github.com/linux-test-project/lcov/releases/download/v1.15/lcov-1.15-1.noarch.rpm && \
     yum localinstall -y lcov-1.15-1.noarch.rpm && \
     yum clean all && rm -v lcov-1.15-1.noarch.rpm && \
-    curl -Lo apache-maven-3.6.3-bin.tar.gz https://mirrors.ocf.berkeley.edu/apache/maven/maven-3/3.6.3/binaries/apache-maven-3.6.3-bin.tar.gz && \
-    tar xzf apache-maven-3.6.3-bin.tar.gz -C /usr/local --strip-components=1 && \
     curl -Lo zookeeper.tar.gz https://archive.apache.org/dist/zookeeper/zookeeper-3.4.14/zookeeper-3.4.14.tar.gz && \
     mkdir -p /deps/src && \
     tar xzf zookeeper.tar.gz -C /deps/src && \

diff --git a/src/nameserver/name_server_impl.h b/src/nameserver/name_server_impl.h
@@ -111,7 +111,12 @@ class NameServerImpl : public NameServer {
     NameServerImpl();
 
     ~NameServerImpl() override;
-
+    void CloseThreadpool() {
+        running_.store(false, std::memory_order_release);
+        thread_pool_.Stop(true);
+        task_thread_pool_.Stop(true);
+        UpdateTableStatus();
+    }
     bool Init(const std::string& real_endpoint);
     bool Init(const std::string& zk_cluster, const std::string& zk_path, const std::string& endpoint,
               const std::string& real_endpoint);

diff --git a/src/sdk/mini_cluster.h b/src/sdk/mini_cluster.h
@@ -105,7 +105,7 @@ class MiniCluster {
             }
         }
         sleep(4);
-        ::openmldb::nameserver::NameServerImpl* nameserver = new ::openmldb::nameserver::NameServerImpl();
+        nameserver = new ::openmldb::nameserver::NameServerImpl();
         bool ok = nameserver->Init(zk_cluster_, zk_path_, ns_endpoint, "");
         if (!ok) {
             return false;
@@ -135,6 +135,7 @@ class MiniCluster {
     }
 
     void Close() {
+        nameserver->CloseThreadpool();
         ns_.Stop(10);
         ns_.Join();
 
@@ -207,7 +208,7 @@ class MiniCluster {
         tb_clients_.emplace(tb_endpoint, client);
         return true;
     }
-
+    ::openmldb::nameserver::NameServerImpl* nameserver;
     int32_t zk_port_;
     brpc::Server ns_;
     int32_t tablet_num_;
@@ -250,7 +251,7 @@ class StandaloneEnv {
         FLAGS_sync_deploy_stats_timeout = 2000;
         ns_port_ = GenRand();
         std::string ns_endpoint = "127.0.0.1:" + std::to_string(ns_port_);
-        ::openmldb::nameserver::NameServerImpl* nameserver = new ::openmldb::nameserver::NameServerImpl();
+        nameserver = new ::openmldb::nameserver::NameServerImpl();
         bool ok = nameserver->Init("", "", ns_endpoint, "");
         if (!ok) {
             return false;
@@ -278,6 +279,7 @@ class StandaloneEnv {
     }
 
     void Close() {
+        nameserver->CloseThreadpool();
         ns_.Stop(10);
         ns_.Join();
         tb_server_.Stop(10);
@@ -323,7 +325,7 @@ class StandaloneEnv {
         tb_client_ = client;
         return true;
     }
-
+    ::openmldb::nameserver::NameServerImpl* nameserver;
     brpc::Server ns_;
     brpc::Server tb_server_;
     std::string ns_endpoint_;

diff --git a/third-party/CMakeLists.txt b/third-party/CMakeLists.txt
@@ -68,7 +68,7 @@ set(MAKEOPTS "$ENV{MAKEOPTS}" CACHE STRING "Extra options to make")
 message(STATUS "Install bundled dependencies into ${DEPS_INSTALL_DIR}")
 
 set(HYBRIDSQL_ASSERTS_HOME https://github.com/4paradigm/hybridsql-asserts)
-set(HYBRIDSQL_ASSERTS_VERSION 0.5.2)
+set(HYBRIDSQL_ASSERTS_VERSION 0.6.0)
 
 function(get_linux_lsb_release_information)
   execute_process(COMMAND bash ${CMAKE_SOURCE_DIR}/get-lsb-release.sh
@@ -90,17 +90,17 @@ function(init_hybridsql_thirdparty_urls)
     else()
       if (LSB_RELEASE_ID_SHORT STREQUAL "centos")
         set(HYBRIDSQL_ASSERTS_URL "${HYBRIDSQL_ASSERTS_HOME}/releases/download/v${HYBRIDSQL_ASSERTS_VERSION}/thirdparty-${HYBRIDSQL_ASSERTS_VERSION}-linux-gnu-x86_64-centos.tar.gz" PARENT_SCOPE)
-        set(HYBRIDSQL_ASSERTS_HASH 919ee7aee4c89846f4e242530519b3c34a34567ddcf9f4361d413a44e2f7408c PARENT_SCOPE)
+        set(HYBRIDSQL_ASSERTS_HASH c415dfdc95a127cdce888aec84c7fa3c02f3c9cb973805dcf23b54517e422e36 PARENT_SCOPE)
       elseif(LSB_RELEASE_ID_SHORT STREQUAL "ubuntu")
         set(HYBRIDSQL_ASSERTS_URL "${HYBRIDSQL_ASSERTS_HOME}/releases/download/v${HYBRIDSQL_ASSERTS_VERSION}/thirdparty-${HYBRIDSQL_ASSERTS_VERSION}-linux-gnu-x86_64-ubuntu.tar.gz" PARENT_SCOPE)
-        set(HYBRIDSQL_ASSERTS_HASH 8bb1f7685bf778539e1f4ba499020504ebc89e8cefa9a294aa0122578ca70716 PARENT_SCOPE)
+        set(HYBRIDSQL_ASSERTS_HASH 8c95b5fd539c8362d934ae58879d9ae1c27bc0977ca09cc8316ba207e8aaaf1e PARENT_SCOPE)
       else()
         message(FATAL_ERROR "no pre-compiled thirdparty for your operation system, try compile thirdparty from source with '-DBUILD_BUNDLED=ON'")
       endif()
     endif()
   elseif(CMAKE_SYSTEM_NAME STREQUAL "Darwin")
     set(HYBRIDSQL_ASSERTS_URL "${HYBRIDSQL_ASSERTS_HOME}/releases/download/v${HYBRIDSQL_ASSERTS_VERSION}/thirdparty-${HYBRIDSQL_ASSERTS_VERSION}-darwin-i386.tar.gz" PARENT_SCOPE)
-    set(HYBRIDSQL_ASSERTS_HASH 663b0d945c95034b1e17411f3e795f98053bf248860a60025c7802634ce526d8 PARENT_SCOPE)
+    set(HYBRIDSQL_ASSERTS_HASH 062e606f1d76fe27003bdc23e643305bfa032eadec8c075e7ce6dc22d70f5044 PARENT_SCOPE)
   endif()
 endfunction()