meta/sql: support new backup (#5344)

Signed-off-by: jiefenghuang <[email protected]>
juicedata · Dec 5, 2024 · 607ec6f · 607ec6f
1 parent a33d9bb
commit 607ec6f
Show file tree

Hide file tree

Showing 16 changed files with 3,884 additions and 122 deletions.
diff --git a/.github/scripts/prepare_db.sh b/.github/scripts/prepare_db.sh
@@ -1,39 +1,40 @@
-#!/bin/bash -e 
+#!/bin/bash -e
 source .github/scripts/start_meta_engine.sh
 [ -z "$TEST" ] && echo "TEST is not set" && exit 1
 
-# check port is ready until 60s, sleep 1s for each query 
-check_port(){
+# check port is ready until 60s, sleep 1s for each query
+check_port() {
     port=$1
     echo "check for port:" $port
     for i in {1..30}; do
-        sudo lsof -i :$port && echo "port is available: $port after $i sec" && return 0 \
-        || (echo "port is not available after $i" && sleep 1)
+        sudo lsof -i :$port && echo "port is available: $port after $i sec" && return 0 ||
+            (echo "port is not available after $i" && sleep 1)
     done
     echo "service not ready on: $port" && exit 1
 }
 
-install_mysql(){
+install_mysql() {
     sudo service mysql start
     sudo mysql -uroot -proot -e "use mysql;alter user 'root'@'localhost' identified with mysql_native_password by '';"
     sudo mysql -e "create database dev;"
+    sudo mysql -e "create database dev2;"
     check_port 3306
 }
 
-install_postgres(){
+install_postgres() {
     sudo service postgresql start
     sudo chmod 777 /etc/postgresql/*/main/pg_hba.conf
-    sudo sed  -i "s?local.*all.*postgres.*peer?local   all             postgres                                trust?" /etc/postgresql/*/main/pg_hba.conf
-    sudo sed  -i "s?host.*all.*all.*32.*scram-sha-256?host    all             all             127.0.0.1/32            trust?" /etc/postgresql/*/main/pg_hba.conf
-    sudo sed  -i "s?host.*all.*all.*128.*scram-sha-256?host    all             all             ::1/128                 trust?" /etc/postgresql/*/main/pg_hba.conf
+    sudo sed -i "s?local.*all.*postgres.*peer?local   all             postgres                                trust?" /etc/postgresql/*/main/pg_hba.conf
+    sudo sed -i "s?host.*all.*all.*32.*scram-sha-256?host    all             all             127.0.0.1/32            trust?" /etc/postgresql/*/main/pg_hba.conf
+    sudo sed -i "s?host.*all.*all.*128.*scram-sha-256?host    all             all             ::1/128                 trust?" /etc/postgresql/*/main/pg_hba.conf
     cat /etc/postgresql/*/main/pg_hba.conf
     sudo service postgresql restart
     psql -c "create user runner superuser;" -U postgres
     sudo service postgresql restart
     psql -c 'create database test;' -U postgres
 }
 
-install_etcd(){
+install_etcd() {
     docker run -d \
         -p 3379:2379 \
         -p 3380:2380 \
@@ -49,29 +50,29 @@ install_etcd(){
     check_port 3380
 }
 
-install_keydb(){
+install_keydb() {
     echo "deb https://download.keydb.dev/open-source-dist $(lsb_release -sc) main" | sudo tee /etc/apt/sources.list.d/keydb.list
     sudo wget -O /etc/apt/trusted.gpg.d/keydb.gpg https://download.keydb.dev/open-source-dist/keyring.gpg
     sudo .github/scripts/apt_install.sh keydb
-    keydb-server --storage-provider  flash /tmp/ --port 6378 --bind 127.0.0.1 --daemonize yes
+    keydb-server --storage-provider flash /tmp/ --port 6378 --bind 127.0.0.1 --daemonize yes
     keydb-server --port 6377 --bind 127.0.0.1 --daemonize yes
-    check_port 6377 
+    check_port 6377
     check_port 6378
 }
 
-install_minio(){
+install_minio() {
     docker run -d -p 9000:9000 -p 9001:9001 -e "MINIO_ROOT_USER=testUser" -e "MINIO_ROOT_PASSWORD=testUserPassword" quay.io/minio/minio:RELEASE.2022-01-25T19-56-04Z server /data --console-address ":9001"
     go install github.com/minio/[email protected] && mc config host add local http://127.0.0.1:9000 testUser testUserPassword && mc mb local/testbucket
 }
 
-install_fdb(){
+install_fdb() {
     wget -O /home/travis/.m2/foundationdb-clients_6.3.23-1_amd64.deb https://github.com/apple/foundationdb/releases/download/6.3.23/foundationdb-clients_6.3.23-1_amd64.deb
     wget -O /home/travis/.m2/foundationdb-server_6.3.23-1_amd64.deb https://github.com/apple/foundationdb/releases/download/6.3.23/foundationdb-server_6.3.23-1_amd64.deb
     sudo dpkg -i /home/travis/.m2/foundationdb-clients_6.3.23-1_amd64.deb /home/travis/.m2/foundationdb-server_6.3.23-1_amd64.deb
     check_port 4500
 }
 
-install_gluster(){
+install_gluster() {
     sudo systemctl start glusterd.service
     mkdir -p /tmp/gluster/gv0
     sudo hostname jfstest
@@ -80,29 +81,29 @@ install_gluster(){
     sudo gluster volume info gv0
 }
 
-install_litmus(){
+install_litmus() {
     wget -O /home/travis/.m2/litmus-0.13.tar.gz http://www.webdav.org/neon/litmus/litmus-0.13.tar.gz
     tar -zxvf /home/travis/.m2/litmus-0.13.tar.gz -C /home/travis/.m2/
-    cd  /home/travis/.m2/litmus-0.13/ && ./configure && make && cd -
+    cd /home/travis/.m2/litmus-0.13/ && ./configure && make && cd -
 }
 
-install_webdav(){
+install_webdav() {
     wget -O /home/travis/.m2/rclone-v1.57.0-linux-amd64.zip --no-check-certificate https://downloads.rclone.org/v1.57.0/rclone-v1.57.0-linux-amd64.zip
     unzip /home/travis/.m2/rclone-v1.57.0-linux-amd64.zip -d /home/travis/.m2/
-    nohup /home/travis/.m2/rclone-v1.57.0-linux-amd64/rclone serve webdav local --addr 127.0.0.1:9007 >> rclone.log 2>&1 &
+    nohup /home/travis/.m2/rclone-v1.57.0-linux-amd64/rclone serve webdav local --addr 127.0.0.1:9007 >>rclone.log 2>&1 &
 }
 
-prepare_db(){
+prepare_db() {
     case "$TEST" in
     "test.meta.core")
         retry install_tikv
         install_mysql
-    ;;
+        ;;
     "test.meta.non-core")
         install_postgres
         install_etcd
         install_keydb
-    ;;
+        ;;
     "test.cmd")
         install_minio
         install_litmus
@@ -116,7 +117,7 @@ prepare_db(){
         install_minio
         install_gluster
         install_webdav
-        docker run -d --name sftp -p 2222:22  juicedata/ci-sftp
+        docker run -d --name sftp -p 2222:22 juicedata/ci-sftp
         install_etcd
         .github/scripts/setup-hdfs.sh
         ;;
@@ -127,5 +128,3 @@ prepare_db(){
 }
 
 prepare_db
-
-
diff --git a/go.mod b/go.mod
@@ -75,7 +75,7 @@ require (
 	golang.org/x/crypto v0.21.0
 	golang.org/x/net v0.23.0
 	golang.org/x/oauth2 v0.7.0
-	golang.org/x/sync v0.2.0
+	golang.org/x/sync v0.9.0
 	golang.org/x/sys v0.18.0
 	golang.org/x/term v0.18.0
 	golang.org/x/text v0.14.0

diff --git a/go.sum b/go.sum
@@ -1070,8 +1070,8 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI=
-golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.9.0 h1:fEo0HyrW1GIgZdpbhCRO0PkJajUS5H9IFUztCgEo2jQ=
+golang.org/x/sync v0.9.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20180622082034-63fc586f45fe/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=