repo-sync-2024-05-08T11:29:43+0800

.bazelrc

@@ -56,6 +56,9 @@ build:ubsan --linkopt -fsanitize=undefined
 build:macos-asan --features=asan
 build:macos-ubsan --features=ubsan
 
+# OPENSOURCE-CLEANUP REMOVE BEGIN
+build --remote_cache=http://bazelcache-pool.cz00b.alipay.net:8080
+# OPENSOURCE-CLEANUP REMOVE END
 
 test --keep_going
 test --test_output=errors

.circleci/config.yml

@@ -52,7 +52,7 @@ commands:
             esac
 
             GOLANG_DIR="/usr/local"
-            GOLANG_VERSION="1.19.7"
+            GOLANG_VERSION="1.19.17"
             GOLANG_URL="https://golang.org/dl/go${GOLANG_VERSION}.linux-${ARCH}.tar.gz"
 
             if ! command -v go &> /dev/null; then
@@ -79,7 +79,7 @@ commands:
             esac
 
             GOLANG_DIR="/usr/local"
-            GOLANG_VERSION="1.19.7"
+            GOLANG_VERSION="1.19.17"
             GOLANG_URL="https://golang.org/dl/go${GOLANG_VERSION}.linux-${ARCH}.tar.gz"
 
             if ! command -v go &> /dev/null; then

.circleci/deps-config.yml

@@ -47,7 +47,7 @@ commands:
             esac
 
             GOLANG_DIR="/opt"
-            GOLANG_VERSION="1.19.7"
+            GOLANG_VERSION="1.22.2"
             GOLANG_URL="https://golang.org/dl/go${GOLANG_VERSION}.linux-${ARCH}.tar.gz"
 
             if ! command -v go &> /dev/null; then

Makefile

@@ -15,8 +15,10 @@ IMG := secretflow/kuscia:${TAG}
 # TEST_SUITE used by integration test
 TEST_SUITE ?= all
 
-ENVOY_IMAGE ?= secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/kuscia-envoy:0.4.0.dev20240407
-DEPS_IMAGE ?= secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/kuscia-deps:0.5.0b0
+# OPENSOURCE-CLEANUP SUB reg.docker.alibaba-inc.com/secretflow/kuscia-envoy:0.6.x.dev20240424 secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/kuscia-envoy:0.4.0.dev20240407
+ENVOY_IMAGE ?= reg.docker.alibaba-inc.com/secretflow/kuscia-envoy:0.6.x.dev20240424
+# OPENSOURCE-CLEANUP SUB reg.docker.alibaba-inc.com/secretflow/kuscia-deps:0.2.0 secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/kuscia-deps:0.5.0b0
+DEPS_IMAGE ?= reg.docker.alibaba-inc.com/secretflow/kuscia-deps:0.2.0
 
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
@@ -142,3 +144,16 @@ integration_test: image ## Run Integration Test
 	mkdir -p run/test
 	cd run && KUSCIA_IMAGE=${IMG} docker run --rm ${IMG} cat /home/kuscia/scripts/test/integration_test.sh > ./test/integration_test.sh && chmod u+x ./test/integration_test.sh
 	cd run && KUSCIA_IMAGE=${IMG} ./test/integration_test.sh ${TEST_SUITE}
+
+# OPENSOURCE-CLEANUP REMOVE BEGIN
+
+.PHONY: alios-base-image
+alios-base-image:
+	docker build -t alios7u2-py-base:${DATETIME} --platform linux/amd64 -f ./build/dockerfile/base/kuscia-alios-base.Dockerfile .
+
+.PHONY: alios-image
+alios-image: export GOOS=linux
+alios-image: export GOARCH=amd64
+alios-image: build
+	docker build -t ${IMG} --build-arg KUSCIA_ENVOY_IMAGE=${ENVOY_IMAGE} --build-arg DEPS_IMAGE=${DEPS_IMAGE} -f ./build/dockerfile/kuscia-alios.Dockerfile .
+# OPENSOURCE-CLEANUP REMOVE END

MetaPigeon.toml

@@ -0,0 +1,2 @@
+# OPENSOURCE-CLEANUP DELETE_FILE
+BlackList = ["BE1054006"]

OWNERS

@@ -0,0 +1,3 @@
+# OPENSOURCE-CLEANUP DELETE_FILE
+reviewers: ["wangzijie.wzj", "xiaol.huxl", "dongwen.hdw", "guoshilei.gsl", "caochen.cao", "yansi.zy", "jiamingyang.jmy", "lilinchuan.zwz","zhanhanbing.zhb","zhuangzhuangyan.yz","yj01523683"]
+threshold: 2

WORKSPACE

@@ -3,6 +3,8 @@ workspace(name = "kuscia")
 load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
 load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
 
+# OPENSOURCE-CLEANUP REMOVE LINE_WITH_KEYWORD DEPS_FAST_MIRROR
+DEPS_FAST_MIRROR = "https://antsys-nebula-archive.cn-hangzhou-alipay-b.oss-cdn.aliyun-inc.com/third_party"
 
 maybe(
     http_archive,

build/dockerfile/base/kuscia-alios-base.Dockerfile

@@ -0,0 +1,69 @@
+# OPENSOURCE-CLEANUP DELETE_FILE
+# latest image version: reg.docker.alibaba-inc.com/nueva-stack/alios7u2-py-base:0.20
+
+FROM openanolis/anolisos:8.4-x86_64 AS build
+
+RUN yum groupinstall -y "Development Tools" && yum install -y wget fuse fuse-devel fuse3 fuse3-devel
+RUN wget -O unionfs-fuse.tgz https://github.com/rpodgorny/unionfs-fuse/archive/refs/tags/v2.1.tar.gz && \
+    tar xf unionfs-fuse.tgz && cd unionfs-fuse-2.1/ && make && make install
+
+
+FROM reg.docker.alibaba-inc.com/alipay/7u2-common:202202.0T
+
+LABEL maintainer="[email protected]"
+
+ENV TZ=Asia/Shanghai
+
+USER root
+WORKDIR /home/admin
+
+COPY --from=build /usr/local/bin/unionfs* /usr/local/bin/
+
+# update glibc to 2.30
+# https://yuque.antfin-inc.com/baseos/manual/gcc92
+
+RUN yum clean all && yum -y install glibc-devel && \
+    yum -y update util-linux bash-completion gnupg2 python perl systemd && \
+    yum -y install alios7u-2_30-gcc-9-repo.noarch && \
+    yum -y install libdb binutils && \
+    yum -y install glibc glibc-langpack-en && \
+    yum -y install glibc-devel libstdc++-devel libstdc++-static libstdc++ libasan && \
+    yum clean all
+
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+
+RUN yum install -y wget && \
+    wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo && \
+    wget -O /etc/yum.repos.d/openresty.repo https://openresty.org/package/centos/openresty.repo && \
+    yum install --setopt=skip_missing_names_on_install=False -y \
+        net-tools-2.0 epel-release tar iproute sudo logrotate \
+        tcpdump less rsync tree which \
+        zlib-devel libffi-devel openssl-devel fuse fuse-devel fuse3 fuse3-devel fuse-overlayfs perl perl-Digest-MD5 python3 java openresty openresty-opm  && \
+    yum clean all
+
+RUN opm get knyar/nginx-lua-prometheus ledgetech/lua-resty-http cdbattags/lua-resty-jwt && \
+    wget -O /usr/bin/ossutil64 http://gosspublic.alicdn.com/ossutil/1.6.0/ossutil64 && \
+    chmod 0755 /usr/bin/ossutil64
+
+RUN pushd /usr/bin/ && rm -rf python && ln -s python3 python && popd && \
+    sed -i 's/\/usr\/bin\/python/\/usr\/bin\/python2.7/g' /usr/bin/yum /usr/libexec/urlgrabber-ext-down && \
+    yum install -y python3-devel
+
+RUN pip3 install --upgrade pip && \
+    pip3 install cryptography==3.3.2 paramiko scp
+
+RUN yum install -y libxml2-devel
+
+RUN pushd /tmp && \
+    ln -s /usr/lib64/liblua-5.1.so /usr/lib64/liblua.so && \
+    wget https://mpc-antbase.oss-cn-hangzhou.aliyuncs.com/release/linux/acjail/t-aliyun-acjail-2.0.1.an8.x86_64.rpm && \
+    rpm -ivh t-aliyun-acjail-2.0.1.an8.x86_64.rpm && \
+    sed -i '/blacklist_ip = {/c\blacklist_ip = blacklist_ip or {}' /usr/local/aliyun_acjail/config.lua && \
+    rm -rfv t-aliyun-acjail-2.0.1.an8.x86_64.rpm && popd
+
+RUN pushd /tmp && wget https://bindfs.org/downloads/bindfs-1.15.1.tar.gz --no-check-certificate && \
+    tar xf bindfs-1.15.1.tar.gz && cd bindfs-1.15.1 && ./configure && make && make install && \
+    cd .. && rm -rfv bindfs* && popd
+
+RUN yum -y remove openssh && rm -rf /etc/runit/sshd && \
+    yum -y remove telnet

build/dockerfile/base/kuscia-deps.Dockerfile

@@ -1,3 +1,6 @@
+# OPENSOURCE-CLEANUP REMOVE 2
+# latest image version: reg.docker.alibaba-inc.com/secretflow/kuscia-deps:0.2.0
+# changelog: upgrade k3s from v1.26.6-k3s1 to v1.26.11-k3s2
 ARG K3S_VER=v1.26.11-k3s2
 ARG K3S_IMAGE=rancher/k3s:${K3S_VER}
 FROM ${K3S_IMAGE} as k3s-image

build/dockerfile/dev/kuscia-dev.Dockerfile

@@ -0,0 +1,34 @@
+# latest image version: reg.docker.alibaba-inc.com/secretflow/kuscia-dev:0.6
+
+FROM reg.docker.alibaba-inc.com/nueva-stack/dev-base:0.3
+
+LABEL maintainer="[email protected]"
+
+ENV TZ=Asia/Shanghai
+
+RUN yum update -y \
+    && yum clean all
+
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone && \
+    yum install -y wget sudo less epel-release net-tools iproute git jq tree unzip gcc perl perl-Digest-MD5 git-lfs which && \
+    yum clean all && \
+    wget -O /usr/bin/ossutil64 http://gosspublic.alicdn.com/ossutil/1.6.0/ossutil64 && \
+    chmod 0755 /usr/bin/ossutil64 && \
+    wget https://github.com/protocolbuffers/protobuf/releases/download/v21.8/protoc-21.8-linux-x86_64.zip && \
+    unzip protoc-21.8-linux-x86_64.zip -d /usr/local && chgrp -R users /usr/local/include/google /usr/local/bin/protoc && cd ../ && \
+    wget http://mpc-antbase.oss-cn-hangzhou.aliyuncs.com/influxdb/influxd -O /usr/local/bin/influxd && chmod 755 /usr/local/bin/influxd  && \
+    wget https://mpc-antbase.oss-cn-hangzhou.aliyuncs.com/mysql-8.0.28-lite/mysql.tar.gz  && tar -xvf mysql.tar.gz -C  /usr/local && chmod 755 /usr/local/bin/mysqld
+
+RUN wget https://golang.google.cn/dl/go1.19.10.linux-amd64.tar.gz -O - | tar -xz -C /usr/local && \
+    echo export GOPROXY=https://goproxy.cn,direct >> ~/.bashrc && \
+    echo export GO111MODULE=on >> ~/.bashrc && \
+    echo export GOPATH=$HOME/gopath >> ~/.bashrc && \
+    echo export PATH=$PATH:/usr/local/go/bin:'$GOPATH'/bin >> ~/.bashrc && \
+    echo export GOPRIVATE=gitlab.alipay-inc.com >> ~/.bashrc && \
+    source ~/.bashrc && \
+    ssh-keyscan -t rsa gitlab.alipay-inc.com >> ~/.ssh/known_hosts && \
+    git config --global url."[email protected]:".insteadOf "http://gitlab.alipay-inc.com/" && \
+    go install google.golang.org/protobuf/cmd/[email protected] && \
+    go install github.com/t-yuki/gocover-cobertura@latest && \
+    go install github.com/jstemmer/go-junit-report/v2@latest && \
+    go install github.com/golangci/golangci-lint/cmd/[email protected]

build/dockerfile/kuscia-alios.Dockerfile

@@ -0,0 +1,52 @@
+# OPENSOURCE-CLEANUP DELETE_FILE
+ARG DEPS_IMAGE="reg.docker.alibaba-inc.com/secretflow/kuscia-deps:0.2.0"
+ARG KUSCIA_ENVOY_IMAGE="reg.docker.alibaba-inc.com/secretflow/kuscia-envoy:0.5.0b0"
+ARG PROM_NODE_EXPORTER="prom/node-exporter:v1.7.0"
+
+FROM ${DEPS_IMAGE} as deps
+FROM ${PROM_NODE_EXPORTER} as node_exporter
+FROM ${KUSCIA_ENVOY_IMAGE} as kuscia_envoy
+
+FROM reg.docker.alibaba-inc.com/nueva-stack/alios7u2-py-base:0.20
+
+ENV TZ=Asia/Shanghai
+
+ARG ROOT_DIR="/home/kuscia"
+RUN yum install -y openssl net-tools which jq logrotate && \
+    yum clean all && \
+    mkdir -p ${ROOT_DIR}/bin && \
+    mkdir -p /bin/aux && \
+    mkdir -p ${ROOT_DIR}/scripts && \
+    mkdir -p ${ROOT_DIR}/var/storage && \
+    mkdir -p ${ROOT_DIR}/pause
+
+COPY --from=deps /image/home/kuscia/bin ${ROOT_DIR}/bin
+COPY --from=deps /image/bin/aux /bin/aux
+COPY --from=node_exporter /bin/node_exporter ${ROOT_DIR}/bin
+RUN pushd ${ROOT_DIR}/bin && \
+    ln -s k3s crictl && \
+    ln -s k3s ctr && \
+    ln -s k3s kubectl && \
+    ln -s cni bridge && \
+    ln -s cni flannel && \
+    ln -s cni host-local && \
+    ln -s cni loopback && \
+    ln -s cni portmap && \
+    popd
+
+COPY build/apps/kuscia/kuscia ${ROOT_DIR}/bin
+COPY build/pause/pause-amd64.tar ${ROOT_DIR}/pause/pause.tar
+COPY crds/v1alpha1 ${ROOT_DIR}/crds/v1alpha1
+COPY etc ${ROOT_DIR}/etc
+COPY testdata ${ROOT_DIR}/var/storage/data
+COPY scripts ${ROOT_DIR}/scripts
+
+COPY thirdparty/*/scripts ${ROOT_DIR}/scripts
+
+COPY --from=kuscia_envoy /home/kuscia/bin/envoy ${ROOT_DIR}/bin
+
+ENV PATH="${PATH}:${ROOT_DIR}/bin:/bin/aux"
+
+WORKDIR ${ROOT_DIR}
+
+ENTRYPOINT ["tini", "--"]

build/dockerfile/kuscia-anolis.Dockerfile

@@ -1,5 +1,7 @@
-ARG DEPS_IMAGE="secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/kuscia-deps:0.5.0b0"
-ARG KUSCIA_ENVOY_IMAGE="secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/kuscia-envoy:0.6.0.dev20240507"
+# OPENSOURCE-CLEANUP SUB reg.docker.alibaba-inc.com/secretflow/kuscia-deps:0.2.0 secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/kuscia-deps:0.5.0b0
+ARG DEPS_IMAGE="reg.docker.alibaba-inc.com/secretflow/kuscia-deps:0.2.0"
+# OPENSOURCE-CLEANUP SUB reg.docker.alibaba-inc.com/secretflow/kuscia-envoy:0.5.0b0 secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/kuscia-envoy:0.4.0.dev20240407
+ARG KUSCIA_ENVOY_IMAGE="reg.docker.alibaba-inc.com/secretflow/kuscia-envoy:0.5.0b0"
 ARG PROM_NODE_EXPORTER="prom/node-exporter:v1.7.0"
 
 FROM ${DEPS_IMAGE} as deps

build/dockerfile/kuscia-secretflow.Dockerfile

@@ -12,7 +12,7 @@ RUN yum install -y protobuf libnl3 libgomp && \
     grep -rl '#!/root/miniconda3/envs/secretflow/bin' /usr/local/bin/ | xargs sed -i -e 's/#!\/root\/miniconda3\/envs\/secretflow/#!\/usr\/local/g' && \
     rm /usr/local/bin/openssl
 
-ARG SF_VERSION="1.5.0b0"
+ARG SF_VERSION="1.6.0b0"
 RUN pip install secretflow-lite==${SF_VERSION} --extra-index-url https://mirrors.aliyun.com/pypi/simple/ && rm -rf /root/.cache
 
 RUN kuscia image builtin secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/secretflow-lite-anolis8:${SF_VERSION} --store /home/kuscia/var/images

cmd/kuscia/confloader/secretbackendloader.go

@@ -22,6 +22,9 @@ import (
 	"github.com/secretflow/kuscia/pkg/secretbackend"
 	// register driver
 	_ "github.com/secretflow/kuscia/pkg/secretbackend/mem"
+	// OPENSOURCE-CLEANUP REMOVE 2
+	// register mist driver
+	_ "github.com/secretflow/kuscia/pkg/secretbackend/mist"
 )
 
 type SecretBackendParams struct {

cmd/kusciastorage/main.go

@@ -0,0 +1,27 @@
+// Copyright 2023 Ant Group Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"github.com/secretflow/kuscia/pkg/kusciastorage"
+	"github.com/secretflow/kuscia/pkg/utils/nlog"
+)
+
+func main() {
+	err := kusciastorage.RunKusciaStorage()
+	if err != nil {
+		nlog.Errorf("Run kuscia storage service failed: %v", err.Error())
+	}
+}

cmd/webdemo/TestBean.go

@@ -0,0 +1,61 @@
+// Copyright 2023 Ant Group Co., Ltd.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"context"
+	"strconv"
+
+	"github.com/secretflow/kuscia/pkg/utils/nlog"
+	"github.com/secretflow/kuscia/pkg/web/asserts"
+	"github.com/secretflow/kuscia/pkg/web/errorcode"
+	"github.com/secretflow/kuscia/pkg/web/framework"
+	"github.com/secretflow/kuscia/pkg/web/framework/config"
+)
+
+type TestBean struct {
+	framework.ConfigLoader
+	// config
+	DataStoreType string `name:"data-store-type" usage:"Data Store Type" required:""`
+	RPCPort       int    `name:"rpc-port" usage:"rpc-port" default:"8118"`
+
+	DataStoreRPCPort string
+	DemoConfig       *DemoConfig
+}
+
+func NewTestBean(c *DemoConfig) *TestBean {
+	return &TestBean{
+		ConfigLoader: &config.FlagEnvConfigLoader{Source: config.SourceFlag},
+		DemoConfig:   c,
+	}
+}
+
+func (b *TestBean) Validate(errs *errorcode.Errs) {
+	errs.AppendErr(asserts.NotEmpty(b.DataStoreType, "DataStoreType is empty"))
+	errs.AppendErr(asserts.IsTrue(b.RPCPort > 0, "RpcPort is invalid"))
+}
+
+func (b *TestBean) Init(e framework.ConfBeanRegistry) error {
+	b.DataStoreRPCPort = b.DataStoreType + strconv.Itoa(b.RPCPort)
+	// TODO Init ...
+	nlog.Info("Init: " + b.DataStoreRPCPort)
+	return nil
+}
+
+func (b *TestBean) Start(ctx context.Context, e framework.ConfBeanRegistry) error {
+	nlog.Info("Start" + b.DataStoreRPCPort)
+	b.DemoConfig.Print()
+	return nil
+}