From de64eb4a3098a064084e2574eb6be6f94be3cafb Mon Sep 17 00:00:00 2001 From: yj01523683 Date: Wed, 8 May 2024 11:29:59 +0800 Subject: [PATCH] repo-sync-2024-05-08T11:29:43+0800 --- .aci.yml | 450 ++++++++++++++++++ .bazelrc | 3 + .circleci/config.yml | 4 +- .circleci/deps-config.yml | 2 +- Makefile | 19 +- MetaPigeon.toml | 2 + OWNERS | 3 + WORKSPACE | 2 + .../base/kuscia-alios-base.Dockerfile | 69 +++ build/dockerfile/base/kuscia-deps.Dockerfile | 3 + .../dockerfile/dev/.OPENSOURCE-CLEANUP-RMDIR | 0 build/dockerfile/dev/kuscia-dev.Dockerfile | 34 ++ build/dockerfile/kuscia-alios.Dockerfile | 52 ++ build/dockerfile/kuscia-anolis.Dockerfile | 6 +- build/dockerfile/kuscia-secretflow.Dockerfile | 2 +- cmd/kuscia/confloader/secretbackendloader.go | 3 + cmd/kusciastorage/.OPENSOURCE-CLEANUP-RMDIR | 0 cmd/kusciastorage/main.go | 27 ++ cmd/webdemo/.OPENSOURCE-CLEANUP-RMDIR | 0 cmd/webdemo/TestBean.go | 61 +++ cmd/webdemo/any.go | 41 ++ cmd/webdemo/api.pb.go | 409 ++++++++++++++++ cmd/webdemo/api.proto | 41 ++ cmd/webdemo/demo.go | 116 +++++ cmd/webdemo/demo_config.go | 45 ++ cmd/webdemo/hello.go | 57 +++ cmd/webdemo/login.go | 67 +++ docs/development/register_custom_image.md | 33 +- docs/getting_started/run_secretflow_cn.md | 52 +- docs/reference/apis/kusciajob_cn.md | 14 +- docs/reference/concepts/kusciajob_cn.md | 28 +- docs/reference/concepts/kusciatask_cn.md | 163 +------ docs/tutorial/run_sf_job_with_api_cn.md | 35 +- go.mod | 33 +- go.sum | 147 +++++- hack/k3s/README.md | 40 ++ hack/k3s/build.sh | 3 +- internaldoc/.OPENSOURCE-CLEANUP-RMDIR | 0 internaldoc/README.md | 112 +++++ internaldoc/app_access_profile.md | 329 +++++++++++++ internaldoc/docs_tutorial.md | 62 +++ internaldoc/quick_start.md | 135 ++++++ .../provider/node/capacity_manager_test.go | 4 +- pkg/agent/provider/pod/kubebackend/sigma.go | 178 +++++++ .../provider/pod/kubebackend/sigma_test.go | 68 +++ pkg/client/.OPENSOURCE-CLEANUP-RMDIR | 0 pkg/client/kusciastorage/rest/client.go | 180 +++++++ pkg/client/kusciastorage/rest/client_test.go | 249 ++++++++++ pkg/client/kusciastorage/rest/config.go | 78 +++ pkg/client/kusciastorage/rest/config_test.go | 142 ++++++ pkg/client/kusciastorage/rest/utils.go | 136 ++++++ pkg/client/kusciastorage/rest/utils_test.go | 256 ++++++++++ pkg/confmanager/commands/root.go | 3 + .../kusciajob/handler/scheduler.go | 2 + pkg/gateway/utils/transit_test.go | 93 ++-- pkg/kusciastorage/.OPENSOURCE-CLEANUP-RMDIR | 0 pkg/kusciastorage/beans/beans.go | 45 ++ pkg/kusciastorage/common/common.go | 50 ++ pkg/kusciastorage/common/error_code.go | 43 ++ pkg/kusciastorage/controller/common.go | 139 ++++++ pkg/kusciastorage/controller/common_test.go | 208 ++++++++ .../controller/resource_create.go | 171 +++++++ .../controller/resource_create_test.go | 305 ++++++++++++ .../controller/resource_delete.go | 68 +++ .../controller/resource_delete_test.go | 159 +++++++ .../controller/resource_query.go | 68 +++ .../controller/resource_query_test.go | 159 +++++++ pkg/kusciastorage/domain/model/model.go | 63 +++ pkg/kusciastorage/domain/service/service.go | 87 ++++ .../domain/service/service_test.go | 271 +++++++++++ pkg/kusciastorage/kuscia_storage.go | 39 ++ .../repository/mysql/mysql_repo.go | 179 +++++++ pkg/kusciastorage/repository/repository.go | 31 ++ pkg/kusciastorage/server/routers.go | 103 ++++ pkg/kusciastorage/server/server.go | 188 ++++++++ .../mist/.OPENSOURCE-CLEANUP-RMDIR | 0 pkg/secretbackend/mist/mist.go | 266 +++++++++++ pkg/secretbackend/mist/mist_test.go | 168 +++++++ .../task_input_config.2pc_balanced_psi.json | 7 +- ...task_input_config.2pc_balanced_psi_dp.json | 7 +- 80 files changed, 6575 insertions(+), 342 deletions(-) create mode 100644 .aci.yml create mode 100644 MetaPigeon.toml create mode 100644 OWNERS create mode 100644 build/dockerfile/base/kuscia-alios-base.Dockerfile create mode 100644 build/dockerfile/dev/.OPENSOURCE-CLEANUP-RMDIR create mode 100644 build/dockerfile/dev/kuscia-dev.Dockerfile create mode 100644 build/dockerfile/kuscia-alios.Dockerfile create mode 100644 cmd/kusciastorage/.OPENSOURCE-CLEANUP-RMDIR create mode 100644 cmd/kusciastorage/main.go create mode 100644 cmd/webdemo/.OPENSOURCE-CLEANUP-RMDIR create mode 100644 cmd/webdemo/TestBean.go create mode 100644 cmd/webdemo/any.go create mode 100644 cmd/webdemo/api.pb.go create mode 100644 cmd/webdemo/api.proto create mode 100644 cmd/webdemo/demo.go create mode 100644 cmd/webdemo/demo_config.go create mode 100644 cmd/webdemo/hello.go create mode 100644 cmd/webdemo/login.go create mode 100644 hack/k3s/README.md create mode 100644 internaldoc/.OPENSOURCE-CLEANUP-RMDIR create mode 100644 internaldoc/README.md create mode 100644 internaldoc/app_access_profile.md create mode 100644 internaldoc/docs_tutorial.md create mode 100644 internaldoc/quick_start.md create mode 100644 pkg/agent/provider/pod/kubebackend/sigma.go create mode 100644 pkg/agent/provider/pod/kubebackend/sigma_test.go create mode 100644 pkg/client/.OPENSOURCE-CLEANUP-RMDIR create mode 100644 pkg/client/kusciastorage/rest/client.go create mode 100644 pkg/client/kusciastorage/rest/client_test.go create mode 100644 pkg/client/kusciastorage/rest/config.go create mode 100644 pkg/client/kusciastorage/rest/config_test.go create mode 100644 pkg/client/kusciastorage/rest/utils.go create mode 100644 pkg/client/kusciastorage/rest/utils_test.go create mode 100644 pkg/kusciastorage/.OPENSOURCE-CLEANUP-RMDIR create mode 100644 pkg/kusciastorage/beans/beans.go create mode 100644 pkg/kusciastorage/common/common.go create mode 100644 pkg/kusciastorage/common/error_code.go create mode 100644 pkg/kusciastorage/controller/common.go create mode 100644 pkg/kusciastorage/controller/common_test.go create mode 100644 pkg/kusciastorage/controller/resource_create.go create mode 100644 pkg/kusciastorage/controller/resource_create_test.go create mode 100644 pkg/kusciastorage/controller/resource_delete.go create mode 100644 pkg/kusciastorage/controller/resource_delete_test.go create mode 100644 pkg/kusciastorage/controller/resource_query.go create mode 100644 pkg/kusciastorage/controller/resource_query_test.go create mode 100644 pkg/kusciastorage/domain/model/model.go create mode 100644 pkg/kusciastorage/domain/service/service.go create mode 100644 pkg/kusciastorage/domain/service/service_test.go create mode 100644 pkg/kusciastorage/kuscia_storage.go create mode 100644 pkg/kusciastorage/repository/mysql/mysql_repo.go create mode 100644 pkg/kusciastorage/repository/repository.go create mode 100644 pkg/kusciastorage/server/routers.go create mode 100644 pkg/kusciastorage/server/server.go create mode 100644 pkg/secretbackend/mist/.OPENSOURCE-CLEANUP-RMDIR create mode 100644 pkg/secretbackend/mist/mist.go create mode 100644 pkg/secretbackend/mist/mist_test.go diff --git a/.aci.yml b/.aci.yml new file mode 100644 index 000000000..8af43e898 --- /dev/null +++ b/.aci.yml @@ -0,0 +1,450 @@ +# OPENSOURCE-CLEANUP DELETE_FILE +# ACI 语法文档：https://yuque.antfin-inc.com/antci/help/syntax-plugins + +version: "1.5" #用于使用go-sast + +git: + depth: 200 + +stages: + - LintAndAnalysis + - EnvPrepare + - Test + - Build + - Delivery + - IntegrationTest + - Extend + +parameters: + BUILD_IMAGE: "reg.docker.alibaba-inc.com/secretflow/kuscia-dev:0.6" + KUSCIA_REPO_URL: "git@gitlab.alipay-inc.com:secretflow/kuscia.git" + +only: + triggerType: + - pullRequest # 创建PR及向PR分支push代码时触发 + - tagPush + - push + triggerBranch: # 设置触发分支 + pushOriginalBranch: # 只有向以下分支提交push才会触发 + - master + - release/* + - codereview/* + - beta/* + targetBranch: #只有向以下分支提交pullRequest时才触发pipeline执行 + - master + - release/* + - codereview/* + - beta/* + +golangCILint: + stage: LintAndAnalysis + # 开启强制扫描结果卡点 + allowFailure: false + plugin: CMD + pluginConfig: + encoding: "UTF-8" + excludes: + - "build/**" + - "crd/**" + - "hack/**" + - "pkg/crd/clientset/**" + - "pkg/crd/informers/**" + - "pkg/crd/listers/**" + - "proto/**" + - "*_test.go" + taskKind: CMD_GOLANGCI_LINT + checkRule: + - blocker = 0 && critical = 0 && major = 0 + +# 蚂蚁编码手册：https://yuque.antfin-inc.com/tb8i41/egiubq/zw54bzd2pgvitnms +#antStaticAnalysis: +# stage: LintAndAnalysis +# component: go-sast +# inputs: +# excludes: +# - "build/**" +# - "crd/**" +# - "hack/**" +# - "pkg/crd/clientset/**" +# - "pkg/crd/informers/**" +# - "pkg/crd/listers/**" +# - "pkg/agent/status/**" +# - "proto/**" +# - "*_test.go" +# codePath: "." +# config: +# execute: +# timeout: 1000 #任务超时时间 +# afterExecute: +# checkRule: +# - ${{outputs.blocker}} = 0 + +agent: + docker: + # image build from docker/dev.Dockerfile + image: ${BUILD_IMAGE} + resourceRequirements: # 2c4g20g的配置 cpu/memory/ephemeral-storage取值均为整型 + cpu: 4 # 最大值16 + memory: 8 # 最大值32 + ephemeral-storage: 10 # 最大值100 + +variableExport: + stage: EnvPrepare + script: + - | + cd ${WORKSPACE} + # Note: git clone requires answer 'yes' + ssh-keyscan -t rsa gitlab.alipay-inc.com >> ~/.ssh/known_hosts + source $HOME/.bashrc + + git fetch --tags ${KUSCIA_REPO_URL} + git describe --tags --always + + export KUSCIA_VERSION_TAG="$(git describe --tags --always)" + export BUILD_DATE="$(date +'%Y%m%d-%H%M%S')" + export IMAGE_VERSION="ACI-${KUSCIA_VERSION_TAG}-${BUILD_DATE}" + variables: + - KUSCIA_VERSION_TAG + - IMAGE_VERSION + - BUILD_DATE + +unittest: + stage: Test + aciTags: DOCKER + pluginConfig: + appname: mpc-worker + source: mpc-worker + encoding: "UTF-8" + steps: + - plugin: clone + - plugin: shell + resources: + cpu: 4 + mem: 8 + disk: 10 + inputs: + image: ${BUILD_IMAGE} + command: + - | + set -ex + # Note: git clone requires answer 'yes' + ssh-keyscan -t rsa gitlab.alipay-inc.com >> ~/.ssh/known_hosts + source $HOME/.bashrc + + # cd $WORKSPACE + mkdir -p /home/admin + ln -s $WORKSPACE /home/admin/dev + cd /home/admin/dev + + git fetch --tags ${KUSCIA_REPO_URL} + git describe --always + + whoami && cat /proc/cpuinfo | grep process | wc -l && cat /proc/meminfo | grep MemTotal && df -h + + make test + + - plugin: coverage-client-upload-compass + inputs: + filePath: ./test-results/coverage.xml + publisher: + junit: + testResults: "test-results/TEST-*.xml" + allowEmptyResults: true + checkRule: + - lineCoverage > 50 +# - changeLineCoverage > 50 + +build-framework: + stage: Build + plugin: ANT-BUILD + pluginConfig: + image: ${BUILD_IMAGE} + script: + - | + set -ex + # Note: git clone requires answer 'yes' + ssh-keyscan -t rsa gitlab.alipay-inc.com >> ~/.ssh/known_hosts + source $HOME/.bashrc + + # add user admin if not exists + id -u admin &>/dev/null || useradd admin + ln -s $ANTBUILD_WORKSPACE /home/admin/dev + cd /home/admin/dev + whoami && cat /proc/cpuinfo | grep process | wc -l && cat /proc/meminfo | grep MemTotal && df -h + + # .aci.yaml 开头的 depth 配置没有生效，aci 流水线写死 depth=5，这边强制修改 depth==200 + git fetch --deepen=200 ${KUSCIA_REPO_URL} + git fetch --tags ${KUSCIA_REPO_URL} + git describe --always + + make build + + afterScript: + - ls + outputs: + - name: "kuscia-framework" + desc: "kuscia framework package" + type: custom + paths: + - build/apps + - build/linux + - crds + - etc + - hack + - testdata + - scripts + only: + triggerType: + - push + - tagPush + - pullRequest + triggerBranch: # 设置触发分支 + pushOriginalBranch: # 只有向以下分支提交push才会触发 + - master + - release/* + - beta/* + targetBranch: #只有向以下分支提交pullRequest时才触发pipeline执行 + - release/* + - beta/* + +framework-anolis: + stage: Delivery + plugin: ANT-BUILD + pluginConfig: + image: ${BUILD_IMAGE} + beforeScript: + - echo $PWD + - ls + afterScript: + - ls + inputs: + dependency: build-framework + params: + - name: DIRECTORY + value: ./ + - name: DOCKERFILE + value: build/dockerfile/kuscia-anolis.Dockerfile + outputs: + - name: "kuscia-anolis" + type: image + repository: reg.docker.alibaba-inc.com + namespace: secretflow + desc: "kuscia framework image" + tag: ${IMAGE_VERSION} + only: + triggerType: + - push + - tagPush + - pullRequest + triggerBranch: # 设置触发分支 + pushOriginalBranch: # 只有向以下分支提交push才会触发 + - master + - release/* + - beta/* + targetBranch: #只有向以下分支提交pullRequest时才触发pipeline执行 + - release/* + - beta/* + +framework-alios: + stage: Delivery + plugin: ANT-BUILD + pluginConfig: + image: ${BUILD_IMAGE} + beforeScript: + - echo $PWD + - ls + afterScript: + - ls + inputs: + dependency: build-framework + params: + - name: DIRECTORY + value: ./ + - name: DOCKERFILE + value: build/dockerfile/kuscia-alios.Dockerfile + outputs: + - name: "kuscia-alios" + type: image + repository: reg.docker.alibaba-inc.com + namespace: secretflow + desc: "kuscia framework image" + tag: ${IMAGE_VERSION} + only: + triggerType: + - push + - tagPush + - pullRequest + triggerBranch: # 设置触发分支 + pushOriginalBranch: # 只有向以下分支提交push才会触发 + - master + - release/* + - beta/* + targetBranch: #只有向以下分支提交pullRequest时才触发pipeline执行 + - release/* + - beta/* + +integration-test: + stage: IntegrationTest + component: ant-code-build + inputs: + name: kuscia-aci-integration-test + params: + - name: image-version + value: ${{parameters.IMAGE_VERSION}} + - name: user + default: admin + value: ${{parameters.KUSCIA_VERSION_TAG}} + environment: + type: LINUX_MACHINE + labels: + - kuscia_aci_test + resources: null + buildTasks: + - name: Test + commands: + - | + set -ex + export USER=${BUILD_PARAMS.user} + export KUSCIA_IMAGE=reg.docker.alibaba-inc.com/secretflow/kuscia-anolis:${BUILD_PARAMS.image-version} + cd ${ACB_BUILD_DIR} && mkdir test + + echo '#!/bin/bash + + if [ $# -eq 0 ]; then + echo "Usage: $0 " + exit 1 + fi + + last_line=$(tail -n 1 "$1") + + if echo "$last_line" | grep -q "OK"; then + echo "Test succeed." + exit 0 + else + echo "Test failed." + exit 1 + fi + ' > check.sh + + chmod +x check.sh + + docker pull ${KUSCIA_IMAGE} && docker run --rm ${KUSCIA_IMAGE} cat /home/kuscia/scripts/test/integration_test.sh > ./test/integration_test.sh && chmod u+x ./test/integration_test.sh + script -qec "./test/integration_test.sh all" test_result.log + - name: Cleanup + commands: + - | + export USER=${BUILD_PARAMS.user} + cd ${ACB_BUILD_DIR} + rm -rf test/ code-repo/ $USER-kuscia-master/ $USER-kuscia-lite-alice/ $USER-kuscia-lite-bob/ $USER-kuscia-autonomy-bob/ $USER-kuscia-autonomy-alice/ + docker rmi reg.docker.alibaba-inc.com/secretflow/kuscia-anolis:${BUILD_PARAMS.image-version} + - name: Check + commands: + - | + cd ${ACB_BUILD_DIR} + ./check.sh ${ACB_BUILD_DIR}/test_result.log + timeout: 7200 # 7200s + only: + triggerType: + - push + - tagPush + - pullRequest + triggerBranch: # 设置触发分支 + pushOriginalBranch: # 只有向以下分支提交push才会触发 + - master + - release/* + - beta/* + targetBranch: #只有向以下分支提交pullRequest时才触发pipeline执行 + - release/* + - beta/* + +framework-anolis-with-secretflow: + stage: Extend + plugin: ANT-BUILD + pluginConfig: + image: ${BUILD_IMAGE} + beforeScript: + - set -ex + - echo $PWD + - cp build/dockerfile/kuscia-secretflow.Dockerfile build/dockerfile/kuscia-secretflow-anolis.Dockerfile + - export KUSCIA_IMAGE=reg.docker.alibaba-inc.com/secretflow/kuscia-anolis:${IMAGE_VERSION} + - sed -i "s|secretflow/kuscia:latest|${KUSCIA_IMAGE}|g" build/dockerfile/kuscia-secretflow-anolis.Dockerfile + - ls build/dockerfile + afterScript: + - ls + inputs: + dependency: framework-anolis + params: + - name: DIRECTORY + value: ./ + - name: DOCKERFILE + value: build/dockerfile/kuscia-secretflow-anolis.Dockerfile + outputs: + - name: "kuscia-secretflow-anolis" + type: image + repository: reg.docker.alibaba-inc.com + namespace: secretflow + desc: "kuscia framework image with secretflow" + tag: ${IMAGE_VERSION} + only: + triggerType: + - push + - tagPush + # - pullRequest + triggerBranch: # 设置触发分支 + pushOriginalBranch: # 只有向以下分支提交push才会触发 + # - master # 该步骤比较耗时，先不轻易在 master 上启用 + - release/* + - beta/* + targetBranch: #只有向以下分支提交pullRequest时才触发pipeline执行 + - release/* + - beta/* + +framework-alios-with-secretflow: + stage: Extend + plugin: ANT-BUILD + pluginConfig: + image: ${BUILD_IMAGE} + beforeScript: + - echo $PWD + - cp build/dockerfile/kuscia-secretflow.Dockerfile build/dockerfile/kuscia-secretflow-alios.Dockerfile + - export KUSCIA_IMAGE=reg.docker.alibaba-inc.com/secretflow/kuscia-alios:${IMAGE_VERSION} + - sed -i "s|secretflow/kuscia:latest|${KUSCIA_IMAGE}|g" build/dockerfile/kuscia-secretflow-alios.Dockerfile + - ls + afterScript: + - ls + inputs: + dependency: framework-alios + params: + - name: DIRECTORY + value: ./ + - name: DOCKERFILE + value: build/dockerfile/kuscia-secretflow-alios.Dockerfile + outputs: + - name: "kuscia-secretflow-alios" + type: image + repository: reg.docker.alibaba-inc.com + namespace: secretflow + desc: "kuscia framework image with secretflow" + tag: ${IMAGE_VERSION} + only: + triggerType: + - push + - tagPush + # - pullRequest + triggerBranch: # 设置触发分支 + pushOriginalBranch: # 只有向以下分支提交push才会触发 + # - master # 该步骤比较耗时，先不轻易在 master 上启用 + - release/* + - beta/* + targetBranch: #只有向以下分支提交pullRequest时才触发pipeline执行 + - release/* + - beta/* + +build_doc: + stage: Build + plugin: ANT-BUILD + pluginConfig: + image: ${BUILD_IMAGE} + script: + - | + curl -kv http://100.83.15.142:8089/kuscia/${ACI_COMMIT_REF_NAME} \ No newline at end of file diff --git a/.bazelrc b/.bazelrc index 40f4b502c..af0f14614 100644 --- a/.bazelrc +++ b/.bazelrc @@ -56,6 +56,9 @@ build:ubsan --linkopt -fsanitize=undefined build:macos-asan --features=asan build:macos-ubsan --features=ubsan +# OPENSOURCE-CLEANUP REMOVE BEGIN +build --remote_cache=http://bazelcache-pool.cz00b.alipay.net:8080 +# OPENSOURCE-CLEANUP REMOVE END test --keep_going test --test_output=errors diff --git a/.circleci/config.yml b/.circleci/config.yml index f7d54b86e..615d32569 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -52,7 +52,7 @@ commands: esac GOLANG_DIR="/usr/local" - GOLANG_VERSION="1.19.7" + GOLANG_VERSION="1.19.17" GOLANG_URL="https://golang.org/dl/go${GOLANG_VERSION}.linux-${ARCH}.tar.gz" if ! command -v go &> /dev/null; then @@ -79,7 +79,7 @@ commands: esac GOLANG_DIR="/usr/local" - GOLANG_VERSION="1.19.7" + GOLANG_VERSION="1.19.17" GOLANG_URL="https://golang.org/dl/go${GOLANG_VERSION}.linux-${ARCH}.tar.gz" if ! command -v go &> /dev/null; then diff --git a/.circleci/deps-config.yml b/.circleci/deps-config.yml index c5bdc8003..d6006dc2b 100644 --- a/.circleci/deps-config.yml +++ b/.circleci/deps-config.yml @@ -47,7 +47,7 @@ commands: esac GOLANG_DIR="/opt" - GOLANG_VERSION="1.19.7" + GOLANG_VERSION="1.22.2" GOLANG_URL="https://golang.org/dl/go${GOLANG_VERSION}.linux-${ARCH}.tar.gz" if ! command -v go &> /dev/null; then diff --git a/Makefile b/Makefile index 88269a8e6..fe0b1d408 100644 --- a/Makefile +++ b/Makefile @@ -15,8 +15,10 @@ IMG := secretflow/kuscia:${TAG} # TEST_SUITE used by integration test TEST_SUITE ?= all -ENVOY_IMAGE ?= secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/kuscia-envoy:0.4.0.dev20240407 -DEPS_IMAGE ?= secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/kuscia-deps:0.5.0b0 +# OPENSOURCE-CLEANUP SUB reg.docker.alibaba-inc.com/secretflow/kuscia-envoy:0.6.x.dev20240424 secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/kuscia-envoy:0.4.0.dev20240407 +ENVOY_IMAGE ?= reg.docker.alibaba-inc.com/secretflow/kuscia-envoy:0.6.x.dev20240424 +# OPENSOURCE-CLEANUP SUB reg.docker.alibaba-inc.com/secretflow/kuscia-deps:0.2.0 secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/kuscia-deps:0.5.0b0 +DEPS_IMAGE ?= reg.docker.alibaba-inc.com/secretflow/kuscia-deps:0.2.0 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) ifeq (,$(shell go env GOBIN)) @@ -142,3 +144,16 @@ integration_test: image ## Run Integration Test mkdir -p run/test cd run && KUSCIA_IMAGE=${IMG} docker run --rm ${IMG} cat /home/kuscia/scripts/test/integration_test.sh > ./test/integration_test.sh && chmod u+x ./test/integration_test.sh cd run && KUSCIA_IMAGE=${IMG} ./test/integration_test.sh ${TEST_SUITE} + +# OPENSOURCE-CLEANUP REMOVE BEGIN + +.PHONY: alios-base-image +alios-base-image: + docker build -t alios7u2-py-base:${DATETIME} --platform linux/amd64 -f ./build/dockerfile/base/kuscia-alios-base.Dockerfile . + +.PHONY: alios-image +alios-image: export GOOS=linux +alios-image: export GOARCH=amd64 +alios-image: build + docker build -t ${IMG} --build-arg KUSCIA_ENVOY_IMAGE=${ENVOY_IMAGE} --build-arg DEPS_IMAGE=${DEPS_IMAGE} -f ./build/dockerfile/kuscia-alios.Dockerfile . +# OPENSOURCE-CLEANUP REMOVE END \ No newline at end of file diff --git a/MetaPigeon.toml b/MetaPigeon.toml new file mode 100644 index 000000000..e497e5ccb --- /dev/null +++ b/MetaPigeon.toml @@ -0,0 +1,2 @@ +# OPENSOURCE-CLEANUP DELETE_FILE +BlackList = ["BE1054006"] \ No newline at end of file diff --git a/OWNERS b/OWNERS new file mode 100644 index 000000000..fda47bb5e --- /dev/null +++ b/OWNERS @@ -0,0 +1,3 @@ +# OPENSOURCE-CLEANUP DELETE_FILE +reviewers: ["wangzijie.wzj", "xiaol.huxl", "dongwen.hdw", "guoshilei.gsl", "caochen.cao", "yansi.zy", "jiamingyang.jmy", "lilinchuan.zwz","zhanhanbing.zhb","zhuangzhuangyan.yz","yj01523683"] +threshold: 2 diff --git a/WORKSPACE b/WORKSPACE index dbfd6ca89..b4ef85fc3 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -3,6 +3,8 @@ workspace(name = "kuscia") load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe") +# OPENSOURCE-CLEANUP REMOVE LINE_WITH_KEYWORD DEPS_FAST_MIRROR +DEPS_FAST_MIRROR = "https://antsys-nebula-archive.cn-hangzhou-alipay-b.oss-cdn.aliyun-inc.com/third_party" maybe( http_archive, diff --git a/build/dockerfile/base/kuscia-alios-base.Dockerfile b/build/dockerfile/base/kuscia-alios-base.Dockerfile new file mode 100644 index 000000000..95237d8a3 --- /dev/null +++ b/build/dockerfile/base/kuscia-alios-base.Dockerfile @@ -0,0 +1,69 @@ +# OPENSOURCE-CLEANUP DELETE_FILE +# latest image version: reg.docker.alibaba-inc.com/nueva-stack/alios7u2-py-base:0.20 + +FROM openanolis/anolisos:8.4-x86_64 AS build + +RUN yum groupinstall -y "Development Tools" && yum install -y wget fuse fuse-devel fuse3 fuse3-devel +RUN wget -O unionfs-fuse.tgz https://github.com/rpodgorny/unionfs-fuse/archive/refs/tags/v2.1.tar.gz && \ + tar xf unionfs-fuse.tgz && cd unionfs-fuse-2.1/ && make && make install + + +FROM reg.docker.alibaba-inc.com/alipay/7u2-common:202202.0T + +LABEL maintainer="zhuangzhuangyan.yz@digital-engine.com" + +ENV TZ=Asia/Shanghai + +USER root +WORKDIR /home/admin + +COPY --from=build /usr/local/bin/unionfs* /usr/local/bin/ + +# update glibc to 2.30 +# https://yuque.antfin-inc.com/baseos/manual/gcc92 + +RUN yum clean all && yum -y install glibc-devel && \ + yum -y update util-linux bash-completion gnupg2 python perl systemd && \ + yum -y install alios7u-2_30-gcc-9-repo.noarch && \ + yum -y install libdb binutils && \ + yum -y install glibc glibc-langpack-en && \ + yum -y install glibc-devel libstdc++-devel libstdc++-static libstdc++ libasan && \ + yum clean all + +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone + +RUN yum install -y wget && \ + wget -O /etc/yum.repos.d/CentOS-Base.repo http://mirrors.aliyun.com/repo/Centos-7.repo && \ + wget -O /etc/yum.repos.d/openresty.repo https://openresty.org/package/centos/openresty.repo && \ + yum install --setopt=skip_missing_names_on_install=False -y \ + net-tools-2.0 epel-release tar iproute sudo logrotate \ + tcpdump less rsync tree which \ + zlib-devel libffi-devel openssl-devel fuse fuse-devel fuse3 fuse3-devel fuse-overlayfs perl perl-Digest-MD5 python3 java openresty openresty-opm && \ + yum clean all + +RUN opm get knyar/nginx-lua-prometheus ledgetech/lua-resty-http cdbattags/lua-resty-jwt && \ + wget -O /usr/bin/ossutil64 http://gosspublic.alicdn.com/ossutil/1.6.0/ossutil64 && \ + chmod 0755 /usr/bin/ossutil64 + +RUN pushd /usr/bin/ && rm -rf python && ln -s python3 python && popd && \ + sed -i 's/\/usr\/bin\/python/\/usr\/bin\/python2.7/g' /usr/bin/yum /usr/libexec/urlgrabber-ext-down && \ + yum install -y python3-devel + +RUN pip3 install --upgrade pip && \ + pip3 install cryptography==3.3.2 paramiko scp + +RUN yum install -y libxml2-devel + +RUN pushd /tmp && \ + ln -s /usr/lib64/liblua-5.1.so /usr/lib64/liblua.so && \ + wget https://mpc-antbase.oss-cn-hangzhou.aliyuncs.com/release/linux/acjail/t-aliyun-acjail-2.0.1.an8.x86_64.rpm && \ + rpm -ivh t-aliyun-acjail-2.0.1.an8.x86_64.rpm && \ + sed -i '/blacklist_ip = {/c\blacklist_ip = blacklist_ip or {}' /usr/local/aliyun_acjail/config.lua && \ + rm -rfv t-aliyun-acjail-2.0.1.an8.x86_64.rpm && popd + +RUN pushd /tmp && wget https://bindfs.org/downloads/bindfs-1.15.1.tar.gz --no-check-certificate && \ + tar xf bindfs-1.15.1.tar.gz && cd bindfs-1.15.1 && ./configure && make && make install && \ + cd .. && rm -rfv bindfs* && popd + +RUN yum -y remove openssh && rm -rf /etc/runit/sshd && \ + yum -y remove telnet \ No newline at end of file diff --git a/build/dockerfile/base/kuscia-deps.Dockerfile b/build/dockerfile/base/kuscia-deps.Dockerfile index e32746710..786186fb6 100644 --- a/build/dockerfile/base/kuscia-deps.Dockerfile +++ b/build/dockerfile/base/kuscia-deps.Dockerfile @@ -1,3 +1,6 @@ +# OPENSOURCE-CLEANUP REMOVE 2 +# latest image version: reg.docker.alibaba-inc.com/secretflow/kuscia-deps:0.2.0 +# changelog: upgrade k3s from v1.26.6-k3s1 to v1.26.11-k3s2 ARG K3S_VER=v1.26.11-k3s2 ARG K3S_IMAGE=rancher/k3s:${K3S_VER} FROM ${K3S_IMAGE} as k3s-image diff --git a/build/dockerfile/dev/.OPENSOURCE-CLEANUP-RMDIR b/build/dockerfile/dev/.OPENSOURCE-CLEANUP-RMDIR new file mode 100644 index 000000000..e69de29bb diff --git a/build/dockerfile/dev/kuscia-dev.Dockerfile b/build/dockerfile/dev/kuscia-dev.Dockerfile new file mode 100644 index 000000000..d69610343 --- /dev/null +++ b/build/dockerfile/dev/kuscia-dev.Dockerfile @@ -0,0 +1,34 @@ +# latest image version: reg.docker.alibaba-inc.com/secretflow/kuscia-dev:0.6 + +FROM reg.docker.alibaba-inc.com/nueva-stack/dev-base:0.3 + +LABEL maintainer="caochen.cao@antgroup.com" + +ENV TZ=Asia/Shanghai + +RUN yum update -y \ + && yum clean all + +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone && \ + yum install -y wget sudo less epel-release net-tools iproute git jq tree unzip gcc perl perl-Digest-MD5 git-lfs which && \ + yum clean all && \ + wget -O /usr/bin/ossutil64 http://gosspublic.alicdn.com/ossutil/1.6.0/ossutil64 && \ + chmod 0755 /usr/bin/ossutil64 && \ + wget https://github.com/protocolbuffers/protobuf/releases/download/v21.8/protoc-21.8-linux-x86_64.zip && \ + unzip protoc-21.8-linux-x86_64.zip -d /usr/local && chgrp -R users /usr/local/include/google /usr/local/bin/protoc && cd ../ && \ + wget http://mpc-antbase.oss-cn-hangzhou.aliyuncs.com/influxdb/influxd -O /usr/local/bin/influxd && chmod 755 /usr/local/bin/influxd && \ + wget https://mpc-antbase.oss-cn-hangzhou.aliyuncs.com/mysql-8.0.28-lite/mysql.tar.gz && tar -xvf mysql.tar.gz -C /usr/local && chmod 755 /usr/local/bin/mysqld + +RUN wget https://golang.google.cn/dl/go1.19.10.linux-amd64.tar.gz -O - | tar -xz -C /usr/local && \ + echo export GOPROXY=https://goproxy.cn,direct >> ~/.bashrc && \ + echo export GO111MODULE=on >> ~/.bashrc && \ + echo export GOPATH=$HOME/gopath >> ~/.bashrc && \ + echo export PATH=$PATH:/usr/local/go/bin:'$GOPATH'/bin >> ~/.bashrc && \ + echo export GOPRIVATE=gitlab.alipay-inc.com >> ~/.bashrc && \ + source ~/.bashrc && \ + ssh-keyscan -t rsa gitlab.alipay-inc.com >> ~/.ssh/known_hosts && \ + git config --global url."git@gitlab.alipay-inc.com:".insteadOf "http://gitlab.alipay-inc.com/" && \ + go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.1 && \ + go install github.com/t-yuki/gocover-cobertura@latest && \ + go install github.com/jstemmer/go-junit-report/v2@latest && \ + go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.50.1 \ No newline at end of file diff --git a/build/dockerfile/kuscia-alios.Dockerfile b/build/dockerfile/kuscia-alios.Dockerfile new file mode 100644 index 000000000..de375c6f7 --- /dev/null +++ b/build/dockerfile/kuscia-alios.Dockerfile @@ -0,0 +1,52 @@ +# OPENSOURCE-CLEANUP DELETE_FILE +ARG DEPS_IMAGE="reg.docker.alibaba-inc.com/secretflow/kuscia-deps:0.2.0" +ARG KUSCIA_ENVOY_IMAGE="reg.docker.alibaba-inc.com/secretflow/kuscia-envoy:0.5.0b0" +ARG PROM_NODE_EXPORTER="prom/node-exporter:v1.7.0" + +FROM ${DEPS_IMAGE} as deps +FROM ${PROM_NODE_EXPORTER} as node_exporter +FROM ${KUSCIA_ENVOY_IMAGE} as kuscia_envoy + +FROM reg.docker.alibaba-inc.com/nueva-stack/alios7u2-py-base:0.20 + +ENV TZ=Asia/Shanghai + +ARG ROOT_DIR="/home/kuscia" +RUN yum install -y openssl net-tools which jq logrotate && \ + yum clean all && \ + mkdir -p ${ROOT_DIR}/bin && \ + mkdir -p /bin/aux && \ + mkdir -p ${ROOT_DIR}/scripts && \ + mkdir -p ${ROOT_DIR}/var/storage && \ + mkdir -p ${ROOT_DIR}/pause + +COPY --from=deps /image/home/kuscia/bin ${ROOT_DIR}/bin +COPY --from=deps /image/bin/aux /bin/aux +COPY --from=node_exporter /bin/node_exporter ${ROOT_DIR}/bin +RUN pushd ${ROOT_DIR}/bin && \ + ln -s k3s crictl && \ + ln -s k3s ctr && \ + ln -s k3s kubectl && \ + ln -s cni bridge && \ + ln -s cni flannel && \ + ln -s cni host-local && \ + ln -s cni loopback && \ + ln -s cni portmap && \ + popd + +COPY build/apps/kuscia/kuscia ${ROOT_DIR}/bin +COPY build/pause/pause-amd64.tar ${ROOT_DIR}/pause/pause.tar +COPY crds/v1alpha1 ${ROOT_DIR}/crds/v1alpha1 +COPY etc ${ROOT_DIR}/etc +COPY testdata ${ROOT_DIR}/var/storage/data +COPY scripts ${ROOT_DIR}/scripts + +COPY thirdparty/*/scripts ${ROOT_DIR}/scripts + +COPY --from=kuscia_envoy /home/kuscia/bin/envoy ${ROOT_DIR}/bin + +ENV PATH="${PATH}:${ROOT_DIR}/bin:/bin/aux" + +WORKDIR ${ROOT_DIR} + +ENTRYPOINT ["tini", "--"] diff --git a/build/dockerfile/kuscia-anolis.Dockerfile b/build/dockerfile/kuscia-anolis.Dockerfile index 0b95f32f7..390fb52d4 100644 --- a/build/dockerfile/kuscia-anolis.Dockerfile +++ b/build/dockerfile/kuscia-anolis.Dockerfile @@ -1,5 +1,7 @@ -ARG DEPS_IMAGE="secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/kuscia-deps:0.5.0b0" -ARG KUSCIA_ENVOY_IMAGE="secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/kuscia-envoy:0.6.0.dev20240507" +# OPENSOURCE-CLEANUP SUB reg.docker.alibaba-inc.com/secretflow/kuscia-deps:0.2.0 secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/kuscia-deps:0.5.0b0 +ARG DEPS_IMAGE="reg.docker.alibaba-inc.com/secretflow/kuscia-deps:0.2.0" +# OPENSOURCE-CLEANUP SUB reg.docker.alibaba-inc.com/secretflow/kuscia-envoy:0.5.0b0 secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/kuscia-envoy:0.4.0.dev20240407 +ARG KUSCIA_ENVOY_IMAGE="reg.docker.alibaba-inc.com/secretflow/kuscia-envoy:0.5.0b0" ARG PROM_NODE_EXPORTER="prom/node-exporter:v1.7.0" FROM ${DEPS_IMAGE} as deps diff --git a/build/dockerfile/kuscia-secretflow.Dockerfile b/build/dockerfile/kuscia-secretflow.Dockerfile index 83f2c9f43..cf8726e5d 100644 --- a/build/dockerfile/kuscia-secretflow.Dockerfile +++ b/build/dockerfile/kuscia-secretflow.Dockerfile @@ -12,7 +12,7 @@ RUN yum install -y protobuf libnl3 libgomp && \ grep -rl '#!/root/miniconda3/envs/secretflow/bin' /usr/local/bin/ | xargs sed -i -e 's/#!\/root\/miniconda3\/envs\/secretflow/#!\/usr\/local/g' && \ rm /usr/local/bin/openssl -ARG SF_VERSION="1.5.0b0" +ARG SF_VERSION="1.6.0b0" RUN pip install secretflow-lite==${SF_VERSION} --extra-index-url https://mirrors.aliyun.com/pypi/simple/ && rm -rf /root/.cache RUN kuscia image builtin secretflow-registry.cn-hangzhou.cr.aliyuncs.com/secretflow/secretflow-lite-anolis8:${SF_VERSION} --store /home/kuscia/var/images diff --git a/cmd/kuscia/confloader/secretbackendloader.go b/cmd/kuscia/confloader/secretbackendloader.go index 933d3b2cd..82ee5c91e 100644 --- a/cmd/kuscia/confloader/secretbackendloader.go +++ b/cmd/kuscia/confloader/secretbackendloader.go @@ -22,6 +22,9 @@ import ( "github.com/secretflow/kuscia/pkg/secretbackend" // register driver _ "github.com/secretflow/kuscia/pkg/secretbackend/mem" + // OPENSOURCE-CLEANUP REMOVE 2 + // register mist driver + _ "github.com/secretflow/kuscia/pkg/secretbackend/mist" ) type SecretBackendParams struct { diff --git a/cmd/kusciastorage/.OPENSOURCE-CLEANUP-RMDIR b/cmd/kusciastorage/.OPENSOURCE-CLEANUP-RMDIR new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/kusciastorage/main.go b/cmd/kusciastorage/main.go new file mode 100644 index 000000000..2f2b7d68e --- /dev/null +++ b/cmd/kusciastorage/main.go @@ -0,0 +1,27 @@ +// Copyright 2023 Ant Group Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "github.com/secretflow/kuscia/pkg/kusciastorage" + "github.com/secretflow/kuscia/pkg/utils/nlog" +) + +func main() { + err := kusciastorage.RunKusciaStorage() + if err != nil { + nlog.Errorf("Run kuscia storage service failed: %v", err.Error()) + } +} diff --git a/cmd/webdemo/.OPENSOURCE-CLEANUP-RMDIR b/cmd/webdemo/.OPENSOURCE-CLEANUP-RMDIR new file mode 100644 index 000000000..e69de29bb diff --git a/cmd/webdemo/TestBean.go b/cmd/webdemo/TestBean.go new file mode 100644 index 000000000..27cd1f7c3 --- /dev/null +++ b/cmd/webdemo/TestBean.go @@ -0,0 +1,61 @@ +// Copyright 2023 Ant Group Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "strconv" + + "github.com/secretflow/kuscia/pkg/utils/nlog" + "github.com/secretflow/kuscia/pkg/web/asserts" + "github.com/secretflow/kuscia/pkg/web/errorcode" + "github.com/secretflow/kuscia/pkg/web/framework" + "github.com/secretflow/kuscia/pkg/web/framework/config" +) + +type TestBean struct { + framework.ConfigLoader + // config + DataStoreType string `name:"data-store-type" usage:"Data Store Type" required:""` + RPCPort int `name:"rpc-port" usage:"rpc-port" default:"8118"` + + DataStoreRPCPort string + DemoConfig *DemoConfig +} + +func NewTestBean(c *DemoConfig) *TestBean { + return &TestBean{ + ConfigLoader: &config.FlagEnvConfigLoader{Source: config.SourceFlag}, + DemoConfig: c, + } +} + +func (b *TestBean) Validate(errs *errorcode.Errs) { + errs.AppendErr(asserts.NotEmpty(b.DataStoreType, "DataStoreType is empty")) + errs.AppendErr(asserts.IsTrue(b.RPCPort > 0, "RpcPort is invalid")) +} + +func (b *TestBean) Init(e framework.ConfBeanRegistry) error { + b.DataStoreRPCPort = b.DataStoreType + strconv.Itoa(b.RPCPort) + // TODO Init ... + nlog.Info("Init: " + b.DataStoreRPCPort) + return nil +} + +func (b *TestBean) Start(ctx context.Context, e framework.ConfBeanRegistry) error { + nlog.Info("Start" + b.DataStoreRPCPort) + b.DemoConfig.Print() + return nil +} diff --git a/cmd/webdemo/any.go b/cmd/webdemo/any.go new file mode 100644 index 000000000..fb6c1e39c --- /dev/null +++ b/cmd/webdemo/any.go @@ -0,0 +1,41 @@ +// Copyright 2023 Ant Group Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "reflect" + + "github.com/secretflow/kuscia/pkg/web/api" + "github.com/secretflow/kuscia/pkg/web/errorcode" +) + +type AnyHandler struct { +} + +var _ api.ProtoHandler = (*AnyHandler)(nil) + +func (h *AnyHandler) Validate(ctx *api.BizContext, request api.ProtoRequest, errs *errorcode.Errs) { + // Unsupported +} + +func (h *AnyHandler) Handle(ctx *api.BizContext, request api.ProtoRequest) api.ProtoResponse { + return &api.AnyStringProto{ + Content: `{"header": {"is_success": true}, "content": "hahaha"}`, + } +} + +func (h *AnyHandler) GetType() (reqType, respType reflect.Type) { + return reflect.TypeOf(api.AnyStringProto{}), reflect.TypeOf(api.AnyStringProto{}) +} diff --git a/cmd/webdemo/api.pb.go b/cmd/webdemo/api.pb.go new file mode 100644 index 000000000..2c7795367 --- /dev/null +++ b/cmd/webdemo/api.pb.go @@ -0,0 +1,409 @@ +// +//Copyright 2023 Ant Group Co., Ltd. +// +//Licensed under the Apache License, Version 2.0 (the "License"); +//you may not use this file except in compliance with the License. +//You may obtain a copy of the License at +// +//http://www.apache.org/licenses/LICENSE-2.0 +// +//Unless required by applicable law or agreed to in writing, software +//distributed under the License is distributed on an "AS IS" BASIS, +//WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +//See the License for the specific language governing permissions and +//limitations under the License. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.28.0 +// protoc v3.21.8 +// source: api.proto + +package main + +import ( + reflect "reflect" + sync "sync" + + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type RequestHeader struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + IsLoadtest bool `protobuf:"varint,1,opt,name=is_loadtest,json=isLoadtest,proto3" json:"is_loadtest,omitempty"` + TraceId string `protobuf:"bytes,2,opt,name=trace_id,json=traceId,proto3" json:"trace_id,omitempty"` +} + +func (x *RequestHeader) Reset() { + *x = RequestHeader{} + if protoimpl.UnsafeEnabled { + mi := &file_api_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *RequestHeader) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RequestHeader) ProtoMessage() {} + +func (x *RequestHeader) ProtoReflect() protoreflect.Message { + mi := &file_api_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RequestHeader.ProtoReflect.Descriptor instead. +func (*RequestHeader) Descriptor() ([]byte, []int) { + return file_api_proto_rawDescGZIP(), []int{0} +} + +func (x *RequestHeader) GetIsLoadtest() bool { + if x != nil { + return x.IsLoadtest + } + return false +} + +func (x *RequestHeader) GetTraceId() string { + if x != nil { + return x.TraceId + } + return "" +} + +type ResponseHeader struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + IsSuccess bool `protobuf:"varint,1,opt,name=is_success,json=isSuccess,proto3" json:"is_success,omitempty"` + ErrorCode int32 `protobuf:"varint,2,opt,name=error_code,json=errorCode,proto3" json:"error_code,omitempty"` + ErrorMsg string `protobuf:"bytes,3,opt,name=error_msg,json=errorMsg,proto3" json:"error_msg,omitempty"` + TraceInfo string `protobuf:"bytes,4,opt,name=trace_info,json=traceInfo,proto3" json:"trace_info,omitempty"` +} + +func (x *ResponseHeader) Reset() { + *x = ResponseHeader{} + if protoimpl.UnsafeEnabled { + mi := &file_api_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ResponseHeader) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ResponseHeader) ProtoMessage() {} + +func (x *ResponseHeader) ProtoReflect() protoreflect.Message { + mi := &file_api_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ResponseHeader.ProtoReflect.Descriptor instead. +func (*ResponseHeader) Descriptor() ([]byte, []int) { + return file_api_proto_rawDescGZIP(), []int{1} +} + +func (x *ResponseHeader) GetIsSuccess() bool { + if x != nil { + return x.IsSuccess + } + return false +} + +func (x *ResponseHeader) GetErrorCode() int32 { + if x != nil { + return x.ErrorCode + } + return 0 +} + +func (x *ResponseHeader) GetErrorMsg() string { + if x != nil { + return x.ErrorMsg + } + return "" +} + +func (x *ResponseHeader) GetTraceInfo() string { + if x != nil { + return x.TraceInfo + } + return "" +} + +type HelloRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Header *RequestHeader `protobuf:"bytes,1,opt,name=header,proto3" json:"header,omitempty"` + Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` +} + +func (x *HelloRequest) Reset() { + *x = HelloRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_api_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *HelloRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*HelloRequest) ProtoMessage() {} + +func (x *HelloRequest) ProtoReflect() protoreflect.Message { + mi := &file_api_proto_msgTypes[2] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use HelloRequest.ProtoReflect.Descriptor instead. +func (*HelloRequest) Descriptor() ([]byte, []int) { + return file_api_proto_rawDescGZIP(), []int{2} +} + +func (x *HelloRequest) GetHeader() *RequestHeader { + if x != nil { + return x.Header + } + return nil +} + +func (x *HelloRequest) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +type HelloResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Header *ResponseHeader `protobuf:"bytes,1,opt,name=header,proto3" json:"header,omitempty"` + Content string `protobuf:"bytes,2,opt,name=content,proto3" json:"content,omitempty"` +} + +func (x *HelloResponse) Reset() { + *x = HelloResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_api_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *HelloResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*HelloResponse) ProtoMessage() {} + +func (x *HelloResponse) ProtoReflect() protoreflect.Message { + mi := &file_api_proto_msgTypes[3] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use HelloResponse.ProtoReflect.Descriptor instead. +func (*HelloResponse) Descriptor() ([]byte, []int) { + return file_api_proto_rawDescGZIP(), []int{3} +} + +func (x *HelloResponse) GetHeader() *ResponseHeader { + if x != nil { + return x.Header + } + return nil +} + +func (x *HelloResponse) GetContent() string { + if x != nil { + return x.Content + } + return "" +} + +var File_api_proto protoreflect.FileDescriptor + +var file_api_proto_rawDesc = []byte{ + 0x0a, 0x09, 0x61, 0x70, 0x69, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x04, 0x6d, 0x61, 0x69, + 0x6e, 0x22, 0x4b, 0x0a, 0x0d, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, 0x65, 0x61, 0x64, + 0x65, 0x72, 0x12, 0x1f, 0x0a, 0x0b, 0x69, 0x73, 0x5f, 0x6c, 0x6f, 0x61, 0x64, 0x74, 0x65, 0x73, + 0x74, 0x18, 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x0a, 0x69, 0x73, 0x4c, 0x6f, 0x61, 0x64, 0x74, + 0x65, 0x73, 0x74, 0x12, 0x19, 0x0a, 0x08, 0x74, 0x72, 0x61, 0x63, 0x65, 0x5f, 0x69, 0x64, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x74, 0x72, 0x61, 0x63, 0x65, 0x49, 0x64, 0x22, 0x8a, + 0x01, 0x0a, 0x0e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x65, 0x61, 0x64, 0x65, + 0x72, 0x12, 0x1d, 0x0a, 0x0a, 0x69, 0x73, 0x5f, 0x73, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x08, 0x52, 0x09, 0x69, 0x73, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, + 0x12, 0x1d, 0x0a, 0x0a, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x5f, 0x63, 0x6f, 0x64, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x05, 0x52, 0x09, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x43, 0x6f, 0x64, 0x65, 0x12, + 0x1b, 0x0a, 0x09, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x5f, 0x6d, 0x73, 0x67, 0x18, 0x03, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x08, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x4d, 0x73, 0x67, 0x12, 0x1d, 0x0a, 0x0a, + 0x74, 0x72, 0x61, 0x63, 0x65, 0x5f, 0x69, 0x6e, 0x66, 0x6f, 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, + 0x52, 0x09, 0x74, 0x72, 0x61, 0x63, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x22, 0x4f, 0x0a, 0x0c, 0x48, + 0x65, 0x6c, 0x6c, 0x6f, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x2b, 0x0a, 0x06, 0x68, + 0x65, 0x61, 0x64, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x6d, 0x61, + 0x69, 0x6e, 0x2e, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x48, 0x65, 0x61, 0x64, 0x65, 0x72, + 0x52, 0x06, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, + 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x22, 0x57, 0x0a, 0x0d, + 0x48, 0x65, 0x6c, 0x6c, 0x6f, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x2c, 0x0a, + 0x06, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, + 0x6d, 0x61, 0x69, 0x6e, 0x2e, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x48, 0x65, 0x61, + 0x64, 0x65, 0x72, 0x52, 0x06, 0x68, 0x65, 0x61, 0x64, 0x65, 0x72, 0x12, 0x18, 0x0a, 0x07, 0x63, + 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x63, 0x6f, + 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x42, 0x09, 0x5a, 0x07, 0x2e, 0x2f, 0x3b, 0x6d, 0x61, 0x69, 0x6e, + 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_api_proto_rawDescOnce sync.Once + file_api_proto_rawDescData = file_api_proto_rawDesc +) + +func file_api_proto_rawDescGZIP() []byte { + file_api_proto_rawDescOnce.Do(func() { + file_api_proto_rawDescData = protoimpl.X.CompressGZIP(file_api_proto_rawDescData) + }) + return file_api_proto_rawDescData +} + +var file_api_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_api_proto_goTypes = []interface{}{ + (*RequestHeader)(nil), // 0: main.RequestHeader + (*ResponseHeader)(nil), // 1: main.ResponseHeader + (*HelloRequest)(nil), // 2: main.HelloRequest + (*HelloResponse)(nil), // 3: main.HelloResponse +} +var file_api_proto_depIdxs = []int32{ + 0, // 0: main.HelloRequest.header:type_name -> main.RequestHeader + 1, // 1: main.HelloResponse.header:type_name -> main.ResponseHeader + 2, // [2:2] is the sub-list for method output_type + 2, // [2:2] is the sub-list for method input_type + 2, // [2:2] is the sub-list for extension type_name + 2, // [2:2] is the sub-list for extension extendee + 0, // [0:2] is the sub-list for field type_name +} + +func init() { file_api_proto_init() } +func file_api_proto_init() { + if File_api_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_api_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*RequestHeader); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_api_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ResponseHeader); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_api_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*HelloRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_api_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*HelloResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_api_proto_rawDesc, + NumEnums: 0, + NumMessages: 4, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_api_proto_goTypes, + DependencyIndexes: file_api_proto_depIdxs, + MessageInfos: file_api_proto_msgTypes, + }.Build() + File_api_proto = out.File + file_api_proto_rawDesc = nil + file_api_proto_goTypes = nil + file_api_proto_depIdxs = nil +} diff --git a/cmd/webdemo/api.proto b/cmd/webdemo/api.proto new file mode 100644 index 000000000..ef07b085e --- /dev/null +++ b/cmd/webdemo/api.proto @@ -0,0 +1,41 @@ +// Copyright 2023 Ant Group Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +package main; + +option go_package = "./;main"; + +message RequestHeader { + bool is_loadtest = 1; + string trace_id = 2; +} + +message ResponseHeader { + bool is_success = 1; + int32 error_code = 2; + string error_msg = 3; + string trace_info = 4; +} + +message HelloRequest { + RequestHeader header = 1; + string name = 2; +}; + +message HelloResponse { + ResponseHeader header = 1; + string content = 2; +}; \ No newline at end of file diff --git a/cmd/webdemo/demo.go b/cmd/webdemo/demo.go new file mode 100644 index 000000000..ee0e9e2e7 --- /dev/null +++ b/cmd/webdemo/demo.go @@ -0,0 +1,116 @@ +// Copyright 2023 Ant Group Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "net/http" + + "github.com/gin-gonic/gin" + "github.com/prometheus/client_golang/prometheus" + "github.com/spf13/pflag" + + "github.com/secretflow/kuscia/pkg/utils/nlog" + "github.com/secretflow/kuscia/pkg/utils/nlog/zlogwriter" + "github.com/secretflow/kuscia/pkg/utils/signals" + "github.com/secretflow/kuscia/pkg/web/decorator" + "github.com/secretflow/kuscia/pkg/web/framework" + "github.com/secretflow/kuscia/pkg/web/framework/engine" + "github.com/secretflow/kuscia/pkg/web/framework/router" +) + +func main() { + defaultProtoDecorator := decorator.DefaultProtoDecoratorMaker(-1, -2) + interConnProtoDecorator := decorator.InterConnProtoDecoratorMaker(-1, -2) + // blank framework + r := engine.New(&framework.AppConfig{ + Name: "new", + Usage: "new framework without gin server and etc.", + Version: "v0.0.1", + }) + + // use Default settings (initedClass) + r = engine.Default(&framework.AppConfig{ + Name: "demo", + Usage: "just a demo", + Version: "v0.0.1", + }) + // r.UseConfig("test", &config.FlagEnvConfig{ + // Config: &TestConfig{}, + // Source: "flag", + // }) + + logCfg := zlogwriter.InstallPFlags(pflag.CommandLine) + logWriter, err := zlogwriter.New(logCfg) + if err != nil { + + } + nlog.Setup(nlog.SetWriter(logWriter)) + testB := NewDemoConfig() + if err := r.UseConfig("demo", testB); err != nil { + nlog.Fatalf("Init config err: %v", err) + } + + testC := NewTestBean(testB) + if err := r.UseBeanWithConfig("test", testC); err != nil { + nlog.Fatalf("Init test bean err: %v", err) + } + + // Metrics demo + userCnt := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Subsystem: "demo_metrics", + Name: "user_count", + Help: "user login request count.", + }, + []string{"user"}, + ) + prometheus.Register(userCnt) + userCnt.With(prometheus.Labels{"user": "aaa"}).Add(1) + // err when default gin not init + r.UseRouters(router.Routers{ + { + HTTPMethod: http.MethodPost, + RelativePath: "/api/v1/test", + Handlers: []gin.HandlerFunc{defaultProtoDecorator(r, &HelloHandler{})}, + }, + }) + + r.UseRouterGroups(router.GroupsRouters{ + { + Group: "/api/v2", + Routes: router.Routers{ + { + HTTPMethod: http.MethodPost, + RelativePath: "login", + Handlers: []gin.HandlerFunc{defaultProtoDecorator(r, GetNewLoginHander(userCnt))}, + }, + { + HTTPMethod: http.MethodPost, + RelativePath: "test1", + Handlers: []gin.HandlerFunc{defaultProtoDecorator(r, &HelloHandler{})}, + }, + { + HTTPMethod: http.MethodGet, + RelativePath: "test2", + Handlers: []gin.HandlerFunc{interConnProtoDecorator(r, &HelloHandler{})}, + }, + }, + }, + }) + + if err := r.RunCommand(signals.NewKusciaContextWithStopCh(signals.SetupSignalHandler())); err != nil { + nlog.Fatalf("App run failed: %v", err) + } +} diff --git a/cmd/webdemo/demo_config.go b/cmd/webdemo/demo_config.go new file mode 100644 index 000000000..a03e96cfe --- /dev/null +++ b/cmd/webdemo/demo_config.go @@ -0,0 +1,45 @@ +// Copyright 2023 Ant Group Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "github.com/secretflow/kuscia/pkg/utils/nlog" + "github.com/secretflow/kuscia/pkg/web/asserts" + "github.com/secretflow/kuscia/pkg/web/errorcode" + "github.com/secretflow/kuscia/pkg/web/framework" + "github.com/secretflow/kuscia/pkg/web/framework/config" +) + +type DemoConfig struct { + framework.ConfigLoader + // config + TestTag string `name:"test-type" usage:"Test type" required:""` + Author string `name:"author" usage:"Print demo author" default:"ant"` + Token string `name:"token" usage:"Token" secret:"pwd"` +} + +func NewDemoConfig() *DemoConfig { + return &DemoConfig{ + ConfigLoader: &config.FlagEnvConfigLoader{Source: config.SourceFlag}, + } +} + +func (c *DemoConfig) Validate(errs *errorcode.Errs) { + errs.AppendErr(asserts.NotEmpty(c.TestTag, "DataStoreType is empty")) +} + +func (c *DemoConfig) Print() { + nlog.Infof("TestTag=%v, Autho=%v, Token=%v", c.TestTag, c.Author, c.Token) +} diff --git a/cmd/webdemo/hello.go b/cmd/webdemo/hello.go new file mode 100644 index 000000000..daa5e92c5 --- /dev/null +++ b/cmd/webdemo/hello.go @@ -0,0 +1,57 @@ +// Copyright 2023 Ant Group Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "errors" + "reflect" + + "github.com/secretflow/kuscia/pkg/web/api" + "github.com/secretflow/kuscia/pkg/web/asserts" + "github.com/secretflow/kuscia/pkg/web/errorcode" +) + +type HelloHandler struct { +} + +var _ api.ProtoHandler = (*HelloHandler)(nil) + +func (h *HelloHandler) Validate(ctx *api.BizContext, request api.ProtoRequest, errs *errorcode.Errs) { + req, ok := request.(*HelloRequest) + if !ok { + errs.AppendErr(errors.New("unable convert request to HelloRequest")) + return + } + errs.AppendErr(asserts.NotEmpty(req.Name, "request.name is empty")) +} + +func (h *HelloHandler) Handle(ctx *api.BizContext, request api.ProtoRequest) api.ProtoResponse { + req, ok := request.(*HelloRequest) + if !ok { + return nil + } + if req.Name == "Invaild name" { + res := &HelloResponse{} + return res + } + c := "hello, " + req.Name + return &HelloResponse{ + Content: c, + } +} + +func (h *HelloHandler) GetType() (reqType, respType reflect.Type) { + return reflect.TypeOf(HelloRequest{}), reflect.TypeOf(HelloResponse{}) +} diff --git a/cmd/webdemo/login.go b/cmd/webdemo/login.go new file mode 100644 index 000000000..df8019027 --- /dev/null +++ b/cmd/webdemo/login.go @@ -0,0 +1,67 @@ +// Copyright 2023 Ant Group Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "errors" + "reflect" + + "github.com/prometheus/client_golang/prometheus" + + "github.com/secretflow/kuscia/pkg/web/api" + "github.com/secretflow/kuscia/pkg/web/asserts" + "github.com/secretflow/kuscia/pkg/web/errorcode" +) + +type LoginHandler struct { + collector *prometheus.CounterVec +} + +var _ api.ProtoHandler = (*LoginHandler)(nil) + +func GetNewLoginHander(collector *prometheus.CounterVec) *LoginHandler { + return &LoginHandler{ + collector: collector, + } +} + +func (h *LoginHandler) Validate(ctx *api.BizContext, request api.ProtoRequest, errs *errorcode.Errs) { + req, ok := request.(*HelloRequest) + if !ok { + errs.AppendErr(errors.New("unable convert request to HelloRequest")) + return + } + errs.AppendErr(asserts.NotEmpty(req.Name, "request.name is empty")) +} + +func (h *LoginHandler) Handle(ctx *api.BizContext, request api.ProtoRequest) api.ProtoResponse { + req, ok := request.(*HelloRequest) + if !ok { + return nil + } + if req.Name == "Invaild name" { + res := &HelloResponse{} + return res + } + c := "hello, " + req.Name + h.collector.WithLabelValues(req.Name).Inc() + return &HelloResponse{ + Content: c, + } +} + +func (h *LoginHandler) GetType() (reqType, respType reflect.Type) { + return reflect.TypeOf(HelloRequest{}), reflect.TypeOf(HelloResponse{}) +} diff --git a/docs/development/register_custom_image.md b/docs/development/register_custom_image.md index f87087813..8da9054f5 100644 --- a/docs/development/register_custom_image.md +++ b/docs/development/register_custom_image.md @@ -86,35 +86,4 @@ docker cp ${USER}-kuscia-autonomy-alice:/home/kuscia/scripts/tools/register_app_ 通过前面步骤注册完自定义算法镜像后，你可以获取算法镜像对应的 AppImage 资源名称。后续使用自定义算法镜像运行任务时，只需修改相应的字段即可。 -下面以名称为`secretflow-image`的 AppImage 为例，使用自定义算法镜像运行 [KusciaJob](../reference/concepts/kusciajob_cn.md) 作业。 - -- 修改 KusciaJob 下 `spec.tasks[].appImage`字段的值。 - -```yaml -apiVersion: kuscia.secretflow/v1alpha1 -kind: KusciaJob -metadata: - name: job-best-effort-linear -spec: - initiator: alice - scheduleMode: BestEffort - maxParallelism: 2 - tasks: - - taskID: job-psi - alias: job-psi - priority: 100 - taskInputConfig: '{"sf_datasource_config":{"alice":{"id":"default-data-source"},"bob":{"id":"default-data-source"}},"sf_cluster_desc":{"parties":["alice","bob"],"devices":[{"name":"spu","type":"spu","parties":["alice","bob"],"config":"{\"runtime_config\":{\"protocol\":\"REF2K\",\"field\":\"FM64\"},\"link_desc\":{\"connect_retry_times\":60,\"connect_retry_interval_ms\":1000,\"brpc_channel_protocol\":\"http\",\"brpc_channel_connection_type\":\"pooled\",\"recv_timeout_ms\":1200000,\"http_timeout_ms\":1200000}}"},{"name":"heu","type":"heu","parties":["alice","bob"],"config":"{\"mode\": \"PHEU\", \"schema\": \"paillier\", \"key_size\": 2048}"}],"ray_fed_config":{"cross_silo_comm_backend":"brpc_link"}},"sf_node_eval_param":{"domain":"data_prep","name":"psi","version":"0.0.1","attr_paths":["input/receiver_input/key","input/sender_input/key","protocol","precheck_input","bucket_size","curve_type"],"attrs":[{"ss":["id1"]},{"ss":["id2"]},{"s":"ECDH_PSI_2PC"},{"b":true},{"i64":"1048576"},{"s":"CURVE_FOURQ"}]},"sf_input_ids":["alice-table","bob-table"],"sf_output_ids":["psi-output"],"sf_output_uris":["psi-output.csv"]}' - appImage: secretflow-image - parties: - - domainID: alice - - domainID: bob - - taskID: job-split - alias: job-split - priority: 100 - dependencies: ['job-psi'] - taskInputConfig: '{"sf_datasource_config":{"alice":{"id":"default-data-source"},"bob":{"id":"default-data-source"}},"sf_cluster_desc":{"parties":["alice","bob"],"devices":[{"name":"spu","type":"spu","parties":["alice","bob"],"config":"{\"runtime_config\":{\"protocol\":\"REF2K\",\"field\":\"FM64\"},\"link_desc\":{\"connect_retry_times\":60,\"connect_retry_interval_ms\":1000,\"brpc_channel_protocol\":\"http\",\"brpc_channel_connection_type\":\"pooled\",\"recv_timeout_ms\":1200000,\"http_timeout_ms\":1200000}}"},{"name":"heu","type":"heu","parties":["alice","bob"],"config":"{\"mode\": \"PHEU\", \"schema\": \"paillier\", \"key_size\": 2048}"}],"ray_fed_config":{"cross_silo_comm_backend":"brpc_link"}},"sf_node_eval_param":{"domain":"data_prep","name":"train_test_split","version":"0.0.1","attr_paths":["train_size","test_size","random_state","shuffle"],"attrs":[{"f":0.75},{"f":0.25},{"i64":1234},{"b":true}]},"sf_output_uris":["train-dataset.csv","test-dataset.csv"],"sf_output_ids":["train-dataset","test-dataset"],"sf_input_ids":["psi-output"]}' - appImage: secretflow-image - parties: - - domainID: alice - - domainID: bob -``` +以名称为`secretflow-image`的 AppImage 为例，使用自定义算法镜像运行 [KusciaJob](../reference/concepts/kusciajob_cn.md) 作业，修改[KusciaJob 示例](../reference/concepts/kusciajob_cn.md#创建-kusciajob) 中 `spec.tasks[].appImage`字段的值。 \ No newline at end of file diff --git a/docs/getting_started/run_secretflow_cn.md b/docs/getting_started/run_secretflow_cn.md index 984d28070..6989f4669 100644 --- a/docs/getting_started/run_secretflow_cn.md +++ b/docs/getting_started/run_secretflow_cn.md @@ -57,61 +57,13 @@ docker exec -it ${USER}-kuscia-master bash ### 使用 Kuscia 示例数据配置 KusciaJob -下面的示例展示了一个 KusciaJob， 该任务流完成 2 个任务： +此处以[KusciaJob 示例](../reference/concepts/kusciajob_cn.md#创建-kusciajob)作为展示，该任务流完成 2 个任务： 1. job-psi 读取 alice 和 bob 的数据文件，进行隐私求交，求交的结果分别保存为两个参与方的 `psi-output.csv`。 2. job-split 读取 alice 和 bob 上一步中求交的结果文件，并拆分成训练集和测试集，分别保存为两个参与方的 `train-dataset.csv`、`test-dataset.csv`。 这个 KusciaJob 的名称为 job-best-effort-linear，在一个 Kuscia 集群中，这个名称必须是唯一的，由 `.metadata.name` 指定。 -在 kuscia-master 容器中，在任意路径创建文件 job-best-effort-linear.yaml，内容如下： - -```yaml -apiVersion: kuscia.secretflow/v1alpha1 -kind: KusciaJob -metadata: - name: job-best-effort-linear - namespace: cross-domain -spec: - initiator: alice - scheduleMode: BestEffort - maxParallelism: 2 - tasks: - - taskID: job-psi - alias: job-psi - priority: 100 - taskInputConfig: '{"sf_datasource_config":{"alice":{"id":"default-data-source"},"bob":{"id":"default-data-source"}},"sf_cluster_desc":{"parties":["alice","bob"],"devices":[{"name":"spu","type":"spu","parties":["alice","bob"],"config":"{\"runtime_config\":{\"protocol\":\"REF2K\",\"field\":\"FM64\"},\"link_desc\":{\"connect_retry_times\":60,\"connect_retry_interval_ms\":1000,\"brpc_channel_protocol\":\"http\",\"brpc_channel_connection_type\":\"pooled\",\"recv_timeout_ms\":1200000,\"http_timeout_ms\":1200000}}"},{"name":"heu","type":"heu","parties":["alice","bob"],"config":"{\"mode\": \"PHEU\", \"schema\": \"paillier\", \"key_size\": 2048}"}],"ray_fed_config":{"cross_silo_comm_backend":"brpc_link"}},"sf_node_eval_param":{"domain":"data_prep","name":"psi","version":"0.0.1","attr_paths":["input/receiver_input/key","input/sender_input/key","protocol","precheck_input","bucket_size","curve_type"],"attrs":[{"ss":["id1"]},{"ss":["id2"]},{"s":"ECDH_PSI_2PC"},{"b":true},{"i64":"1048576"},{"s":"CURVE_FOURQ"}]},"sf_input_ids":["alice-table","bob-table"],"sf_output_ids":["psi-output"],"sf_output_uris":["psi-output.csv"]}' - appImage: secretflow-image - parties: - - domainID: alice - - domainID: bob - - taskID: job-split - alias: job-split - priority: 100 - dependencies: ['job-psi'] - taskInputConfig: '{"sf_datasource_config":{"alice":{"id":"default-data-source"},"bob":{"id":"default-data-source"}},"sf_cluster_desc":{"parties":["alice","bob"],"devices":[{"name":"spu","type":"spu","parties":["alice","bob"],"config":"{\"runtime_config\":{\"protocol\":\"REF2K\",\"field\":\"FM64\"},\"link_desc\":{\"connect_retry_times\":60,\"connect_retry_interval_ms\":1000,\"brpc_channel_protocol\":\"http\",\"brpc_channel_connection_type\":\"pooled\",\"recv_timeout_ms\":1200000,\"http_timeout_ms\":1200000}}"},{"name":"heu","type":"heu","parties":["alice","bob"],"config":"{\"mode\": \"PHEU\", \"schema\": \"paillier\", \"key_size\": 2048}"}],"ray_fed_config":{"cross_silo_comm_backend":"brpc_link"}},"sf_node_eval_param":{"domain":"data_prep","name":"train_test_split","version":"0.0.1","attr_paths":["train_size","test_size","random_state","shuffle"],"attrs":[{"f":0.75},{"f":0.25},{"i64":1234},{"b":true}]},"sf_output_uris":["train-dataset.csv","test-dataset.csv"],"sf_output_ids":["train-dataset","test-dataset"],"sf_input_ids":["psi-output"]}' - appImage: secretflow-image - parties: - - domainID: alice - - domainID: bob -``` - -:::{tip} - -更多有关 KusciaJob 配置的信息，请查看 [KusciaJob](../reference/concepts/kusciajob_cn.md)。 - -KusciaJob 的算子参数由 `taskInputConfig` 字段定义，对于不同的算子，算子的参数不同。 - -本教程使用的是 SecretFlow 的算子参数定义，以 SecretFlow 引擎任务为例： -- `sf_datasource_config`：表示 SecretFlow 输入输出所需要的节点数据源信息。 -- `sf_cluster_desc`：表示 SecretFlow 集群信息，详情请查阅 [SecretFlow 集群文档](https://www.secretflow.org.cn/docs/secretflow/latest/zh-Hans/component/comp_spec_design#sfclusterdesc)。 -- `sf_node_eval_param`：表示 SecretFlow 算子的详细配置，详情请查阅 [SecretFlow 算子运行配置文档](https://www.secretflow.org.cn/docs/spec/latest/zh-Hans/intro#nodeevalparam)。 -- `sf_input_ids`：表示 SecretFlow 输入数据 `id` ，SecretFlow 引擎会将 Kuscia 定义的输入数据 [DomainData](../reference/concepts/domaindata_cn.md) 转换成引擎所需要的 [DistData](https://www.secretflow.org.cn/docs/spec/latest/zh-Hans/spec#distdata)。 -- `sf_output_ids`：表示 SecretFlow 输出数据 `id` ，SecretFlow 引擎会将输出的 [DistData](https://www.secretflow.org.cn/docs/spec/latest/zh-Hans/spec#distdata) 转换成 Kuscia 的 [DomainData](../reference/concepts/domaindata_cn.md)。 -- `sf_output_uris`：表示 SecretFlow 输出数据路径。 - -::: - ### 使用你自己的数据配置 KusciaJob 如果你要使用你自己的数据，可以将两个算子中的 `taskInputConfig.sf_input_ids` 的数据文件 `id` 修改为你在 [准备你自己的数据](#prepare-your-own-data) 中的 `domaindata_id` 即可。 @@ -175,7 +127,7 @@ spec: priority: 100 taskID: job-psi alias: job-psi - taskInputConfig: '{"sf_datasource_config":{"alice":{"id":"default-data-source"},"bob":{"id":"default-data-source"}},"sf_cluster_desc":{"parties":["alice","bob"],"devices":[{"name":"spu","type":"spu","parties":["alice","bob"],"config":"{\"runtime_config\":{\"protocol\":\"REF2K\",\"field\":\"FM64\"},\"link_desc\":{\"connect_retry_times\":60,\"connect_retry_interval_ms\":1000,\"brpc_channel_protocol\":\"http\",\"brpc_channel_connection_type\":\"pooled\",\"recv_timeout_ms\":1200000,\"http_timeout_ms\":1200000}}"},{"name":"heu","type":"heu","parties":["alice","bob"],"config":"{\"mode\": \"PHEU\", \"schema\": \"paillier\", \"key_size\": 2048}"}],"ray_fed_config":{"cross_silo_comm_backend":"brpc_link"}},"sf_node_eval_param":{"domain":"data_prep","name":"psi","version":"0.0.1","attr_paths":["input/receiver_input/key","input/sender_input/key","protocol","precheck_input","bucket_size","curve_type"],"attrs":[{"ss":["id1"]},{"ss":["id2"]},{"s":"ECDH_PSI_2PC"},{"b":true},{"i64":"1048576"},{"s":"CURVE_FOURQ"}]},"sf_input_ids":["alice-table","bob-table"],"sf_output_ids":["psi-output"],"sf_output_uris":["psi-output.csv"]}' + taskInputConfig: '{"sf_datasource_config":{"alice":{"id":"default-data-source"},"bob":{"id":"default-data-source"}},"sf_cluster_desc":{"parties":["alice","bob"],"devices":[{"name":"spu","type":"spu","parties":["alice","bob"],"config":"{\"runtime_config\":{\"protocol\":\"REF2K\",\"field\":\"FM64\"},\"link_desc\":{\"connect_retry_times\":60,\"connect_retry_interval_ms\":1000,\"brpc_channel_protocol\":\"http\",\"brpc_channel_connection_type\":\"pooled\",\"recv_timeout_ms\":1200000,\"http_timeout_ms\":1200000}}"},{"name":"heu","type":"heu","parties":["alice","bob"],"config":"{\"mode\": \"PHEU\", \"schema\": \"paillier\", \"key_size\": 2048}"}],"ray_fed_config":{"cross_silo_comm_backend":"brpc_link"}},"sf_node_eval_param":{"domain":"data_prep","name":"psi","version":"0.0.4","attr_paths":["input/receiver_input/key","input/sender_input/key","protocol","precheck_input","bucket_size","curve_type","left_side"],"attrs":[{"ss":["id1"]},{"ss":["id2"]},{"s":"PROTOCOL_ECDH"},{"b":true},{"i64":"1048576"},{"s":"CURVE_FOURQ"},{"is_na": false,"ss": ["alice"]}]},"sf_input_ids":["alice-table","bob-table"],"sf_output_ids":["psi-output"],"sf_output_uris":["psi-output.csv"]}' tolerable: false - appImage: secretflow-image dependencies: diff --git a/docs/reference/apis/kusciajob_cn.md b/docs/reference/apis/kusciajob_cn.md index 9d125edbd..4f0a68059 100644 --- a/docs/reference/apis/kusciajob_cn.md +++ b/docs/reference/apis/kusciajob_cn.md @@ -81,7 +81,7 @@ curl -k -X POST 'https://localhost:8082/api/v1/job/create' \ ], "alias": "job-psi", "dependencies": [], - "task_input_config": "{\"sf_datasource_config\":{\"alice\":{\"id\":\"default-data-source\"},\"bob\":{\"id\":\"default-data-source\"}},\"sf_cluster_desc\":{\"parties\":[\"alice\",\"bob\"],\"devices\":[{\"name\":\"spu\",\"type\":\"spu\",\"parties\":[\"alice\",\"bob\"],\"config\":\"{\\\"runtime_config\\\":{\\\"protocol\\\":\\\"REF2K\\\",\\\"field\\\":\\\"FM64\\\"},\\\"link_desc\\\":{\\\"connect_retry_times\\\":60,\\\"connect_retry_interval_ms\\\":1000,\\\"brpc_channel_protocol\\\":\\\"http\\\",\\\"brpc_channel_connection_type\\\":\\\"pooled\\\",\\\"recv_timeout_ms\\\":1200000,\\\"http_timeout_ms\\\":1200000}}\"},{\"name\":\"heu\",\"type\":\"heu\",\"parties\":[\"alice\",\"bob\"],\"config\":\"{\\\"mode\\\": \\\"PHEU\\\", \\\"schema\\\": \\\"paillier\\\", \\\"key_size\\\": 2048}\"}],\"ray_fed_config\":{\"cross_silo_comm_backend\":\"brpc_link\"}},\"sf_node_eval_param\":{\"domain\":\"preprocessing\",\"name\":\"psi\",\"version\":\"0.0.1\",\"attr_paths\":[\"input/receiver_input/key\",\"input/sender_input/key\",\"protocol\",\"precheck_input\",\"bucket_size\",\"curve_type\"],\"attrs\":[{\"ss\":[\"id1\"]},{\"ss\":[\"id2\"]},{\"s\":\"ECDH_PSI_2PC\"},{\"b\":true},{\"i64\":\"1048576\"},{\"s\":\"CURVE_FOURQ\"}]},\"sf_input_ids\":[\"alice-table\",\"bob-table\"],\"sf_output_ids\":[\"psi-output\"],\"sf_output_uris\":[\"psi-output.csv\"]}", + "task_input_config": "{\"sf_datasource_config\":{\"alice\":{\"id\":\"default-data-source\"},\"bob\":{\"id\":\"default-data-source\"}},\"sf_cluster_desc\":{\"parties\":[\"alice\",\"bob\"],\"devices\":[{\"name\":\"spu\",\"type\":\"spu\",\"parties\":[\"alice\",\"bob\"],\"config\":\"{\\\"runtime_config\\\":{\\\"protocol\\\":\\\"REF2K\\\",\\\"field\\\":\\\"FM64\\\"},\\\"link_desc\\\":{\\\"connect_retry_times\\\":60,\\\"connect_retry_interval_ms\\\":1000,\\\"brpc_channel_protocol\\\":\\\"http\\\",\\\"brpc_channel_connection_type\\\":\\\"pooled\\\",\\\"recv_timeout_ms\\\":1200000,\\\"http_timeout_ms\\\":1200000}}\"},{\"name\":\"heu\",\"type\":\"heu\",\"parties\":[\"alice\",\"bob\"],\"config\":\"{\\\"mode\\\": \\\"PHEU\\\", \\\"schema\\\": \\\"paillier\\\", \\\"key_size\\\": 2048}\"}],\"ray_fed_config\":{\"cross_silo_comm_backend\":\"brpc_link\"}},\"sf_node_eval_param\":{\"domain\":\"data_prep\",\"name\":\"psi\",\"version\":\"0.0.4\",\"attr_paths\":[\"input/receiver_input/key\",\"input/sender_input/key\",\"protocol\",\"precheck_input\",\"bucket_size\",\"curve_type\",\"left_side\"],\"attrs\":[{\"ss\":[\"id1\"]},{\"ss\":[\"id2\"]},{\"s\":\"PROTOCOL_ECDH\"},{\"b\":true},{\"i64\":\"1048576\"},{\"s\":\"CURVE_FOURQ\"},{\"is_na\": false,\"ss\": [\"alice\"]}]},\"sf_input_ids\":[\"alice-table\",\"bob-table\"],\"sf_output_ids\":[\"psi-output\"],\"sf_output_uris\":[\"psi-output.csv\"]}", "priority": 100 }, { @@ -101,7 +101,7 @@ curl -k -X POST 'https://localhost:8082/api/v1/job/create' \ "dependencies": [ "job-psi" ], - "task_input_config": "{\"sf_datasource_config\":{\"alice\":{\"id\":\"default-data-source\"},\"bob\":{\"id\":\"default-data-source\"}},\"sf_cluster_desc\":{\"parties\":[\"alice\",\"bob\"],\"devices\":[{\"name\":\"spu\",\"type\":\"spu\",\"parties\":[\"alice\",\"bob\"],\"config\":\"{\\\"runtime_config\\\":{\\\"protocol\\\":\\\"REF2K\\\",\\\"field\\\":\\\"FM64\\\"},\\\"link_desc\\\":{\\\"connect_retry_times\\\":60,\\\"connect_retry_interval_ms\\\":1000,\\\"brpc_channel_protocol\\\":\\\"http\\\",\\\"brpc_channel_connection_type\\\":\\\"pooled\\\",\\\"recv_timeout_ms\\\":1200000,\\\"http_timeout_ms\\\":1200000}}\"},{\"name\":\"heu\",\"type\":\"heu\",\"parties\":[\"alice\",\"bob\"],\"config\":\"{\\\"mode\\\": \\\"PHEU\\\", \\\"schema\\\": \\\"paillier\\\", \\\"key_size\\\": 2048}\"}],\"ray_fed_config\":{\"cross_silo_comm_backend\":\"brpc_link\"}},\"sf_node_eval_param\":{\"domain\":\"preprocessing\",\"name\":\"train_test_split\",\"version\":\"0.0.1\",\"attr_paths\":[\"train_size\",\"test_size\",\"random_state\",\"shuffle\"],\"attrs\":[{\"f\":0.75},{\"f\":0.25},{\"i64\":1234},{\"b\":true}]},\"sf_output_uris\":[\"train-dataset.csv\",\"test-dataset.csv\"],\"sf_output_ids\":[\"train-dataset\",\"test-dataset\"],\"sf_input_ids\":[\"psi-output\"]}", + "task_input_config": "{\"sf_datasource_config\":{\"alice\":{\"id\":\"default-data-source\"},\"bob\":{\"id\":\"default-data-source\"}},\"sf_cluster_desc\":{\"parties\":[\"alice\",\"bob\"],\"devices\":[{\"name\":\"spu\",\"type\":\"spu\",\"parties\":[\"alice\",\"bob\"],\"config\":\"{\\\"runtime_config\\\":{\\\"protocol\\\":\\\"REF2K\\\",\\\"field\\\":\\\"FM64\\\"},\\\"link_desc\\\":{\\\"connect_retry_times\\\":60,\\\"connect_retry_interval_ms\\\":1000,\\\"brpc_channel_protocol\\\":\\\"http\\\",\\\"brpc_channel_connection_type\\\":\\\"pooled\\\",\\\"recv_timeout_ms\\\":1200000,\\\"http_timeout_ms\\\":1200000}}\"},{\"name\":\"heu\",\"type\":\"heu\",\"parties\":[\"alice\",\"bob\"],\"config\":\"{\\\"mode\\\": \\\"PHEU\\\", \\\"schema\\\": \\\"paillier\\\", \\\"key_size\\\": 2048}\"}],\"ray_fed_config\":{\"cross_silo_comm_backend\":\"brpc_link\"}},\"sf_node_eval_param\":{\"domain\":\"data_prep\",\"name\":\"train_test_split\",\"version\":\"0.0.1\",\"attr_paths\":[\"train_size\",\"test_size\",\"random_state\",\"shuffle\"],\"attrs\":[{\"f\":0.75},{\"f\":0.25},{\"i64\":1234},{\"b\":true}]},\"sf_output_uris\":[\"train-dataset.csv\",\"test-dataset.csv\"],\"sf_output_ids\":[\"train-dataset\",\"test-dataset\"],\"sf_input_ids\":[\"psi-output\"]}", "priority": 100 } ] @@ -123,6 +123,12 @@ curl -k -X POST 'https://localhost:8082/api/v1/job/create' \ } ``` +:::{tip} + +上述请求示例中的引擎镜像基于 SecretFlow `1.5.0b0` 版本。算子参数的 `taskInputConfig` 内容可参考[KusciaJob](../concepts/kusciajob_cn.md#创建-kusciajob) + +::: + {#query-job} ### 查询 Job @@ -200,7 +206,7 @@ curl -k -X POST 'https://localhost:8082/api/v1/job/query' \ "dependencies": [ "" ], - "task_input_config": "{\"sf_datasource_config\":{\"alice\":{\"id\":\"default-data-source\"},\"bob\":{\"id\":\"default-data-source\"}},\"sf_cluster_desc\":{\"parties\":[\"alice\",\"bob\"],\"devices\":[{\"name\":\"spu\",\"type\":\"spu\",\"parties\":[\"alice\",\"bob\"],\"config\":\"{\\\"runtime_config\\\":{\\\"protocol\\\":\\\"REF2K\\\",\\\"field\\\":\\\"FM64\\\"},\\\"link_desc\\\":{\\\"connect_retry_times\\\":60,\\\"connect_retry_interval_ms\\\":1000,\\\"brpc_channel_protocol\\\":\\\"http\\\",\\\"brpc_channel_connection_type\\\":\\\"pooled\\\",\\\"recv_timeout_ms\\\":1200000,\\\"http_timeout_ms\\\":1200000}}\"},{\"name\":\"heu\",\"type\":\"heu\",\"parties\":[\"alice\",\"bob\"],\"config\":\"{\\\"mode\\\": \\\"PHEU\\\", \\\"schema\\\": \\\"paillier\\\", \\\"key_size\\\": 2048}\"}],\"ray_fed_config\":{\"cross_silo_comm_backend\":\"brpc_link\"}},\"sf_node_eval_param\":{\"domain\":\"preprocessing\",\"name\":\"psi\",\"version\":\"0.0.1\",\"attr_paths\":[\"input/receiver_input/key\",\"input/sender_input/key\",\"protocol\",\"precheck_input\",\"bucket_size\",\"curve_type\"],\"attrs\":[{\"ss\":[\"id1\"]},{\"ss\":[\"id2\"]},{\"s\":\"ECDH_PSI_2PC\"},{\"b\":true},{\"i64\":\"1048576\"},{\"s\":\"CURVE_FOURQ\"}]},\"sf_input_ids\":[\"alice-table\",\"bob-table\"],\"sf_output_ids\":[\"psi-output\"],\"sf_output_uris\":[\"psi-output.csv\"]}", + "task_input_config": "{\"sf_datasource_config\":{\"alice\":{\"id\":\"default-data-source\"},\"bob\":{\"id\":\"default-data-source\"}},\"sf_cluster_desc\":{\"parties\":[\"alice\",\"bob\"],\"devices\":[{\"name\":\"spu\",\"type\":\"spu\",\"parties\":[\"alice\",\"bob\"],\"config\":\"{\\\"runtime_config\\\":{\\\"protocol\\\":\\\"REF2K\\\",\\\"field\\\":\\\"FM64\\\"},\\\"link_desc\\\":{\\\"connect_retry_times\\\":60,\\\"connect_retry_interval_ms\\\":1000,\\\"brpc_channel_protocol\\\":\\\"http\\\",\\\"brpc_channel_connection_type\\\":\\\"pooled\\\",\\\"recv_timeout_ms\\\":1200000,\\\"http_timeout_ms\\\":1200000}}\"},{\"name\":\"heu\",\"type\":\"heu\",\"parties\":[\"alice\",\"bob\"],\"config\":\"{\\\"mode\\\": \\\"PHEU\\\", \\\"schema\\\": \\\"paillier\\\", \\\"key_size\\\": 2048}\"}],\"ray_fed_config\":{\"cross_silo_comm_backend\":\"brpc_link\"}},\"sf_node_eval_param\":{\"domain\":\"data_prep\",\"name\":\"psi\",\"version\":\"0.0.4\",\"attr_paths\":[\"input/receiver_input/key\",\"input/sender_input/key\",\"protocol\",\"precheck_input\",\"bucket_size\",\"curve_type\",\"left_side\"],\"attrs\":[{\"ss\":[\"id1\"]},{\"ss\":[\"id2\"]},{\"s\":\"PROTOCOL_ECDH\"},{\"b\":true},{\"i64\":\"1048576\"},{\"s\":\"CURVE_FOURQ\"},{\"is_na\": false,\"ss\": [\"alice\"]}]},\"sf_input_ids\":[\"alice-table\",\"bob-table\"],\"sf_output_ids\":[\"psi-output\"],\"sf_output_uris\":[\"psi-output.csv\"]}", "priority": 100 }, { @@ -220,7 +226,7 @@ curl -k -X POST 'https://localhost:8082/api/v1/job/query' \ "dependencies": [ "job-psi" ], - "task_input_config": "{\"sf_datasource_config\":{\"alice\":{\"id\":\"default-data-source\"},\"bob\":{\"id\":\"default-data-source\"}},\"sf_cluster_desc\":{\"parties\":[\"alice\",\"bob\"],\"devices\":[{\"name\":\"spu\",\"type\":\"spu\",\"parties\":[\"alice\",\"bob\"],\"config\":\"{\\\"runtime_config\\\":{\\\"protocol\\\":\\\"REF2K\\\",\\\"field\\\":\\\"FM64\\\"},\\\"link_desc\\\":{\\\"connect_retry_times\\\":60,\\\"connect_retry_interval_ms\\\":1000,\\\"brpc_channel_protocol\\\":\\\"http\\\",\\\"brpc_channel_connection_type\\\":\\\"pooled\\\",\\\"recv_timeout_ms\\\":1200000,\\\"http_timeout_ms\\\":1200000}}\"},{\"name\":\"heu\",\"type\":\"heu\",\"parties\":[\"alice\",\"bob\"],\"config\":\"{\\\"mode\\\": \\\"PHEU\\\", \\\"schema\\\": \\\"paillier\\\", \\\"key_size\\\": 2048}\"}],\"ray_fed_config\":{\"cross_silo_comm_backend\":\"brpc_link\"}},\"sf_node_eval_param\":{\"domain\":\"preprocessing\",\"name\":\"train_test_split\",\"version\":\"0.0.1\",\"attr_paths\":[\"train_size\",\"test_size\",\"random_state\",\"shuffle\"],\"attrs\":[{\"f\":0.75},{\"f\":0.25},{\"i64\":1234},{\"b\":true}]},\"sf_output_uris\":[\"train-dataset.csv\",\"test-dataset.csv\"],\"sf_output_ids\":[\"train-dataset\",\"test-dataset\"],\"sf_input_ids\":[\"psi-output\"]}", + "task_input_config": "{\"sf_datasource_config\":{\"alice\":{\"id\":\"default-data-source\"},\"bob\":{\"id\":\"default-data-source\"}},\"sf_cluster_desc\":{\"parties\":[\"alice\",\"bob\"],\"devices\":[{\"name\":\"spu\",\"type\":\"spu\",\"parties\":[\"alice\",\"bob\"],\"config\":\"{\\\"runtime_config\\\":{\\\"protocol\\\":\\\"REF2K\\\",\\\"field\\\":\\\"FM64\\\"},\\\"link_desc\\\":{\\\"connect_retry_times\\\":60,\\\"connect_retry_interval_ms\\\":1000,\\\"brpc_channel_protocol\\\":\\\"http\\\",\\\"brpc_channel_connection_type\\\":\\\"pooled\\\",\\\"recv_timeout_ms\\\":1200000,\\\"http_timeout_ms\\\":1200000}}\"},{\"name\":\"heu\",\"type\":\"heu\",\"parties\":[\"alice\",\"bob\"],\"config\":\"{\\\"mode\\\": \\\"PHEU\\\", \\\"schema\\\": \\\"paillier\\\", \\\"key_size\\\": 2048}\"}],\"ray_fed_config\":{\"cross_silo_comm_backend\":\"brpc_link\"}},\"sf_node_eval_param\":{\"domain\":\"data_prep\",\"name\":\"train_test_split\",\"version\":\"0.0.1\",\"attr_paths\":[\"train_size\",\"test_size\",\"random_state\",\"shuffle\"],\"attrs\":[{\"f\":0.75},{\"f\":0.25},{\"i64\":1234},{\"b\":true}]},\"sf_output_uris\":[\"train-dataset.csv\",\"test-dataset.csv\"],\"sf_output_ids\":[\"train-dataset\",\"test-dataset\"],\"sf_input_ids\":[\"psi-output\"]}", "priority": 100 } ], diff --git a/docs/reference/concepts/kusciajob_cn.md b/docs/reference/concepts/kusciajob_cn.md index ea44cf095..219e2d976 100644 --- a/docs/reference/concepts/kusciajob_cn.md +++ b/docs/reference/concepts/kusciajob_cn.md @@ -64,7 +64,7 @@ spec: - taskID: job-psi alias: job-psi priority: 100 - taskInputConfig: '{"sf_datasource_config":{"alice":{"id":"default-data-source"},"bob":{"id":"default-data-source"}},"sf_cluster_desc":{"parties":["alice","bob"],"devices":[{"name":"spu","type":"spu","parties":["alice","bob"],"config":"{\"runtime_config\":{\"protocol\":\"REF2K\",\"field\":\"FM64\"},\"link_desc\":{\"connect_retry_times\":60,\"connect_retry_interval_ms\":1000,\"brpc_channel_protocol\":\"http\",\"brpc_channel_connection_type\":\"pooled\",\"recv_timeout_ms\":1200000,\"http_timeout_ms\":1200000}}"},{"name":"heu","type":"heu","parties":["alice","bob"],"config":"{\"mode\": \"PHEU\", \"schema\": \"paillier\", \"key_size\": 2048}"}],"ray_fed_config":{"cross_silo_comm_backend":"brpc_link"}},"sf_node_eval_param":{"domain":"data_prep","name":"psi","version":"0.0.1","attr_paths":["input/receiver_input/key","input/sender_input/key","protocol","precheck_input","bucket_size","curve_type"],"attrs":[{"ss":["id1"]},{"ss":["id2"]},{"s":"ECDH_PSI_2PC"},{"b":true},{"i64":"1048576"},{"s":"CURVE_FOURQ"}]},"sf_input_ids":["alice-table","bob-table"],"sf_output_ids":["psi-output"],"sf_output_uris":["psi-output.csv"]}' + taskInputConfig: '{"sf_datasource_config":{"alice":{"id":"default-data-source"},"bob":{"id":"default-data-source"}},"sf_cluster_desc":{"parties":["alice","bob"],"devices":[{"name":"spu","type":"spu","parties":["alice","bob"],"config":"{\"runtime_config\":{\"protocol\":\"REF2K\",\"field\":\"FM64\"},\"link_desc\":{\"connect_retry_times\":60,\"connect_retry_interval_ms\":1000,\"brpc_channel_protocol\":\"http\",\"brpc_channel_connection_type\":\"pooled\",\"recv_timeout_ms\":1200000,\"http_timeout_ms\":1200000}}"},{"name":"heu","type":"heu","parties":["alice","bob"],"config":"{\"mode\": \"PHEU\", \"schema\": \"paillier\", \"key_size\": 2048}"}],"ray_fed_config":{"cross_silo_comm_backend":"brpc_link"}},"sf_node_eval_param":{"domain":"data_prep","name":"psi","version":"0.0.4","attr_paths":["input/receiver_input/key","input/sender_input/key","protocol","precheck_input","bucket_size","curve_type","left_side"],"attrs":[{"ss":["id1"]},{"ss":["id2"]},{"s":"PROTOCOL_ECDH"},{"b":true},{"i64":"1048576"},{"s":"CURVE_FOURQ"},{"is_na": false,"ss": ["alice"]}]},"sf_input_ids":["alice-table","bob-table"],"sf_output_ids":["psi-output"],"sf_output_uris":["psi-output.csv"]}' appImage: secretflow-image parties: - domainID: alice @@ -94,6 +94,20 @@ spec: 计算出两方的交集数据并分别保存为 alice 和 bob 的 psi-output.csv ，而 job-split 将会读取新产生的两个求交数据，并进行随机分割， 随机分割的结果也会保存在 alice 和 bob 两方。 +:::{tip} + +KusciaJob 的算子参数由 `taskInputConfig` 字段定义，对于不同的算子，算子的参数不同。 + +本教程使用的是 SecretFlow 的算子参数定义，以 SecretFlow `1.5.0b0` 版本引擎任务为例，其他版本请参考[SecretFlow 官网](https://www.secretflow.org.cn/zh-CN/docs/secretflow/main/getting_started)： +- `sf_datasource_config`：表示 SecretFlow 输入输出所需要的节点数据源信息。 +- `sf_cluster_desc`：表示 SecretFlow 集群信息，详情请查阅 [SecretFlow 集群文档](https://www.secretflow.org.cn/docs/secretflow/latest/zh-Hans/component/comp_spec_design#sfclusterdesc)。 +- `sf_node_eval_param`：表示 SecretFlow 算子的详细配置，详情请查阅 [SecretFlow 算子运行配置文档](https://www.secretflow.org.cn/docs/spec/latest/zh-Hans/intro#nodeevalparam)。 +- `sf_input_ids`：表示 SecretFlow 输入数据 `id`，SecretFlow 引擎会将 Kuscia 定义的输入数据 [DomainData](../reference/concepts/domaindata_cn.md) 转换成引擎所需要的 [DistData](https://www.secretflow.org.cn/docs/spec/latest/zh-Hans/spec#distdata)。 +- `sf_output_ids`：表示 SecretFlow 输出数据 `id`，SecretFlow 引擎会将输出的 [DistData](https://www.secretflow.org.cn/docs/spec/latest/zh-Hans/spec#distdata) 转换成 Kuscia 的 [DomainData](../reference/concepts/domaindata_cn.md)。 +- `sf_output_uris`：表示 SecretFlow 输出数据路径。 + +::: + ## KusciaJob 的调度 {#task-classification} @@ -155,7 +169,7 @@ spec: - alias: job-psi taskID: job-psi priority: 100 - taskInputConfig: '{"sf_datasource_config":{"alice":{"id":"default-data-source"},"bob":{"id":"default-data-source"}},"sf_cluster_desc":{"parties":["alice","bob"],"devices":[{"name":"spu","type":"spu","parties":["alice","bob"],"config":"{\"runtime_config\":{\"protocol\":\"REF2K\",\"field\":\"FM64\"},\"link_desc\":{\"connect_retry_times\":60,\"connect_retry_interval_ms\":1000,\"brpc_channel_protocol\":\"http\",\"brpc_channel_connection_type\":\"pooled\",\"recv_timeout_ms\":1200000,\"http_timeout_ms\":1200000}}"},{"name":"heu","type":"heu","parties":["alice","bob"],"config":"{\"mode\": \"PHEU\", \"schema\": \"paillier\", \"key_size\": 2048}"}],"ray_fed_config":{"cross_silo_comm_backend":"brpc_link"}},"sf_node_eval_param":{"domain":"data_prep","name":"psi","version":"0.0.1","attr_paths":["input/receiver_input/key","input/sender_input/key","protocol","precheck_input","bucket_size","curve_type"],"attrs":[{"ss":["id1"]},{"ss":["id2"]},{"s":"ECDH_PSI_2PC"},{"b":true},{"i64":"1048576"},{"s":"CURVE_FOURQ"}]},"sf_input_ids":["alice-table","bob-table"],"sf_output_ids":["psi-output"],"sf_output_uris":["psi-output.csv"]}' + taskInputConfig: '{"sf_datasource_config":{"alice":{"id":"default-data-source"},"bob":{"id":"default-data-source"}},"sf_cluster_desc":{"parties":["alice","bob"],"devices":[{"name":"spu","type":"spu","parties":["alice","bob"],"config":"{\"runtime_config\":{\"protocol\":\"REF2K\",\"field\":\"FM64\"},\"link_desc\":{\"connect_retry_times\":60,\"connect_retry_interval_ms\":1000,\"brpc_channel_protocol\":\"http\",\"brpc_channel_connection_type\":\"pooled\",\"recv_timeout_ms\":1200000,\"http_timeout_ms\":1200000}}"},{"name":"heu","type":"heu","parties":["alice","bob"],"config":"{\"mode\": \"PHEU\", \"schema\": \"paillier\", \"key_size\": 2048}"}],"ray_fed_config":{"cross_silo_comm_backend":"brpc_link"}},"sf_node_eval_param":{"domain":"data_prep","name":"psi","version":"0.0.4","attr_paths":["input/receiver_input/key","input/sender_input/key","protocol","precheck_input","bucket_size","curve_type","left_side"],"attrs":[{"ss":["id1"]},{"ss":["id2"]},{"s":"PROTOCOL_ECDH"},{"b":true},{"i64":"1048576"},{"s":"CURVE_FOURQ"},{"is_na": false,"ss": ["alice"]}]},"sf_input_ids":["alice-table","bob-table"],"sf_output_ids":["psi-output"],"sf_output_uris":["psi-output.csv"]}' appImage: secretflow-image parties: - domainID: alice @@ -198,7 +212,7 @@ spec: - alias: job-psi1 taskID: job-psi1 priority: 100 - taskInputConfig: '{"sf_datasource_config":{"alice":{"id":"default-data-source"},"bob":{"id":"default-data-source"}},"sf_cluster_desc":{"parties":["alice","bob"],"devices":[{"name":"spu","type":"spu","parties":["alice","bob"],"config":"{\"runtime_config\":{\"protocol\":\"REF2K\",\"field\":\"FM64\"},\"link_desc\":{\"connect_retry_times\":60,\"connect_retry_interval_ms\":1000,\"brpc_channel_protocol\":\"http\",\"brpc_channel_connection_type\":\"pooled\",\"recv_timeout_ms\":1200000,\"http_timeout_ms\":1200000}}"},{"name":"heu","type":"heu","parties":["alice","bob"],"config":"{\"mode\": \"PHEU\", \"schema\": \"paillier\", \"key_size\": 2048}"}],"ray_fed_config":{"cross_silo_comm_backend":"brpc_link"}},"sf_node_eval_param":{"domain":"data_prep","name":"psi","version":"0.0.1","attr_paths":["input/receiver_input/key","input/sender_input/key","protocol","precheck_input","bucket_size","curve_type"],"attrs":[{"ss":["id1"]},{"ss":["id2"]},{"s":"ECDH_PSI_2PC"},{"b":true},{"i64":"1048576"},{"s":"CURVE_FOURQ"}]},"sf_input_ids":["alice-table","bob-table"],"sf_output_ids":["psi-output"],"sf_output_uris":["psi-output.csv"]}' + taskInputConfig: '{"sf_datasource_config":{"alice":{"id":"default-data-source"},"bob":{"id":"default-data-source"}},"sf_cluster_desc":{"parties":["alice","bob"],"devices":[{"name":"spu","type":"spu","parties":["alice","bob"],"config":"{\"runtime_config\":{\"protocol\":\"REF2K\",\"field\":\"FM64\"},\"link_desc\":{\"connect_retry_times\":60,\"connect_retry_interval_ms\":1000,\"brpc_channel_protocol\":\"http\",\"brpc_channel_connection_type\":\"pooled\",\"recv_timeout_ms\":1200000,\"http_timeout_ms\":1200000}}"},{"name":"heu","type":"heu","parties":["alice","bob"],"config":"{\"mode\": \"PHEU\", \"schema\": \"paillier\", \"key_size\": 2048}"}],"ray_fed_config":{"cross_silo_comm_backend":"brpc_link"}},"sf_node_eval_param":{"domain":"data_prep","name":"psi","version":"0.0.4","attr_paths":["input/receiver_input/key","input/sender_input/key","protocol","precheck_input","bucket_size","curve_type","left_side"],"attrs":[{"ss":["id1"]},{"ss":["id2"]},{"s":"PROTOCOL_ECDH"},{"b":true},{"i64":"1048576"},{"s":"CURVE_FOURQ"},{"is_na": false,"ss": ["alice"]}]},"sf_input_ids":["alice-table","bob-table"],"sf_output_ids":["psi-output"],"sf_output_uris":["psi-output.csv"]}' appImage: secretflow-image parties: - domainID: alice @@ -206,7 +220,7 @@ spec: - alias: job-psi2 taskID: job-psi2 priority: 80 - taskInputConfig: '{"sf_datasource_config":{"alice":{"id":"default-data-source"},"bob":{"id":"default-data-source"}},"sf_cluster_desc":{"parties":["alice","bob"],"devices":[{"name":"spu","type":"spu","parties":["alice","bob"],"config":"{\"runtime_config\":{\"protocol\":\"REF2K\",\"field\":\"FM64\"},\"link_desc\":{\"connect_retry_times\":60,\"connect_retry_interval_ms\":1000,\"brpc_channel_protocol\":\"http\",\"brpc_channel_connection_type\":\"pooled\",\"recv_timeout_ms\":1200000,\"http_timeout_ms\":1200000}}"},{"name":"heu","type":"heu","parties":["alice","bob"],"config":"{\"mode\": \"PHEU\", \"schema\": \"paillier\", \"key_size\": 2048}"}],"ray_fed_config":{"cross_silo_comm_backend":"brpc_link"}},"sf_node_eval_param":{"domain":"data_prep","name":"psi","version":"0.0.1","attr_paths":["input/receiver_input/key","input/sender_input/key","protocol","precheck_input","bucket_size","curve_type"],"attrs":[{"ss":["id1"]},{"ss":["id2"]},{"s":"ECDH_PSI_2PC"},{"b":true},{"i64":"1048576"},{"s":"CURVE_FOURQ"}]},"sf_input_ids":["alice-table","bob-table"],"sf_output_ids":["psi-output2"],"sf_output_uris":["psi-output2.csv"]}' + taskInputConfig: '{"sf_datasource_config":{"alice":{"id":"default-data-source"},"bob":{"id":"default-data-source"}},"sf_cluster_desc":{"parties":["alice","bob"],"devices":[{"name":"spu","type":"spu","parties":["alice","bob"],"config":"{\"runtime_config\":{\"protocol\":\"REF2K\",\"field\":\"FM64\"},\"link_desc\":{\"connect_retry_times\":60,\"connect_retry_interval_ms\":1000,\"brpc_channel_protocol\":\"http\",\"brpc_channel_connection_type\":\"pooled\",\"recv_timeout_ms\":1200000,\"http_timeout_ms\":1200000}}"},{"name":"heu","type":"heu","parties":["alice","bob"],"config":"{\"mode\": \"PHEU\", \"schema\": \"paillier\", \"key_size\": 2048}"}],"ray_fed_config":{"cross_silo_comm_backend":"brpc_link"}},"sf_node_eval_param":{"domain":"data_prep","name":"psi","version":"0.0.4","attr_paths":["input/receiver_input/key","input/sender_input/key","protocol","precheck_input","bucket_size","curve_type","left_side"],"attrs":[{"ss":["id1"]},{"ss":["id2"]},{"s":"PROTOCOL_ECDH"},{"b":true},{"i64":"1048576"},{"s":"CURVE_FOURQ"},{"is_na": false,"ss": ["alice"]}]},"sf_input_ids":["alice-table","bob-table"],"sf_output_ids":["psi-output2"],"sf_output_uris":["psi-output2.csv"]}' appImage: secretflow-image parties: - domainID: alice @@ -236,7 +250,7 @@ spec: - alias: job-psi1 taskID: job-psi1 priority: 100 - taskInputConfig: '{"sf_datasource_config":{"alice":{"id":"default-data-source"},"bob":{"id":"default-data-source"}},"sf_cluster_desc":{"parties":["alice","bob"],"devices":[{"name":"spu","type":"spu","parties":["alice","bob"],"config":"{\"runtime_config\":{\"protocol\":\"REF2K\",\"field\":\"FM64\"},\"link_desc\":{\"connect_retry_times\":60,\"connect_retry_interval_ms\":1000,\"brpc_channel_protocol\":\"http\",\"brpc_channel_connection_type\":\"pooled\",\"recv_timeout_ms\":1200000,\"http_timeout_ms\":1200000}}"},{"name":"heu","type":"heu","parties":["alice","bob"],"config":"{\"mode\": \"PHEU\", \"schema\": \"paillier\", \"key_size\": 2048}"}],"ray_fed_config":{"cross_silo_comm_backend":"brpc_link"}},"sf_node_eval_param":{"domain":"data_prep","name":"psi","version":"0.0.1","attr_paths":["input/receiver_input/key","input/sender_input/key","protocol","precheck_input","bucket_size","curve_type"],"attrs":[{"ss":["id1"]},{"ss":["id2"]},{"s":"ECDH_PSI_2PC"},{"b":true},{"i64":"1048576"},{"s":"CURVE_FOURQ"}]},"sf_input_ids":["alice-table","bob-table"],"sf_output_ids":["psi-output"],"sf_output_uris":["psi-output.csv"]}' + taskInputConfig: '{"sf_datasource_config":{"alice":{"id":"default-data-source"},"bob":{"id":"default-data-source"}},"sf_cluster_desc":{"parties":["alice","bob"],"devices":[{"name":"spu","type":"spu","parties":["alice","bob"],"config":"{\"runtime_config\":{\"protocol\":\"REF2K\",\"field\":\"FM64\"},\"link_desc\":{\"connect_retry_times\":60,\"connect_retry_interval_ms\":1000,\"brpc_channel_protocol\":\"http\",\"brpc_channel_connection_type\":\"pooled\",\"recv_timeout_ms\":1200000,\"http_timeout_ms\":1200000}}"},{"name":"heu","type":"heu","parties":["alice","bob"],"config":"{\"mode\": \"PHEU\", \"schema\": \"paillier\", \"key_size\": 2048}"}],"ray_fed_config":{"cross_silo_comm_backend":"brpc_link"}},"sf_node_eval_param":{"domain":"data_prep","name":"psi","version":"0.0.4","attr_paths":["input/receiver_input/key","input/sender_input/key","protocol","precheck_input","bucket_size","curve_type","left_side"],"attrs":[{"ss":["id1"]},{"ss":["id2"]},{"s":"PROTOCOL_ECDH"},{"b":true},{"i64":"1048576"},{"s":"CURVE_FOURQ"},{"is_na": false,"ss": ["alice"]}]},"sf_input_ids":["alice-table","bob-table"],"sf_output_ids":["psi-output"],"sf_output_uris":["psi-output.csv"]}' appImage: secretflow-image parties: - domainID: alice @@ -244,7 +258,7 @@ spec: - alias: job-psi2 taskID: job-psi2 priority: 80 - taskInputConfig: '{"sf_datasource_config":{"alice":{"id":"default-data-source"},"bob":{"id":"default-data-source"}},"sf_cluster_desc":{"parties":["alice","bob"],"devices":[{"name":"spu","type":"spu","parties":["alice","bob"],"config":"{\"runtime_config\":{\"protocol\":\"REF2K\",\"field\":\"FM64\"},\"link_desc\":{\"connect_retry_times\":60,\"connect_retry_interval_ms\":1000,\"brpc_channel_protocol\":\"http\",\"brpc_channel_connection_type\":\"pooled\",\"recv_timeout_ms\":1200000,\"http_timeout_ms\":1200000}}"},{"name":"heu","type":"heu","parties":["alice","bob"],"config":"{\"mode\": \"PHEU\", \"schema\": \"paillier\", \"key_size\": 2048}"}],"ray_fed_config":{"cross_silo_comm_backend":"brpc_link"}},"sf_node_eval_param":{"domain":"data_prep","name":"psi","version":"0.0.1","attr_paths":["input/receiver_input/key","input/sender_input/key","protocol","precheck_input","bucket_size","curve_type"],"attrs":[{"ss":["id1"]},{"ss":["id2"]},{"s":"ECDH_PSI_2PC"},{"b":true},{"i64":"1048576"},{"s":"CURVE_FOURQ"}]},"sf_input_ids":["alice-table","bob-table"],"sf_output_ids":["psi-output1"],"sf_output_uris":["psi-output1.csv"]}' + taskInputConfig: '{"sf_datasource_config":{"alice":{"id":"default-data-source"},"bob":{"id":"default-data-source"}},"sf_cluster_desc":{"parties":["alice","bob"],"devices":[{"name":"spu","type":"spu","parties":["alice","bob"],"config":"{\"runtime_config\":{\"protocol\":\"REF2K\",\"field\":\"FM64\"},\"link_desc\":{\"connect_retry_times\":60,\"connect_retry_interval_ms\":1000,\"brpc_channel_protocol\":\"http\",\"brpc_channel_connection_type\":\"pooled\",\"recv_timeout_ms\":1200000,\"http_timeout_ms\":1200000}}"},{"name":"heu","type":"heu","parties":["alice","bob"],"config":"{\"mode\": \"PHEU\", \"schema\": \"paillier\", \"key_size\": 2048}"}],"ray_fed_config":{"cross_silo_comm_backend":"brpc_link"}},"sf_node_eval_param":{"domain":"data_prep","name":"psi","version":"0.0.4","attr_paths":["input/receiver_input/key","input/sender_input/key","protocol","precheck_input","bucket_size","curve_type","left_side"],"attrs":[{"ss":["id1"]},{"ss":["id2"]},{"s":"PROTOCOL_ECDH"},{"b":true},{"i64":"1048576"},{"s":"CURVE_FOURQ"},{"is_na": false,"ss": ["alice"]}]},"sf_input_ids":["alice-table","bob-table"],"sf_output_ids":["psi-output1"],"sf_output_uris":["psi-output1.csv"]}' appImage: secretflow-image parties: - domainID: alice @@ -305,7 +319,7 @@ spec: - alias: job-psi taskID: job-psi priority: 100 - taskInputConfig: '{"sf_datasource_config":{"alice":{"id":"default-data-source"},"bob":{"id":"default-data-source"}},"sf_cluster_desc":{"parties":["alice","bob"],"devices":[{"name":"spu","type":"spu","parties":["alice","bob"],"config":"{\"runtime_config\":{\"protocol\":\"REF2K\",\"field\":\"FM64\"},\"link_desc\":{\"connect_retry_times\":60,\"connect_retry_interval_ms\":1000,\"brpc_channel_protocol\":\"http\",\"brpc_channel_connection_type\":\"pooled\",\"recv_timeout_ms\":1200000,\"http_timeout_ms\":1200000}}"},{"name":"heu","type":"heu","parties":["alice","bob"],"config":"{\"mode\": \"PHEU\", \"schema\": \"paillier\", \"key_size\": 2048}"}],"ray_fed_config":{"cross_silo_comm_backend":"brpc_link"}},"sf_node_eval_param":{"domain":"data_prep","name":"psi","version":"0.0.1","attr_paths":["input/receiver_input/key","input/sender_input/key","protocol","precheck_input","bucket_size","curve_type"],"attrs":[{"ss":["id1"]},{"ss":["id2"]},{"s":"ECDH_PSI_2PC"},{"b":true},{"i64":"1048576"},{"s":"CURVE_FOURQ"}]},"sf_input_ids":["alice-table","bob-table"],"sf_output_ids":["psi-output"],"sf_output_uris":["psi-output.csv"]}' + taskInputConfig: '{"sf_datasource_config":{"alice":{"id":"default-data-source"},"bob":{"id":"default-data-source"}},"sf_cluster_desc":{"parties":["alice","bob"],"devices":[{"name":"spu","type":"spu","parties":["alice","bob"],"config":"{\"runtime_config\":{\"protocol\":\"REF2K\",\"field\":\"FM64\"},\"link_desc\":{\"connect_retry_times\":60,\"connect_retry_interval_ms\":1000,\"brpc_channel_protocol\":\"http\",\"brpc_channel_connection_type\":\"pooled\",\"recv_timeout_ms\":1200000,\"http_timeout_ms\":1200000}}"},{"name":"heu","type":"heu","parties":["alice","bob"],"config":"{\"mode\": \"PHEU\", \"schema\": \"paillier\", \"key_size\": 2048}"}],"ray_fed_config":{"cross_silo_comm_backend":"brpc_link"}},"sf_node_eval_param":{"domain":"data_prep","name":"psi","version":"0.0.4","attr_paths":["input/receiver_input/key","input/sender_input/key","protocol","precheck_input","bucket_size","curve_type","left_side"],"attrs":[{"ss":["id1"]},{"ss":["id2"]},{"s":"PROTOCOL_ECDH"},{"b":true},{"i64":"1048576"},{"s":"CURVE_FOURQ"},{"is_na": false,"ss": ["alice"]}]},"sf_input_ids":["alice-table","bob-table"],"sf_output_ids":["psi-output"],"sf_output_uris":["psi-output.csv"]}' appImage: secretflow-image parties: - domainID: alice diff --git a/docs/reference/concepts/kusciatask_cn.md b/docs/reference/concepts/kusciatask_cn.md index 3c68f0d90..f946cdb15 100644 --- a/docs/reference/concepts/kusciatask_cn.md +++ b/docs/reference/concepts/kusciatask_cn.md @@ -29,7 +29,7 @@ spec: domainID: alice - appImageRef: secretflow-image domainID: bob - taskInputConfig: '{"sf_datasource_config":{"alice":{"id":"default-data-source"},"bob":{"id":"default-data-source"}},"sf_cluster_desc":{"parties":["alice","bob"],"devices":[{"name":"spu","type":"spu","parties":["alice","bob"],"config":"{\"runtime_config\":{\"protocol\":\"REF2K\",\"field\":\"FM64\"},\"link_desc\":{\"connect_retry_times\":60,\"connect_retry_interval_ms\":1000,\"brpc_channel_protocol\":\"http\",\"brpc_channel_connection_type\":\"pooled\",\"recv_timeout_ms\":1200000,\"http_timeout_ms\":1200000}}"},{"name":"heu","type":"heu","parties":["alice","bob"],"config":"{\"mode\": \"PHEU\", \"schema\": \"paillier\", \"key_size\": 2048}"}],"ray_fed_config":{"cross_silo_comm_backend":"brpc_link"}},"sf_node_eval_param":{"domain":"data_prep","name":"psi","version":"0.0.1","attr_paths":["input/receiver_input/key","input/sender_input/key","protocol","precheck_input","bucket_size","curve_type"],"attrs":[{"ss":["id1"]},{"ss":["id2"]},{"s":"ECDH_PSI_2PC"},{"b":true},{"i64":"1048576"},{"s":"CURVE_FOURQ"}]},"sf_input_ids":["alice-table","bob-table"],"sf_output_ids":["psi-output"],"sf_output_uris":["psi-output.csv"]}' + taskInputConfig: '{"sf_datasource_config":{"alice":{"id":"default-data-source"},"bob":{"id":"default-data-source"}},"sf_cluster_desc":{"parties":["alice","bob"],"devices":[{"name":"spu","type":"spu","parties":["alice","bob"],"config":"{\"runtime_config\":{\"protocol\":\"REF2K\",\"field\":\"FM64\"},\"link_desc\":{\"connect_retry_times\":60,\"connect_retry_interval_ms\":1000,\"brpc_channel_protocol\":\"http\",\"brpc_channel_connection_type\":\"pooled\",\"recv_timeout_ms\":1200000,\"http_timeout_ms\":1200000}}"},{"name":"heu","type":"heu","parties":["alice","bob"],"config":"{\"mode\": \"PHEU\", \"schema\": \"paillier\", \"key_size\": 2048}"}],"ray_fed_config":{"cross_silo_comm_backend":"brpc_link"}},"sf_node_eval_param":{"domain":"data_prep","name":"psi","version":"0.0.4","attr_paths":["input/receiver_input/key","input/sender_input/key","protocol","precheck_input","bucket_size","curve_type","left_side"],"attrs":[{"ss":["id1"]},{"ss":["id2"]},{"s":"PROTOCOL_ECDH"},{"b":true},{"i64":"1048576"},{"s":"CURVE_FOURQ"},{"is_na": false,"ss": ["alice"]}]},"sf_input_ids":["alice-table","bob-table"],"sf_output_ids":["psi-output"],"sf_output_uris":["psi-output.csv"]}' ``` 在该示例中: @@ -43,165 +43,6 @@ spec: - `.spec.parties[1].appImageRef`：表示节点标识为 `alice` 的任务参与方所依赖的应用镜像 AppImage 名称为 `secretflow-image` 。 - `.spec.taskInputConfig`：表示任务输入参数配置。 -1. 运行以下命令创建 KusciaTask。 - -```shell -kubectl apply -f secretflow-task-psi.yaml -``` - -## 查看 KusciaTask - -下面以 KusciaTask `secretflow-task-psi` 为例，介绍如何查看任务运行状态。 - -1. 运行以下命令查看 KusciaTask。 - -```shell -kubectl get kt secretflow-task-psi -n cross-domain -NAME STARTTIME COMPLETIONTIME LASTRECONCILETIME PHASE -secretflow-task-psi 7s 7s 7s Succeeded -``` - -上述命令输出内容，各个列字段的含义如下： -- `NAME`：表示 KusciaTask 的名称，当前示例为 `secretflow-task-psi` 。 -- `STARTTIME`：表示 KusciaTask 从开始执行到现在经历的时间。 -- `COMPLETIONTIME`：表示 KusciaTask 从完成执行到现在经历的时间。 -- `LASTRECONCILETIME`：表示 KusciaTask 从上次被更新到现在经历的时间。 -- `PHASE`：表示 KusciaTask 当前所处的阶段。当前示例阶段为 `Succeeded` 。 - -2. 运行以下命令查看 KusciaTask 详细的状态信息。 - -```shell -kubectl get kt secretflow-task-psi -n cross-domain -o jsonpath={.status} | jq -{ - "completionTime": "2023-08-21T07:43:34Z", - "conditions": [ - { - "lastTransitionTime": "2023-08-21T07:43:15Z", - "status": "True", - "type": "ResourceCreated" - }, - { - "lastTransitionTime": "2023-08-21T07:43:15Z", - "status": "True", - "type": "Running" - }, - { - "lastTransitionTime": "2023-08-21T07:43:34Z", - "status": "True", - "type": "Success" - } - ], - "lastReconcileTime": "2023-08-21T07:43:34Z", - "partyTaskStatus": [ - { - "domainID": "alice", - "phase": "Succeeded" - }, - { - "domainID": "bob", - "phase": "Succeeded" - } - ], - "phase": "Succeeded", - "podStatuses": { - "alice/secretflow-task-psi-0": { - "createTime": "2023-08-21T07:43:15Z", - "namespace": "alice", - "nodeName": "070a9fc7ff24", - "podName": "secretflow-task-psi-0", - "podPhase": "Succeeded", - "readyTime": "2023-08-21T07:43:18Z", - "reason": "Completed", - "startTime": "2023-08-21T07:43:17Z" - }, - "bob/secretflow-task-psi-0": { - "createTime": "2023-08-21T07:43:15Z", - "namespace": "bob", - "nodeName": "dd3bdda2b853", - "podName": "secretflow-task-psi-0", - "podPhase": "Succeeded", - "readyTime": "2023-08-21T07:43:18Z", - "reason": "Completed", - "startTime": "2023-08-21T07:43:17Z" - } - }, - "serviceStatuses": { - "alice/secretflow-task-psi-0-fed": { - "createTime": "2023-08-21T07:43:15Z", - "namespace": "alice", - "portName": "fed", - "portNumber": 8080, - "readyTime": "2023-08-21T07:43:18Z", - "scope": "Cluster", - "serviceName": "secretflow-task-psi-0-fed" - }, - "alice/secretflow-task-psi-0-global": { - "createTime": "2023-08-21T07:43:15Z", - "namespace": "alice", - "portName": "global", - "portNumber": 8081, - "readyTime": "2023-08-21T07:43:18Z", - "scope": "Domain", - "serviceName": "secretflow-task-psi-0-global" - }, - "alice/secretflow-task-psi-0-spu": { - "createTime": "2023-08-21T07:43:15Z", - "namespace": "alice", - "portName": "spu", - "portNumber": 54509, - "readyTime": "2023-08-21T07:43:18Z", - "scope": "Cluster", - "serviceName": "secretflow-task-psi-0-spu" - }, - "bob/secretflow-task-psi-0-fed": { - "createTime": "2023-08-21T07:43:15Z", - "namespace": "bob", - "portName": "fed", - "portNumber": 8080, - "readyTime": "2023-08-21T07:43:18Z", - "scope": "Cluster", - "serviceName": "secretflow-task-psi-0-fed" - }, - "bob/secretflow-task-psi-0-global": { - "createTime": "2023-08-21T07:43:15Z", - "namespace": "bob", - "portName": "global", - "portNumber": 8081, - "readyTime": "2023-08-21T07:43:18Z", - "scope": "Domain", - "serviceName": "secretflow-task-psi-0-global" - }, - "bob/secretflow-task-psi-0-spu": { - "createTime": "2023-08-21T07:43:15Z", - "namespace": "bob", - "portName": "spu", - "portNumber": 54509, - "readyTime": "2023-08-21T07:43:18Z", - "scope": "Cluster", - "serviceName": "secretflow-task-psi-0-spu" - } - }, - "startTime": "2023-08-21T07:43:15Z" -} -``` - -## 清理 KusciaTask - -下面以 KusciaTask `secretflow-task-psi` 为例，介绍如何清理 KusciaTask。 - -1. 运行以下命令清理 KusciaTask。 - -```shell -kubectl delete kt secretflow-task-psi -n cross-domain -``` - -2. 检查 KusciaTask 是否已被清理。 - -```shell -kubectl get kt secretflow-task-psi -n cross-domain -Error from server (NotFound): kusciatasks.kuscia.secretflow "secretflow-task-psi" not found -``` - ## 参考 下面以 `task-template` 模版为例，介绍 KusciaTask 所包含的完整字段。 @@ -219,7 +60,7 @@ spec: resourceReservedSeconds: 30 lifecycleSeconds: 300 retryIntervalSeconds: 15 - taskInputConfig: '{"sf_datasource_config":{"alice":{"id":"default-data-source"},"bob":{"id":"default-data-source"}},"sf_cluster_desc":{"parties":["alice","bob"],"devices":[{"name":"spu","type":"spu","parties":["alice","bob"],"config":"{\"runtime_config\":{\"protocol\":\"REF2K\",\"field\":\"FM64\"},\"link_desc\":{\"connect_retry_times\":60,\"connect_retry_interval_ms\":1000,\"brpc_channel_protocol\":\"http\",\"brpc_channel_connection_type\":\"pooled\",\"recv_timeout_ms\":1200000,\"http_timeout_ms\":1200000}}"},{"name":"heu","type":"heu","parties":["alice","bob"],"config":"{\"mode\": \"PHEU\", \"schema\": \"paillier\", \"key_size\": 2048}"}],"ray_fed_config":{"cross_silo_comm_backend":"brpc_link"}},"sf_node_eval_param":{"domain":"data_prep","name":"psi","version":"0.0.1","attr_paths":["input/receiver_input/key","input/sender_input/key","protocol","precheck_input","bucket_size","curve_type"],"attrs":[{"ss":["id1"]},{"ss":["id2"]},{"s":"ECDH_PSI_2PC"},{"b":true},{"i64":"1048576"},{"s":"CURVE_FOURQ"}]},"sf_input_ids":["alice-table","bob-table"],"sf_output_ids":["psi-output"],"sf_output_uris":["psi-output.csv"]}' + taskInputConfig: '{"sf_datasource_config":{"alice":{"id":"default-data-source"},"bob":{"id":"default-data-source"}},"sf_cluster_desc":{"parties":["alice","bob"],"devices":[{"name":"spu","type":"spu","parties":["alice","bob"],"config":"{\"runtime_config\":{\"protocol\":\"REF2K\",\"field\":\"FM64\"},\"link_desc\":{\"connect_retry_times\":60,\"connect_retry_interval_ms\":1000,\"brpc_channel_protocol\":\"http\",\"brpc_channel_connection_type\":\"pooled\",\"recv_timeout_ms\":1200000,\"http_timeout_ms\":1200000}}"},{"name":"heu","type":"heu","parties":["alice","bob"],"config":"{\"mode\": \"PHEU\", \"schema\": \"paillier\", \"key_size\": 2048}"}],"ray_fed_config":{"cross_silo_comm_backend":"brpc_link"}},"sf_node_eval_param":{"domain":"data_prep","name":"psi","version":"0.0.4","attr_paths":["input/receiver_input/key","input/sender_input/key","protocol","precheck_input","bucket_size","curve_type","left_side"],"attrs":[{"ss":["id1"]},{"ss":["id2"]},{"s":"PROTOCOL_ECDH"},{"b":true},{"i64":"1048576"},{"s":"CURVE_FOURQ"},{"is_na": false,"ss": ["alice"]}]},"sf_input_ids":["alice-table","bob-table"],"sf_output_ids":["psi-output"],"sf_output_uris":["psi-output.csv"]}' parties: - domainID: alice appImageRef: app-template diff --git a/docs/tutorial/run_sf_job_with_api_cn.md b/docs/tutorial/run_sf_job_with_api_cn.md index 93d619024..ffa97ee11 100644 --- a/docs/tutorial/run_sf_job_with_api_cn.md +++ b/docs/tutorial/run_sf_job_with_api_cn.md @@ -94,7 +94,7 @@ docker exec -it ${USER}-kuscia-autonomy-alice ### 使用 Kuscia 示例数据配置 KusciaJob -下面的示例展示了一个 KusciaJob，该任务流完成 2 个任务： +此处以[KusciaJob 示例](../reference/apis/kusciajob_cn.md#请求示例)作为任务示例展示，该任务流完成 2 个任务： 1. job-psi 读取 alice 和 bob 的数据文件，进行隐私求交，求交的结果分别保存为两个参与方的`psi-output.csv`。 2. job-split 读取 alice 和 bob 上一步中求交的结果文件，并拆分成训练集和测试集，分别保存为两个参与方的`train-dataset.csv`、`test-dataset.csv`。 @@ -103,39 +103,6 @@ docker exec -it ${USER}-kuscia-autonomy-alice 我们请求[创建 Job](../reference/apis/kusciajob_cn.md#请求createjobrequest) 接口来创建并运行这个 KusciaJob。 -在 kuscia-master 容器终端中，执行以下命令，内容如下： - -```shell -curl -k -X POST 'https://localhost:8082/api/v1/job/create' \ ---header "Token: $(cat /home/kuscia/var/certs/token)" \ ---header 'Content-Type: application/json' \ ---cert '/home/kuscia/var/certs/kusciaapi-server.crt' \ ---key '/home/kuscia/var/certs/kusciaapi-server.key' \ ---cacert '/home/kuscia/var/certs/ca.crt' \ --d '{ - "job_id": "job-best-effort-linear", - "initiator": "alice", - "max_parallelism": 2, - "tasks": [{ - "app_image": "secretflow-image", - "parties": [{"domain_id": "alice"},{"domain_id": "bob"}], - "alias": "job-psi", - "task_id": "job-psi", - "task_input_config": "{\"sf_datasource_config\":{\"alice\":{\"id\":\"default-data-source\"},\"bob\":{\"id\":\"default-data-source\"}},\"sf_cluster_desc\":{\"parties\":[\"alice\",\"bob\"],\"devices\":[{\"name\":\"spu\",\"type\":\"spu\",\"parties\":[\"alice\",\"bob\"],\"config\":\"{\\\"runtime_config\\\":{\\\"protocol\\\":\\\"REF2K\\\",\\\"field\\\":\\\"FM64\\\"},\\\"link_desc\\\":{\\\"connect_retry_times\\\":60,\\\"connect_retry_interval_ms\\\":1000,\\\"brpc_channel_protocol\\\":\\\"http\\\",\\\"brpc_channel_connection_type\\\":\\\"pooled\\\",\\\"recv_timeout_ms\\\":1200000,\\\"http_timeout_ms\\\":1200000}}\"},{\"name\":\"heu\",\"type\":\"heu\",\"parties\":[\"alice\",\"bob\"],\"config\":\"{\\\"mode\\\": \\\"PHEU\\\", \\\"schema\\\": \\\"paillier\\\", \\\"key_size\\\": 2048}\"}],\"ray_fed_config\":{\"cross_silo_comm_backend\":\"brpc_link\"}},\"sf_node_eval_param\":{\"domain\":\"data_prep\",\"name\":\"psi\",\"version\":\"0.0.1\",\"attr_paths\":[\"input/receiver_input/key\",\"input/sender_input/key\",\"protocol\",\"precheck_input\",\"bucket_size\",\"curve_type\"],\"attrs\":[{\"ss\":[\"id1\"]},{\"ss\":[\"id2\"]},{\"s\":\"ECDH_PSI_2PC\"},{\"b\":true},{\"i64\":\"1048576\"},{\"s\":\"CURVE_FOURQ\"}]},\"sf_input_ids\":[\"alice-table\",\"bob-table\"],\"sf_output_ids\":[\"psi-output\"],\"sf_output_uris\":[\"psi-output.csv\"]}", - "priority": "100" - }, { - "app_image": "secretflow-image", - "parties": [{"domain_id": "alice"},{"domain_id": "bob"}], - "alias": "job-split", - "task_id": "job-split", - "dependencies": ["job-psi"], - "task_input_config": "{\"sf_datasource_config\":{\"alice\":{\"id\":\"default-data-source\"},\"bob\":{\"id\":\"default-data-source\"}},\"sf_cluster_desc\":{\"parties\":[\"alice\",\"bob\"],\"devices\":[{\"name\":\"spu\",\"type\":\"spu\",\"parties\":[\"alice\",\"bob\"],\"config\":\"{\\\"runtime_config\\\":{\\\"protocol\\\":\\\"REF2K\\\",\\\"field\\\":\\\"FM64\\\"},\\\"link_desc\\\":{\\\"connect_retry_times\\\":60,\\\"connect_retry_interval_ms\\\":1000,\\\"brpc_channel_protocol\\\":\\\"http\\\",\\\"brpc_channel_connection_type\\\":\\\"pooled\\\",\\\"recv_timeout_ms\\\":1200000,\\\"http_timeout_ms\\\":1200000}}\"},{\"name\":\"heu\",\"type\":\"heu\",\"parties\":[\"alice\",\"bob\"],\"config\":\"{\\\"mode\\\": \\\"PHEU\\\", \\\"schema\\\": \\\"paillier\\\", \\\"key_size\\\": 2048}\"}],\"ray_fed_config\":{\"cross_silo_comm_backend\":\"brpc_link\"}},\"sf_node_eval_param\":{\"domain\":\"data_prep\",\"name\":\"train_test_split\",\"version\":\"0.0.1\",\"attr_paths\":[\"train_size\",\"test_size\",\"random_state\",\"shuffle\"],\"attrs\":[{\"f\":0.75},{\"f\":0.25},{\"i64\":1234},{\"b\":true}]},\"sf_output_uris\":[\"train-dataset.csv\",\"test-dataset.csv\"],\"sf_output_ids\":[\"train-dataset\",\"test-dataset\"],\"sf_input_ids\":[\"psi-output\"]}", - "priority": "100" - } - ] -}' -``` - 具体字段数据格式和含义请参考[创建 Job](../reference/apis/kusciajob_cn.md#请求createjobrequest) ，本文不再赘述。 如果你成功了，你将得到如下返回： diff --git a/go.mod b/go.mod index 88f302374..24c0c5e55 100644 --- a/go.mod +++ b/go.mod @@ -39,6 +39,7 @@ require ( github.com/spf13/pflag v1.0.5 github.com/stathat/consistent v1.0.0 github.com/stretchr/testify v1.8.4 + github.com/tidwall/gjson v1.14.4 github.com/tidwall/match v1.1.1 gitlab.com/jonas.jasas/condchan v0.0.0-20190210165812-36637ad2b5bc go.uber.org/atomic v1.9.0 @@ -51,6 +52,8 @@ require ( gopkg.in/natefinch/lumberjack.v2 v2.0.0 gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 + gorm.io/driver/mysql v1.3.6 + gorm.io/gorm v1.23.8 gotest.tools/v3 v3.0.3 k8s.io/api v0.28.4 k8s.io/apiextensions-apiserver v0.26.11 @@ -155,6 +158,8 @@ require ( github.com/imdario/mergo v0.3.12 // indirect github.com/inconshreveable/mousetrap v1.0.1 // indirect github.com/infobloxopen/go-trees v0.0.0-20200715205103-96a057b8dfb9 // indirect + github.com/jinzhu/inflection v1.0.0 // indirect + github.com/jinzhu/now v1.1.5 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/jonboulle/clockwork v0.2.2 // indirect github.com/josharian/intern v1.0.0 // indirect @@ -179,6 +184,7 @@ require ( github.com/mrunalp/fileutils v0.5.1 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f // indirect + github.com/natefinch/lumberjack v2.0.0+incompatible // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/runc v1.1.12 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect @@ -195,7 +201,6 @@ require ( github.com/sirupsen/logrus v1.9.3 // indirect github.com/stoewer/go-strcase v1.2.0 // indirect github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect - github.com/tidwall/gjson v1.14.4 // indirect github.com/tidwall/pretty v1.2.0 // indirect github.com/tklauser/go-sysconf v0.3.10 // indirect github.com/tklauser/numcpus v0.4.0 // indirect @@ -286,3 +291,29 @@ replace ( k8s.io/pod-security-admission => k8s.io/pod-security-admission v0.26.11 k8s.io/sample-apiserver => k8s.io/sample-apiserver v0.26.11 ) + +// OPENSOURCE-CLEANUP REMOVE 1 +require gitlab.alipay-inc.com/mist-sdk/mist_sdk_go v1.0.5 + +// OPENSOURCE-CLEANUP REMOVE 18 +require ( + github.com/hashicorp/errwrap v1.0.0 // indirect + github.com/hashicorp/go-multierror v1.1.0 // indirect + github.com/jpillora/backoff v1.0.0 // indirect + github.com/valyala/fastjson v1.5.1 // indirect + gitlab.alipay-inc.com/sofa-go/sofa-antvip-client-go v1.5.0 // indirect + gitlab.alipay-inc.com/sofa-go/sofa-bolt-go v0.5.5 // indirect + gitlab.alipay-inc.com/sofa-go/sofa-bolt-simplemap-go v0.2.1 // indirect + gitlab.alipay-inc.com/sofa-go/sofa-conn-go v0.2.7 // indirect + gitlab.alipay-inc.com/sofa-go/sofa-helper-go v0.1.0 // indirect + gitlab.alipay-inc.com/sofa-go/sofa-hessian-go v0.2.4 // indirect + gitlab.alipay-inc.com/sofa-go/sofa-logger-go v0.2.5 // indirect + gitlab.alipay-inc.com/sofa-go/sofa-registry-client-go v1.2.2 // indirect + gitlab.alipay-inc.com/sofa-go/sofa-registry-proto-go v0.1.5 // indirect + gitlab.alipay-inc.com/sofa-go/sofa-syncpool-go v0.1.3 // indirect + gitlab.alipay-inc.com/sofa-go/sofa-writer-go v0.2.4 // indirect +) + +// OPENSOURCE-CLEANUP REMOVE 1 + +replace github.com/secretflow/kuscia-envoy => gitlab.alipay-inc.com/secretflow/kuscia-envoy v0.0.0-20240401113711-ea55ab052759 diff --git a/go.sum b/go.sum index ee932ea41..e06d6a6bc 100644 --- a/go.sum +++ b/go.sum @@ -42,6 +42,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab h1:UKkYhof1njT1/xq4SEg5z+VpTgjmNeHwPGRQl7takDI= github.com/JeffAshton/win_pdh v0.0.0-20161109143554-76bb4ee9f0ab/go.mod h1:3VYc5hodBMJ5+l/7J4xAyMeuM2PNuepvHlGs8yilUCA= +github.com/Jeffail/tunny v0.0.0-20190930221602-f13eb662a36a h1:sk14oPN106XTe3WzOIaVGq+cFh1sh4z++2pAg2j4XCo= +github.com/Jeffail/tunny v0.0.0-20190930221602-f13eb662a36a/go.mod h1:BX3q3G70XX0UmIkDWfDHoDRquDS1xFJA5VTbMf+14wM= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible h1:1G1pk05UrOh0NlF1oeaaix1x8XzrfjIDK47TY0Zehcw= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ= @@ -51,6 +53,7 @@ github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migc github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -63,8 +66,10 @@ github.com/apache/arrow/go/v13 v13.0.0 h1:kELrvDQuKZo8csdWYqBQfyi431x6Zs/YJTEgUu github.com/apache/arrow/go/v13 v13.0.0/go.mod h1:W69eByFNO0ZR30q1/7Sr9d83zcVZmF2MiP3fFYAWJOc= github.com/apparentlymart/go-cidr v1.1.0 h1:2mAhrMoF+nhXqxTzSZMUzDHkLjmIHC+Zzn4tdgBZjnU= github.com/apparentlymart/go-cidr v1.1.0/go.mod h1:EBcsNrHc3zQeuaeCeCtQruQm+n9/YjEn/vI25Lg7Gwc= +github.com/apparentlymart/go-textseg/v12 v12.0.0/go.mod h1:S/4uRK2UtaQttw1GenVJEynmyUenKwP++x/+DdGV/Ec= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e h1:QEF07wC0T1rKkctt1RINW/+RMTVmiwxETico2l3gxJA= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio= github.com/aws/aws-sdk-go v1.35.24/go.mod h1:tlPOdRjfxPBpNIwqDj61rmsnA85v9jc0Ps9+muhnW+k= github.com/aws/aws-sdk-go v1.44.116 h1:NpLIhcvLWXJZAEwvPj3TDHeqp7DleK6ZUVYyW01WNHY= @@ -91,6 +96,7 @@ github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyY github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.4.1 h1:iKLQ0xPNFxR/2hzXZMrBo8f1j86j5WHzznCCQxV/b8g= github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= @@ -132,12 +138,18 @@ github.com/coredns/caddy v1.1.1 h1:2eYKZT7i6yxIfGP3qLJoJ7HAsDJqYB+X68g4NYjSrE0= github.com/coredns/caddy v1.1.1/go.mod h1:A6ntJQlAWuQfFlsd9hvigKbo2WS0VUs2l1e2F+BawD4= github.com/coredns/coredns v1.10.0 h1:jCfuWsBjTs0dapkkhISfPCzn5LqvSRtrFtaf/Tjj4DI= github.com/coredns/coredns v1.10.0/go.mod h1:CIfRU5TgpuoIiJBJ4XrofQzfFQpPFh32ERpUevrSlaw= +github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= @@ -150,6 +162,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/daviddengcn/go-colortext v1.0.0 h1:ANqDyC0ys6qCSvuEK7l3g5RaehL/Xck9EX8ATG8oKsE= github.com/daviddengcn/go-colortext v1.0.0/go.mod h1:zDqEI5NVUop5QPpVJUxE9UO10hRnmkD5G4Pmri9+m4c= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/dnstap/golang-dnstap v0.4.0 h1:KRHBoURygdGtBjDI2w4HifJfMAhhOqDuktAokaSa234= github.com/dnstap/golang-dnstap v0.4.0/go.mod h1:FqsSdH58NAmkAvKcpyxht7i4FoBjKu8E4JUPt8ipSUs= github.com/docker/distribution v2.8.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= @@ -190,6 +204,7 @@ github.com/farsightsec/golang-framestream v0.3.0 h1:/spFQHucTle/ZIPkYqrfshQqPe2V github.com/farsightsec/golang-framestream v0.3.0/go.mod h1:eNde4IQyEiA5br02AouhEHCu3p3UzrCdFR4LuQHklMI= github.com/fatih/camelcase v1.0.0 h1:hxNvNX/xYBp0ovncs8WyWZrOrpBNub/JfaMvbURyft8= github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc= +github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= @@ -199,6 +214,7 @@ github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI github.com/form3tech-oss/jwt-go v3.2.3+incompatible h1:7ZaBxOI7TMoYBfyA3cQHErNNyAWIKUMIwqxEtgHOs5c= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/frankban/quicktest v1.14.5 h1:dfYrrRyLtiqT9GyKXgdh+k4inNeTvmGbuSgZ3lx3GhA= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/fvbommel/sortorder v1.0.1 h1:dSnXLt4mJYH25uDDGa3biZNQsozaUWDSWeKJ0qqFfzE= @@ -263,12 +279,15 @@ github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gogo/googleapis v1.4.1/go.mod h1:2lpHqI5OcWCtVElxXnPt+s8oJvMpySlOyM6xDCrzib4= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE= github.com/golang-jwt/jwt/v5 v5.0.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/glog v1.1.2 h1:DVjP2PbBOzHyzA+dn3WhHIq4NdVu3Q+pvivFICf/7fo= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -283,6 +302,7 @@ github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= +github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -353,41 +373,58 @@ github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 h1:pdN6V1QBWetyv/0+wjACpqVH+eVULgEjkurDLq3goeM= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDaL56wXCB/5+wF6uHfaI= github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.0 h1:RtRsiaGvWxcwd8y3BiRZxsylPT8hLWZ5SPcfI+3IDNk= github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.0/go.mod h1:TzP6duP4Py2pHLVPPQp42aoYI92+PCrVotyR5e8Vqlk= github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645 h1:MJG/KsmcqMwFAkh8mTnAwhyKoB+sTAnY4CACC110tbU= github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645/go.mod h1:6iZfnjpejD4L/4DwD7NryNaJyCQdzwWwH2MWhCA90Kw= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.1.0 h1:B9UzwGQJehnUY1yNrnwREHc3fGbC2xefo8g4TbElacI= +github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru/v2 v2.0.4 h1:7GHuZcgid37q8o5i3QI9KMT4nCWQQ3Kx3Ov6bb9MfK0= github.com/hashicorp/golang-lru/v2 v2.0.4/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/infobloxopen/go-trees v0.0.0-20200715205103-96a057b8dfb9 h1:w66aaP3c6SIQ0pi3QH1Tb4AMO3aWoEPxd1CNvLphbkA= github.com/infobloxopen/go-trees v0.0.0-20200715205103-96a057b8dfb9/go.mod h1:BaIJzjD2ZnHmx2acPF6XfGLPzNCMiBbMRqJr+8/8uRI= +github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= +github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= +github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= +github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9qUBdQ= github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -396,8 +433,12 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k= +github.com/k0kubun/pp v3.0.1+incompatible/go.mod h1:GWse8YhT0p8pT4ir3ZgBbfZild3tgzSScAn6HmfYukg= github.com/karrick/godirwalk v1.17.0 h1:b4kY7nqDdioR/6qnbHQyDvmA17u5G1cZ6J+CZXwSWoI= github.com/karrick/godirwalk v1.17.0/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= +github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= @@ -422,17 +463,23 @@ github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhn github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffktY= github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= +github.com/logrusorgru/aurora v0.0.0-20200102142835-e9ef32dff381/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/lufia/plan9stats v0.0.0-20220517141722-cf486979b281 h1:aczX6NMOtt6L4YT0fQvKkDK6LZEtdOso9sUH89V1+P0= github.com/lufia/plan9stats v0.0.0-20220517141722-cf486979b281/go.mod h1:lc+czkgO/8F7puNki5jk8QyujbfK1LOT7Wl0ON2hxyk= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= @@ -448,12 +495,14 @@ github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA= github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= github.com/mindprince/gonvml v0.0.0-20190828220739-9ebdce4bb989 h1:PS1dLCGtD8bb9RPKJrc8bS7qHL6JnW1CZvwzH9dPoUs= github.com/mindprince/gonvml v0.0.0-20190828220739-9ebdce4bb989/go.mod h1:2eu9pRWp8mo84xCg6KswZ+USQHjwgRhNp06sozOdsTY= +github.com/minio/highwayhash v1.0.0/go.mod h1:xQboMTeM9nY9v/LlAOxFctujiv5+Aq2hR5dxBpaMbdc= github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible h1:aKW/4cBs+yK6gpqU3K/oIwk9Q/XICqd3zOX/UFuvqmk= github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-wordwrap v1.0.0 h1:6GlHJ/LTGMrIJbwgdqdl2eEH8o+Exx/0m8ir9Gns0u4= github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= @@ -485,8 +534,11 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f h1:y5//uYreIhSUg3J1GEMiLbxo1LJaP8RfCpH6pymGZus= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/natefinch/lumberjack v2.0.0+incompatible h1:4QJd3OLAMgj7ph+yZTuX13Ld4UpgHp07nNdFX7mqFfM= +github.com/natefinch/lumberjack v2.0.0+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6YfsmYQpsTIOm0B1VNzQg9Mw6nPk= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo/v2 v2.4.0 h1:+Ig9nvqgS5OBSACXNk15PLdp0U9XPYROt9CFzVdFGIs= github.com/onsi/gomega v1.23.0 h1:/oxKu9c2HVap+F3PfKort2Hw5DEU+HGlW8n+tguWsys= @@ -507,8 +559,12 @@ github.com/opencontainers/selinux v1.11.0/go.mod h1:E5dMC3VPuVvVHDYmi78qvhJp8+M5 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= +github.com/panjf2000/ants/v2 v2.4.1 h1:7RtUqj5lGOw0WnZhSKDZ2zzJhaX5490ZW1sUolRXCxY= +github.com/panjf2000/ants/v2 v2.4.1/go.mod h1:f6F0NZVFsGCp5A7QW/Zj/m92atWwOkY0OIhFxRNFr4A= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= +github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI= github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= @@ -524,6 +580,7 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= @@ -538,6 +595,8 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1: github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= @@ -546,6 +605,7 @@ github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJ github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= @@ -553,6 +613,8 @@ github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= @@ -565,8 +627,6 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646 h1:RpforrEYXWkmGwJHIGnLZ3tTWStkjVVstwzNGqxX2Ds= github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= -github.com/secretflow/kuscia-envoy v0.0.0-20240402083426-b0884d002f48 h1:4W2Nx2lalcZebbyytroXk4LltprKzZSBurqD//SMcZ4= -github.com/secretflow/kuscia-envoy v0.0.0-20240402083426-b0884d002f48/go.mod h1:022TT/HpFoj5iYB6Q/XI0NBnqgM5rekwUo6NAY/K8oA= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/shirou/gopsutil/v3 v3.22.6 h1:FnHOFOh+cYAM0C30P+zysPISzlknLC5Z1G4EAElznfQ= github.com/shirou/gopsutil/v3 v3.22.6/go.mod h1:EdIubSnZhbAvBS1yJ7Xi+AShB/hxwLHOMz4MCYz7yMs= @@ -578,13 +638,20 @@ github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrf github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/soheilhy/cmux v0.1.5 h1:jjzc5WVemNEDTLwv9tlmemhC73tI08BNOIGwBOo10Js= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE= github.com/stathat/consistent v1.0.0 h1:ZFJ1QTRn8npNBKW065raSZ8xfOqhpb8vLOkfp4CcL/U= github.com/stathat/consistent v1.0.0/go.mod h1:uajTPbgSygZBJ+V+0mY7meZ8i0XAcZs7AQ6V121XSxw= github.com/stoewer/go-strcase v1.2.0 h1:Z2iHWqGXH00XYgqDmNgQbIBxf3wrNq0F3feEy0ainaU= @@ -613,20 +680,25 @@ github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.0.1/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tklauser/go-sysconf v0.3.10 h1:IJ1AZGZRWbY8T5Vfk04D9WOA5WSejdflXxP03OUqALw= github.com/tklauser/go-sysconf v0.3.10/go.mod h1:C8XykCvCb+Gn0oNCWPIlcb0RuglQTYaQ2hGm7jmxEFk= github.com/tklauser/numcpus v0.4.0 h1:E53Dm1HjH1/R2/aoCtXtPgzmElmn51aOkhCFSuZq//o= github.com/tklauser/numcpus v0.4.0/go.mod h1:1+UI3pD8NW14VMwdgJNJ1ESk2UnwhAnz5hMwiKKqXCQ= +github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802 h1:uruHq4dN7GR16kFc5fp3d1RIYzJW5onx8Ybykw2YQFA= github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI= github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08= +github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ugorji/go/codec v1.2.11 h1:BMaWp1Bb6fHwEtbplGBGJ498wD+LKlNSl25MjdZY4dU= github.com/ugorji/go/codec v1.2.11/go.mod h1:UNopzCgEMSXjBc6AOMqYvWC1ktqTAfzJZUZgYf6w6lg= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fastjson v1.5.1 h1:SXaQZVSwLjZOVhDEhjiCcDtnX0Feu7Z7A1+C5atpoHM= +github.com/valyala/fastjson v1.5.1/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= github.com/vishvananda/netlink v1.2.1-beta.2 h1:Llsql0lnQEbHj0I1OuKyp8otXp0r3q0mPkuhwHfStVs= github.com/vishvananda/netlink v1.2.1-beta.2/go.mod h1:twkDnbuQxJYemMlGd4JFIcuhgX83tXhKS2B/PRMpOho= @@ -634,9 +706,12 @@ github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17 github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/vishvananda/netns v0.0.4 h1:Oeaw1EM2JMxD51g9uhtC0D7erkIjgmj8+JZc26m1YX8= github.com/vishvananda/netns v0.0.4/go.mod h1:SpkAiCQRtJ6TvvxPnOSyH3BMl6unz3xZlaprSwhNNJM= +github.com/vmihailenco/msgpack v3.3.3+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/treeprint v1.1.0 h1:G/1DjNkPpfZCFt9CSh6b5/nY4VimlbHF3Rh4obvtzDk= github.com/xlab/treeprint v1.1.0/go.mod h1:gj5Gd3gPdKtR1ikdDK6fnFLdmIS0X30kTTuNd/WEJu0= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -645,11 +720,48 @@ github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/zclconf/go-cty v1.5.0/go.mod h1:nHzOclRkoj++EU9ZjSrZvRG0BXIWt8c7loYc0qXAFGQ= github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= +gitlab.alipay-inc.com/mist-sdk/mist_sdk_go v1.0.5 h1:5izJu6040FBkc2lSd5ijjB4zJb6Nw2Q73pAPS2sNRuE= +gitlab.alipay-inc.com/mist-sdk/mist_sdk_go v1.0.5/go.mod h1:rCdu+shT1JOYvFPCSBe0uJIUrvA2e4Cyy8UHvjuctSU= +gitlab.alipay-inc.com/secretflow/kuscia-envoy v0.0.0-20240401113711-ea55ab052759 h1:WT/XVx47Z1CB5XcbXtghdbTJBzGIwBHQ6WiZnIU9f/8= +gitlab.alipay-inc.com/secretflow/kuscia-envoy v0.0.0-20240401113711-ea55ab052759/go.mod h1:022TT/HpFoj5iYB6Q/XI0NBnqgM5rekwUo6NAY/K8oA= +gitlab.alipay-inc.com/sofa-go/sofa-antvip-client-go v1.5.0 h1:Gd2Pm+V3hMtDn7gF1aC9VaBDy9r6a60W3Lz8tB/57+g= +gitlab.alipay-inc.com/sofa-go/sofa-antvip-client-go v1.5.0/go.mod h1:2EB1fevaDXOvfZzmfw3OjmscFC2dfK6LRzY6ed3cp1A= +gitlab.alipay-inc.com/sofa-go/sofa-bolt-go v0.3.4/go.mod h1:usIoMnEPEbl2g1jHzHqwlO2giwdvtDYfPTje3qk4q/k= +gitlab.alipay-inc.com/sofa-go/sofa-bolt-go v0.5.5 h1:vUiMH3Ux+pUGxAk6kEb6fIxh3nkAJh6K6GR7B2yfUCg= +gitlab.alipay-inc.com/sofa-go/sofa-bolt-go v0.5.5/go.mod h1:usaupzDvhZsyfFqPiMH8/333sPDbfqmCnqBVJKQCYbw= +gitlab.alipay-inc.com/sofa-go/sofa-bolt-simplemap-go v0.2.1 h1:06uuhwM2MNGAxKa4hKAi4zkWeYsxTtStgivzOAeWuTI= +gitlab.alipay-inc.com/sofa-go/sofa-bolt-simplemap-go v0.2.1/go.mod h1:dJ6T9VtNnvw6qNqNtoHRlZyMGnhqAh3uksH64AB3Qjw= +gitlab.alipay-inc.com/sofa-go/sofa-conn-go v0.2.7 h1:/7/qulC//Z6EWMqwP04lXnOwGSFRYNeWAnhpnjISbqQ= +gitlab.alipay-inc.com/sofa-go/sofa-conn-go v0.2.7/go.mod h1:9cyalwZU11oKB5chfSvoBT1x68orAUxidQ6xA+3oVHk= +gitlab.alipay-inc.com/sofa-go/sofa-dubbo-go v0.2.4/go.mod h1:lqqxEwrrgIhM3PzPWrypoFo1p+FEbRtu443Xc0ngnow= +gitlab.alipay-inc.com/sofa-go/sofa-helper-go v0.0.0-20200701091524-59d4d94d5962/go.mod h1:PIZnhPuUJ+0ZtUb69y3Lc1I5lr9Fbctug4SCeh8+Rb4= +gitlab.alipay-inc.com/sofa-go/sofa-helper-go v0.0.0-20200702062042-ea0c25c33ddc/go.mod h1:VSBhgZY/SchcMGSOd7SQ7Q5zFPdVJm+2vK0vMfzfqEE= +gitlab.alipay-inc.com/sofa-go/sofa-helper-go v0.0.0-20200703064733-390c5a5a46ac/go.mod h1:OmjLLhDplmW54S6PP+LDachPs1uzEXEcWIAe6pxJxxM= +gitlab.alipay-inc.com/sofa-go/sofa-helper-go v0.1.0 h1:O/TF7joZ/bBJlhtIYWSBWWYEUONrCTB5B+JqjWva1lg= +gitlab.alipay-inc.com/sofa-go/sofa-helper-go v0.1.0/go.mod h1:OmjLLhDplmW54S6PP+LDachPs1uzEXEcWIAe6pxJxxM= +gitlab.alipay-inc.com/sofa-go/sofa-hessian-go v0.2.4 h1:iWmgAOmiDGoawgin+FnSV5ySw4qFpOkiAivKZ6LGMu8= +gitlab.alipay-inc.com/sofa-go/sofa-hessian-go v0.2.4/go.mod h1:FgA/OQjeGJ62QKO2DLNlsP6YZZrNLoTiAzBHYh73cZA= +gitlab.alipay-inc.com/sofa-go/sofa-logger-go v0.2.5 h1:tLOHv2vmYiwhTsixj1Z7A+7C0HtcGVbIO8V8/tDfZoQ= +gitlab.alipay-inc.com/sofa-go/sofa-logger-go v0.2.5/go.mod h1:HERXWBP+TckWt1pI7NTKLDYE1SvVCJq98KfaIq3Scwg= +gitlab.alipay-inc.com/sofa-go/sofa-registry-client-go v1.1.0/go.mod h1:X28irvodN9K4NIAzPBxwnj6dVDvxKMLTXq/m+V0kySw= +gitlab.alipay-inc.com/sofa-go/sofa-registry-client-go v1.2.2 h1:Bs3MfxkYglCP+RL7/52EuUZOL4WVGha1uvO95dGx/tI= +gitlab.alipay-inc.com/sofa-go/sofa-registry-client-go v1.2.2/go.mod h1:uP0yeYvwEi4bSINaUrzgAo4/wAdIbXnLRCGU0KRxDGo= +gitlab.alipay-inc.com/sofa-go/sofa-registry-proto-go v0.1.1/go.mod h1:8A7qTHMyXwKH16uW14DqY/Jz4J2Av4ym0uXhj3noJ/4= +gitlab.alipay-inc.com/sofa-go/sofa-registry-proto-go v0.1.5 h1:Il/nYoORezREgnHTG1Ls/9UZx4QFpiI1pIHT9fPVpL8= +gitlab.alipay-inc.com/sofa-go/sofa-registry-proto-go v0.1.5/go.mod h1:8A7qTHMyXwKH16uW14DqY/Jz4J2Av4ym0uXhj3noJ/4= +gitlab.alipay-inc.com/sofa-go/sofa-syncpool-go v0.1.3 h1:mQXuovwFvpVfGm2b1va9D7l8HofCbpQkBdEGrVwd7bM= +gitlab.alipay-inc.com/sofa-go/sofa-syncpool-go v0.1.3/go.mod h1:ZbHcEQDu9CKZX5KXVP/VMQVisPhpnjhCtcMasl5jvGE= +gitlab.alipay-inc.com/sofa-go/sofa-writer-go v0.2.0/go.mod h1:aYzSEiYI34dcs/d49P2Cl55/In7NL7MHmphfrFFwBeM= +gitlab.alipay-inc.com/sofa-go/sofa-writer-go v0.2.3/go.mod h1:pKHJ+4hHct6D3r0HBTAzCjnpRbDBVXrAycmWx64I2hk= +gitlab.alipay-inc.com/sofa-go/sofa-writer-go v0.2.4 h1:CVi6v3w4tIxksGYFuNMt1qMbfzzAlNVTcRilXd8ADK4= +gitlab.alipay-inc.com/sofa-go/sofa-writer-go v0.2.4/go.mod h1:pKHJ+4hHct6D3r0HBTAzCjnpRbDBVXrAycmWx64I2hk= gitlab.com/jonas.jasas/condchan v0.0.0-20190210165812-36637ad2b5bc h1:zCsu+odZEHb2f8U8WWhDgY5N5w3JCLHxuCIqVqCsLcQ= gitlab.com/jonas.jasas/condchan v0.0.0-20190210165812-36637ad2b5bc/go.mod h1:4JS8TdA7HSdK+x43waOdTGodqY/VKsj4w+8pWDL0E88= +go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.8 h1:xs88BrvEv273UsB79e0hcVrlUWmS0a8upikMFhSyAtA= go.etcd.io/bbolt v1.3.8/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= go.etcd.io/etcd/api/v3 v3.5.6 h1:Cy2qx3npLcYqTKqGJzMypnMv2tiRyifZJ17BlWIWA7A= @@ -689,14 +801,26 @@ go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lI go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 h1:+FNtrFTmVw0YZGpBGX56XDee331t6JAXeK2bcyhLOOc= go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0Hg7FvpRQsQh5OSqIylirxKC7o= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.5.1/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= +go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0 h1:ECmE8Bn/WFTYwEW/bpKD3M8VtR/zQVbavAoalC1PYyE= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8= go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= +go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= go.uber.org/zap v1.18.1/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI= go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= @@ -753,12 +877,14 @@ golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -816,8 +942,11 @@ golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -827,6 +956,7 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -902,7 +1032,9 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= @@ -918,6 +1050,8 @@ golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgw golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= @@ -932,6 +1066,7 @@ golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207224406-61798d64f025/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= @@ -1026,6 +1161,7 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20231212172506-995d672761c0 h1: google.golang.org/genproto/googleapis/rpc v0.0.0-20231212172506-995d672761c0/go.mod h1:FUoWkonphQm3RhTS+kOEhF8h0iDpm4tdXolVCeZ9KKA= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= @@ -1072,12 +1208,15 @@ gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= @@ -1087,6 +1226,10 @@ gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gorm.io/driver/mysql v1.3.6 h1:BhX1Y/RyALb+T9bZ3t07wLnPZBukt+IRkMn8UZSNbGM= +gorm.io/driver/mysql v1.3.6/go.mod h1:sSIebwZAVPiT+27jK9HIwvsqOGKx3YMPmrA3mBJR10c= +gorm.io/gorm v1.23.8 h1:h8sGJ+biDgBA1AD1Ha9gFCx7h8npU7AsLdlkX0n2TpE= +gorm.io/gorm v1.23.8/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= diff --git a/hack/k3s/README.md b/hack/k3s/README.md new file mode 100644 index 000000000..4ad5439c9 --- /dev/null +++ b/hack/k3s/README.md @@ -0,0 +1,40 @@ + +## 常用环境变量 + +目前支持下面环境变量配置 + + +| 名称 | 值 | 含义 | +| --------------------------------------- | -------------------------- |---------------------------------------------------------------------------------------------------------------------------------------------------------| +| USE_KINE_MYSQL_V2 | "true"/"false" | 是否使用Kine Mysql V2版本，即内部调优后的版本，默认为"false" | +| KINE_LOG_LEVEL | "trace", "debug", "info" | Kine日志级别 | +| KINE_METRICS_PORT | "11080" | Kine指标服务监听端口，默认为"11080" | +| KINE_DISABLE_PROFILING | "true"/"false" | 关闭 Kine 性能调试，默认为: "false" | +| KINE_SKIP_INIT_MYSQL | "true"/"false" | 是否跳过初始化数据库步骤(创建数据库，表结构)，默认为"false" | +| KINE_MYSQL_ISOLATION_LEVEL | "Serializable" | 设置mysql事物隔离级别，支持两种"ReadCommitted"和"Serializable"。默认为"ReadCommitted" | +| KINE_LIST_TTL_EVENT_INTERVAL_MINUTE | "10" | 检查 ttl event 是否过期间隔时间，默认为："10" | +| KINE_COMPACT_INTERVAL_MINUTE | "5" | comact 间隔时间，默认为"5" | +| KINE_COMPACT_BATCH_SIZE | "1000" | compact 步长大小，默认为: "1000" | +| KINE_COMPACT_DELETE_SIZE | "1000" | compact 删除大小，默认为: "1000"; value(KINE_COMPACT_DELETE_SIZE) <= value(KINE_COMPACT_BATCH_SIZE) | +| KINE_COMPACT_TIMEOUT_SECOND | "10" | compact 超时时间，默认为: "10" | +| KINE_COMPACT_RETENTION_MINUTE | "5" | compact 时，跳过最近几分钟内的数据，默认为 "5" | +| KINE_DB_POLL_WAIT_MILLISECOND | "1000" | 从DB Polll 数据的等待时间，默认为 "1000" | +| KINE_DB_MAX_IDLE_CONNS | "2" | 和DB的最大空闲连接数，默认为 "2"，参考:[https://pkg.go.dev/database/sql#DB.SetMaxIdleConns](https://pkg.go.dev/database/sql#DB.SetMaxIdleConns) | +| KINE_DB_MAX_OPEN_CONNS | "0" | 和DB的最大连接数，默认为"0"，不限制，参考:[https://pkg.go.dev/database/sql#DB.SetMaxOpenConns](https://pkg.go.dev/database/sql#DB.SetMaxOpenConns) | +| KINE_DB_MAX_LIFETIME_SECOND | "0" | 和DB的连接超时时间，默认为"0"，不限制，依赖底层connection超时，参考[https://pkg.go.dev/database/sql#DB.SetConnMaxLifetime](https://pkg.go.dev/database/sql#DB.SetConnMaxLifetime) | +| CATTLE_NEW_SIGNED_CERT_EXPIRATION_DAYS | "3650" | 证书过期时间，默认为"365" | + + +## 性能指标 + +### Metrics +```shell +curl -v http://127.0.0.1:11080/metrics +``` + +### Debug + +- http://127.0.0.1:11080/debug/pprof/profile +- http://127.0.0.1:11080/debug/pprof/trace +- http://127.0.0.1:11080/debug/pprof/symbol +- http://127.0.0.1:11080/debug/pprof/cmdline \ No newline at end of file diff --git a/hack/k3s/build.sh b/hack/k3s/build.sh index 4f34514c3..2f807f9c1 100644 --- a/hack/k3s/build.sh +++ b/hack/k3s/build.sh @@ -22,7 +22,8 @@ function build_k3s() { echo "build k3s binary..." if [[ $(ls -A "build/k3s") = "" ]];then git clone -b $k3s_version --depth 1 https://github.com/k3s-io/k3s.git build/k3s - KINE_VERSION="github.com/k3s-io/kine@v0.11.3" + # OPENSOURCE-CLEANUP SUB gitlab.alipay-inc.com/secretflow/kuscia-kine@latest github.com/k3s-io/kine@v0.11.3 + KINE_VERSION="gitlab.alipay-inc.com/secretflow/kuscia-kine@latest" pushd build/k3s go mod edit -replace github.com/k3s-io/kine="${KINE_VERSION}" pushd diff --git a/internaldoc/.OPENSOURCE-CLEANUP-RMDIR b/internaldoc/.OPENSOURCE-CLEANUP-RMDIR new file mode 100644 index 000000000..e69de29bb diff --git a/internaldoc/README.md b/internaldoc/README.md new file mode 100644 index 000000000..3dd58cd7e --- /dev/null +++ b/internaldoc/README.md @@ -0,0 +1,112 @@ +## 说明 +本说明文档仅限内部使用，不会带到开源仓库。 + +## 开发环境搭建 +安装 golang + +```shell +# mac +wget https://go.dev/dl/go1.19.4.darwin-amd64.tar.gz +sudo tar -C /usr/local -zxvf go1.19.4.darwin-amd64.tar.gz + +# linux +wget https://go.dev/dl/go1.19.4.linux-amd64.tar.gz +sudo tar -C /usr/local -zxvf go1.19.4.linux-amd64.tar.gz +``` + +添加环境变量 +```shell +export GOPROXY="https://goproxy.cn,direct" +export GO111MODULE=on +export GOPATH="$HOME/gopath" +export PATH="$PATH:$GOPATH/bin:/usr/local/go/bin" +export GOPRIVATE="gitlab.alipay-inc.com" +``` + +安装 protoc +```shell +# 更多版本信息：https://github.com/protocolbuffers/protobuf/releases/tag/v21.8 +# mac x86_64 +PROTOC_ZIP=protoc-21.8-osx-x86_64.zip +# linux x86_64 +PROTOC_ZIP=protoc-21.8-linux-x86_64.zip + +curl -OL https://github.com/protocolbuffers/protobuf/releases/download/v21.8/$PROTOC_ZIP +unzip -o $PROTOC_ZIP -d /usr/local bin/protoc +unzip -o $PROTOC_ZIP -d /usr/local 'include/*' +rm -f $PROTOC_ZIP +``` + +git 配置 +```shell +git config --global user.name "xxx" +git config --global user.email "xxx@antgroup.com" + +# http/https 免密码登录（首次需要密码验证） +git config --global credential.helper store + +# 通过 ssh 访问 gitlab，需要加这行配置 +git config --global url."git@gitlab.alipay-inc.com:".insteadOf "http://gitlab.alipay-inc.com/" +``` +ssh 配置 +```shell +# 生成 ssh 公私钥 +ssh-keygen -t rsa -b 4096 -f ~/.ssh/id_rsa -N '' + +# 将公钥内容填至 AntCode->设置->SSH密钥 +cat ~/.ssh/id_rsa.pub +``` + + + +## 构建 + +### 构建环境 + +1. 本地环境 + + 参考“开发环境搭建”搭建本地构建环境。 + +2. docker 环境 + + 后台启动 kuscia-dev 构建镜像： + + ``` + cd ${your kuscia direcotry} + docker run -dit --name kuscia-dev-$(whoami) -v "$(pwd)":/home/admin/dev/ -w /home/admin/dev reg.docker.alibaba-inc.com/secretflow/kuscia-dev:0.5 /sbin/init + + docker exec -it kuscia-dev-$(whoami) bash + cd /home/admin/dev/ && ...... + ``` + + + +### crd 构建 +```shell +# 生成 crd spec +hack/generate-crds.sh + +# 生成 crd client code +hack/update-codegen.sh +``` + + + +### proto 构建 +```shell +# 生成 golang 代码 +hack/proto-to-go.sh +``` + +## 代码规范 + +### 参考： +- https://golang.org/doc/effective_go.html +- https://github.com/golang/go/wiki/CodeReviewComments +### 补充： +todo + + +## 常见问题 +#### 1. Goland 报错 Found several packages [syscall, main] in '/usr/local/go/src/syscall;/usr/local/go/src/syscall' +当前 kuscia 框架使用 go1.19.4，如果 Goland 版本较低会报错，需要升级 Goland 版本 \ No newline at end of file diff --git a/internaldoc/app_access_profile.md b/internaldoc/app_access_profile.md new file mode 100644 index 000000000..8393b32ff --- /dev/null +++ b/internaldoc/app_access_profile.md @@ -0,0 +1,329 @@ +# 概要 + +隐私计算应用接入 Kuscia，需要遵循一些规范，主要包括配置模板、协议、部署模板、镜像等。 + + + +# 镜像 + +Kuscia 支持拉起 docker 镜像，为了支持 Kuscia 的一些特性，应用在构建镜像时，需要将某些配置以 Labels 的形式打到镜像中。 + +### Labels + +- kuscia.secretflow.config-templates：配置模板。 +- kuscia.secretflow.deploy-templates：部署模板。 + +### Dockerfile 示例 + +```dockerfile +FROM openanolis/anolisos:8.8 + +LABEL kuscia.secretflow.config-templates="xxxxxx" +LABEL kuscia.secretflow.kuscia.secretflow.deploy-templates="xxxxxx" + +COPY secretflow /home/admin/secretflow +``` + + + +# 配置模板 + +应用可以提供一份配置模板给 Kuscia，模板中支持变量并由 Kuscia 动态渲染，渲染出的一份或多份配置文件最终会挂载到应用的容器中，挂载的位置请参考部署模版的介绍。 + + + +### 模板格式 + +配置模板的格式 为 YAML，包含多个 key-value 配置项。其中，每一个配置项的 key 为配置模板的名称，value 为配置模板的内容。示例： + +```yaml +task-config.conf: | +{ + task_id: {{.TASK_ID}} + task_input_config: {{.TASK_INPUT_CONFIG}} + task_input_cluster_def: {{.TASK_CLUSTER_DEF}} + allocated_ports: {{.ALLOCATED_PORTS}} + ... +} +server.conf: | +{ + dbhost={{.DB_HOST}} + dbport=10000 +} +key.conf: abedfalc81230dfaacda +``` + +Kuscia 不感知 value 具体的格式，它由应用自己制定，最终也由应用自己解析。 + + + +### 变量支持 + +配置模板中的变量格式需要符合 [Golang template](https://pkg.go.dev/text/template) 的规则，因此理论上，模板内容语法的支持不仅限于变量，还支持条件判断、循环控制等 Golang template 提供的高级语法。 + +Kuscia 默认支持以下变量的渲染： + +| 变量名 | 描述 | +|---------------------|---------| +| TASK_ID | 任务ID | +| TASK_INPUT_CONFIG | 任务参数配置 | +| TASK_CLUSTER_DEFINE | 任务参与方信息 | +| ALLOCATED_PORTS | 给应用的端口 | + +若要支持自定义的变量，只需要在节点本地接入的配置系统配置即可。 + + + +# 协议 + +Kuscia 在拉起应用时，会将一些必要的参数以文件或环境变量的方式传递给应用。 + + + +### TASK_CLUSTER_DEFINE + +TASK_CLUSTER_DEFINE 定义了执行任务的各参与方的信息，包括参与方ID、暴露的端口和对应的 service 等，Value 的格式为 JSON。 + + + +#### 结构定义 + +proto 链接：https://code.alipay.com/secretflow/kuscia/blob/master/proto/api/v1alpha1/appconfig/app_config.proto + +```protobuf +// Service represents the service address corresponding to the port. +message Service { + // Name of port. + string port_name = 1; + // Endpoint list corresponding to the port. + repeated string endpoints = 2; +} + +// Party represents the basic information of the party. +message Party { + // Name of party. + string name = 1; + // role carried by party. Examples: client, server... + string role = 2; + // List of services exposed by pod. + repeated Service services = 3; +} + +// ClusterDefine represents the information of all parties. +message ClusterDefine { + // Basic information of all parties. + repeated Party parties = 1; + // index of self party. + int32 self_party_idx = 2; + // index of self endpoint. + int32 self_endpoint_idx = 3; +} +``` + + + +#### 示例 + +```json +{ + "parties": [ + { + "name": "alice-test-ab96242c2bc1a091", + "role": "client", + "services": [ + { + "port_name": "global", + "endpoints": [ + "fascia-cbd70io7vda5d471l111-client0-global.alice-test-ab96242c2bc1a091.svc" + ] + }, + { + "port_name": "worker", + "endpoints": [ + "fascia-cbd70io7vda5d471l111-client0-worker-xxx.alice-test-ab96242c2bc1a091.svc" + ] + } + ] + }, + { + "name": "bob-test-bf44540af100cd29", + "role": "client", + "services": [ + { + "port_name": "global", + "endpoints": [ + "fascia-cbd70io7vda5d471l222-client0-global.bob-test-bf44540af100cd29.svc" + ] + }, + { + "port_name": "worker1", + "endpoints": [ + "fascia-cbd70io7vda5d471l222-client0-worker1.bob-test-bf44540af100cd29.svc" + ] + } + ] + }, + { + "name": "fl-one-afa7729dea3ca42c", + "role": "server", + "services": [ + { + "port_name": "object-manager", + "endpoints": [ + "fascia-cbd70io7vda5d471lpp2-server0-object-manager.fl-one-afa7729dea3ca42c.svc" + ] + }, + { + "port_name": "global", + "endpoints": [ + "fascia-cbd70io7vda5d471lpp2-server0-global.fl-one-afa7729dea3ca42c.svc" + ] + }, + { + "port_name": "worker1", + "endpoints": [ + "fascia-cbd70io7vda5d471lpp2-server0-worker1.fl-one-afa7729dea3ca42c.svc" + ] + } + ] + } + ], + "self_party_idx": "0", + "self_endpoint_idx": "0" +} +``` + + + +### ALLOCATED_PORTS + +ALLOCATED_PORTS 定义了 Kuscia 给应用分配的端口号，Value 的格式为 JSON。 + +#### 结构定义 + +proto 链接：https://code.alipay.com/secretflow/kuscia/blob/master/proto/api/v1alpha1/appconfig/app_config.proto + +```protobuf +// Port represents an allocated port for pod. +message Port { + // Each named port in a pod must have a unique name. + string name = 1; + // Number of port allocated for pod. + int32 port = 2; + // Scope of port. Must be Cluster,Domain,Local. + // Defaults to "Local". + // +optional + string scope = 3; + // Protocol for port. Must be HTTP,GRPC. + // Defaults to "HTTP". + // +optional + string protocol = 4; +} + +// AllocatedPorts represents allocated ports for pod. +message AllocatedPorts { + // Allocated ports. + repeated Port ports = 1; +} +``` + + + +#### 示例 + +```json +{ + "ports": [ + { + "name": "worker1", + "port": 16619, + "scope": "Cluster", + "protocol": "GRPC" + }, + { + "name": "global", + "port": 16609, + "scope": "Cluster", + "protocol": "GRPC" + } + ] +} +``` + + + +### TASK_INPUT_CONFIG + +TASK_INPUT_CONFIG 定义了任务基础的参数配置，比如输入文件、输出文件、算子类型等，由应用侧定义具体的内容，Kuscia 目前不感知，只做透传。未来 Kuscia 和应用需要共同定义一套结构标准。 + + + +# 部署模板 + +每个应用镜像需要配置一个应用部署模板，它是一个类 K8s Deployment 的 YAML（待定） 格式的描述。 + + + +### 模板格式 + +参照：https://code.alipay.com/secretflow/kuscia/blob/master/pkg/crd/apis/kuscia/v1alpha1/appimage_types.go + + + +### 示例 + +```yaml +- name: secretflow + # R1 + # role 不局限于某类特定的值，可自定义 + # 取值范围示例: 'client'/'server'/'client,server' + # job 使用的模版匹配规则流程如下: + # 1. 若nuevajob.role [in] role。匹配到，则返回 + # 2. 若role 为空，模版通用。 + # 3. 若nuevajob role 为空，选择第一个模版。 + role: 'client' + replicas: 1 + networkPolicy: + ingresses: + - from: + # 只允许角色为 server 的参与方访问本方的 global 端口。 + roles: + - server + ports: + - port: global + spec: + containers: + - name: secretflow + command: + - /home/admin/bin/secretflow + - --client-config=/home/admin/conf/client-config.conf + - --task-config=/home/admin/conf/task-config.conf + # 引用配置模板 + configVolumeMounts: + - # 挂载到容器内的路径 + mountPath: /home/admin/conf/task-config.conf + # 对应配置模板中的 key + subPath: task-config.conf + # 需要 listen 的端口列表 + ports: + - name: global + port: 10010 + # http/https + # 创建service时，在annotation中加入nueva.secretflow/protocol: 'HTTP' + protocol: HTTP + # Cluster: 创建service，外部或节点内访问 + # Domain: 创建service，节点内部访问 + # Local: 不创建service，仅运用于pod内部localhost访问 + scope: Cluster + - name: worker + port: 10011 + protocol: HTTP + scope: Domain + resources: + limits: + cpu: "2" + memory: 2Gi + requests: + cpu: "2" + memory: 2Gi +``` diff --git a/internaldoc/docs_tutorial.md b/internaldoc/docs_tutorial.md new file mode 100644 index 000000000..8e1389b33 --- /dev/null +++ b/internaldoc/docs_tutorial.md @@ -0,0 +1,62 @@ +## 说明 + +本说明文档仅限内部使用，不会带到开源仓库。 + +本文档旨在介绍如何在本地撰写和生成 Kuscia 开源文档以及查看撰写后的文档效果。 + +## 先决条件 + +安装依赖 + +```bash +cd docs/ +pip install -r requirements.txt -i https://artifacts.antgroup-inc.cn/simple/ +``` + +## 撰写文档 + +### rst文档语法参考 + +* [中文文档](http://www.pythondoc.com/sphinx/contents.html) +* [英文文档](https://www.sphinx-doc.org/en/master/) + +### markdown文档语法参考 + +* [中文文档](https://markdown.com.cn/basic-syntax//) +* [英文文档](https://www.markdownguide.org/basic-syntax/) + +### 撰写 Kuscia 开源文档 + +在以下目录中撰写相应的文档内容 + +* docs/getting_started +* docs/reference +* docs/development + +## 浏览文档效果 + +### 浏览本地文档效果 + +1. 将撰写的文档生成html文件 + +```bash +cd docs/ +make html +``` + +2. 用浏览器打开docs/_build/html/index.html + +### 浏览 Antcode secretflow/kuscia 仓库某个分支文档效果 + +注意：下面IP地址为内部开发机，因此需要关闭浏览器加速代理功能 + +1. 在浏览器中使用地址 http://100.83.15.142:8089/kuscia/分支名称 渲染分支文档 + +2. 步骤1完成文档渲染后，会自动跳转到地址 http://100.83.15.142:8088/html/分支名称 + +其他疑问： +Q1. 若分支代码文档内容有更新，如何查看更新后的文档效果？ +ans: 访问步骤1的地址，会重新根据分支最新的内容渲染文档，看到的内容将是最新的文档内容 + +Q2. 若分支代码文档没有更新，是否每次都需访问步骤1中的地址？ +ans: 不需要，若分支代码没有更新，仅需访问步骤2的地址即可，该地址对应一个静态网页。 diff --git a/internaldoc/quick_start.md b/internaldoc/quick_start.md new file mode 100644 index 000000000..34270aa39 --- /dev/null +++ b/internaldoc/quick_start.md @@ -0,0 +1,135 @@ +## 环境 + +### 机器 + +操作系统：macOS, CentOS7, CentOS8 + +资源：8 Core / 16G Memory / 200G Hard Disk + +### 关于 macOS + +MacOS 默认给单个 Docker container 分配了 2G 内存，请参考[官方文档](https://docs.docker.com/desktop/settings/mac/)将内存上限提高为 6G（Kuscia 2G + SecretFlow 4G) 。 + +### 环境准备 + +Kuscia 的部署需要依赖 Docker 环境，Docker 的安装请参考[官方文档](https://docs.docker.com/engine/install/)。以下为 CentOS 系统安装 Docker 的示例： + +```bash +# 安装 docker +yum install -y yum-utils +yum-config-manager \ + --add-repo \ + https://download.docker.com/linux/centos/docker-ce.repo +yum install -y docker-ce docker-ce-cli containerd.io + +# 启动 docker +systemctl start docker +``` + + + +## 预备工作 + +由于内部测试的应用镜像（例如 secretflow 镜像）存储在私有镜像仓库，因此需要提前配置镜像仓库的地址和帐密，此配置内容不对外网暴露。 + +```bash +mkdir -p ~/kuscia + +vim ~/kuscia/env.list +# 写入以下配置项 +REGISTRY_ENDPOINT=registry.cn-hangzhou.aliyuncs.com/nueva-stack +REGISTRY_USERNAME={{your_username}} +REGISTRY_PASSWORD={{your_password}} + +# 或直接使用命令写入 +echo "REGISTRY_ENDPOINT=registry.cn-hangzhou.aliyuncs.com/nueva-stack +REGISTRY_USERNAME={{your_username}} +REGISTRY_PASSWORD={{your_password}} +" > ~/kuscia/env.list +``` + +说明：如果在部署阶段或任务执行前根据部署提示将 secretflow 镜像准备好，则 REGISTRY_USERNAME 和 REGISTRY_PASSWORD 两个配置项无须配置。 + + + +## 部署体验 + +### 前置操作 + +配置 Kuscia 镜像，以下示例选择使用 latest 版本镜像： + +```bash +# 外网latest镜像 +export KUSCIA_IMAGE=secretflow/kuscia +# 内网latest镜像 +export KUSCIA_IMAGE=reg.docker.alibaba-inc.com/secretflow/kuscia-anolis +# 内部阿里云latest镜像 +export KUSCIA_IMAGE=registry.cn-hangzhou.aliyuncs.com/nueva-stack/kuscia-anolis +``` + +获取 Kuscia 安装脚本，安装脚本会下载到当前目录： + +``` +docker run --rm -v $(pwd):/tmp/kuscia $KUSCIA_IMAGE cp -f /home/kuscia/scripts/deploy/start_standalone.sh /tmp/kuscia +``` + +### 中心化组网模式 + +```bash +# 启动集群，会拉起 3 个 docker 容器，包括一个控制平面 master 和两个 Lite 节点 alice 和 bob。 +./start_standalone.sh center + +# 登入 master 容器 +docker exec -it ${USER}-kuscia-master bash + +# 执行作业 (两方PSI任务) +scripts/user/create_example_job.sh + +# 查看作业状态 +kubectl get kj -n cross-domain +``` + +### 点对点组网模式 + +```bash +# 启动集群，会拉起两个 docker 容器，分别表示 Autonomy 节点 alice 和 bob。 +./start_standalone.sh p2p + +# 登入 alice 节点容器（或 bob 节点容器） +docker exec -it ${USER}-kuscia-autonomy-alice bash + +# 执行作业 (两方PSI任务) +scripts/user/create_example_job.sh + +# 查看作业状态 +kubectl get kj -n cross-domain +``` + + + +## 作业状态 + +如果作业执行成功，则 `kubectl get kj -n cross-domain` 命令会显示类似下方的输出，Succeeded 表示成功状态： + +```bash +NAME STARTTIME COMPLETIONTIME LASTRECONCILETIME PHASE +secretflow-task-20230406162606 50s 50s 50s Succeeded +``` + +同时，在 alice 和 bob 节点容器中能看到 PSI 结果输出文件，以中心化集群模式下的 alice 节点为例： + +```bash +docker exec -it ${USER}-kuscia-lite-alice cat var/storage/data/alice_psi_out.csv +``` + +结果输出： + +```bash +id1,item,feature1 +K200,B,BBB +K200,C,CCC +K300,D,DDD +K400,E,EEE +K400,F,FFF +K500,G,GGG +``` diff --git a/pkg/agent/provider/node/capacity_manager_test.go b/pkg/agent/provider/node/capacity_manager_test.go index cd81970b0..06b489c73 100644 --- a/pkg/agent/provider/node/capacity_manager_test.go +++ b/pkg/agent/provider/node/capacity_manager_test.go @@ -104,7 +104,7 @@ func TestBuildCgroupResource(t *testing.T) { tests := []struct { runtime string reservedResCfg *config.ReservedResourcesCfg - wantCPUAvailable int64 + wantCpuAvailable int64 wantMemAvailable int64 wantCgroupCPUQuota int64 wantCgroupCPUPeriod int64 @@ -127,7 +127,7 @@ func TestBuildCgroupResource(t *testing.T) { err := pa.buildCgroupResource(tt.runtime, tt.reservedResCfg) assert.Nil(t, err) - assert.Equal(t, tt.wantCPUAvailable, pa.cpuAvailable.Value()) + assert.Equal(t, tt.wantCpuAvailable, pa.cpuAvailable.Value()) assert.Equal(t, tt.wantMemAvailable, pa.memAvailable.Value()) assert.Equal(t, tt.wantCgroupCPUQuota, pointerToInt64(pa.cgroupCPUQuota)) assert.Equal(t, tt.wantCgroupCPUPeriod, pointerToUint64(pa.cgroupCPUPeriod)) diff --git a/pkg/agent/provider/pod/kubebackend/sigma.go b/pkg/agent/provider/pod/kubebackend/sigma.go new file mode 100644 index 000000000..ed8fe56c1 --- /dev/null +++ b/pkg/agent/provider/pod/kubebackend/sigma.go @@ -0,0 +1,178 @@ +// OPENSOURCE-CLEANUP DELETE_FILE + +// Copyright 2023 Ant Group Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kubebackend + +import ( + "fmt" + "os" + + "gopkg.in/yaml.v3" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + + "github.com/secretflow/kuscia/pkg/utils/nlog" +) + +const ( + labelSigmaBizName = "meta.k8s.alipay.com/biz-name" + labelSigmaMigrationLevel = "meta.k8s.alipay.com/migration-level" + + labelSigmaAppName = "sigma.ali/app-name" + labelSigmaInstanceGroup = "sigma.ali/instance-group" + labelSigmaSite = "sigma.ali/site" + labelSigmaDeployUnit = "sigma.ali/deploy-unit" + labelSigmaZone = "meta.k8s.alipay.com/zone" + labelEnableDefaultRoute = "ali.EnableDefaultRoute" + + envSigmaAppName = "ALIPAY_APP_APPNAME" + envSigmaZone = "ALIPAY_APP_ZONE" + + annotationSigmaAutoEviction = "pod.k8s.alipay.com/auto-eviction" +) + +type sigmaConfig struct { + AllowOverQuota bool `yaml:"allowOverQuota,omitempty"` +} + +type sigma struct { + config sigmaConfig +} + +func (s *sigma) Init(cfg *yaml.Node) error { + if err := cfg.Decode(&s.config); err != nil { + return err + } + + nlog.Infof("Sigma config: %+v", s.config) + + return nil +} + +func (s *sigma) PreSyncPod(bkPod *v1.Pod) { + if bkPod.Labels == nil { + bkPod.Labels = map[string]string{} + } + + fillLabelIfNotExist(bkPod, labelSigmaAppName, os.Getenv(envSigmaAppName)) + fillLabelIfNotExist(bkPod, labelSigmaZone, os.Getenv(envSigmaZone)) + fillLabelIfNotExist(bkPod, labelSigmaInstanceGroup, fmt.Sprintf("%shost", bkPod.Labels[labelSigmaAppName])) + fillLabelIfNotExist(bkPod, labelSigmaMigrationLevel, "L3") + fillLabelIfNotExist(bkPod, labelEnableDefaultRoute, "true") + + if bkPod.Annotations == nil { + bkPod.Annotations = map[string]string{} + } + + fillAnnotationIfNotExist(bkPod, annotationSigmaAutoEviction, "true") + + bkPod.Spec.Volumes = append(bkPod.Spec.Volumes, v1.Volume{ + Name: "logs", + VolumeSource: v1.VolumeSource{ + EmptyDir: &v1.EmptyDirVolumeSource{}, + }, + }) + + for i := range bkPod.Spec.Containers { + c := &bkPod.Spec.Containers[i] + c.VolumeMounts = append(c.VolumeMounts, v1.VolumeMount{ + Name: "logs", + MountPath: "/home/admin/logs", + }) + + fillResource(c) + } + + if s.config.AllowOverQuota { + if bkPod.Spec.Affinity == nil { + bkPod.Spec.Affinity = &v1.Affinity{} + } + if bkPod.Spec.Affinity.NodeAffinity == nil { + bkPod.Spec.Affinity.NodeAffinity = &v1.NodeAffinity{} + } + if bkPod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution == nil { + bkPod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution = &v1.NodeSelector{} + } + + bkPod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms = append( + bkPod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms, + v1.NodeSelectorTerm{ + MatchExpressions: []v1.NodeSelectorRequirement{ + { + Key: "sigma.ali/is-over-quota", + Operator: v1.NodeSelectorOpIn, + Values: []string{"true"}, + }, + }, + }, + ) + + bkPod.Spec.Tolerations = append(bkPod.Spec.Tolerations, v1.Toleration{ + Key: "sigma.ali/is-over-quota", + Operator: v1.TolerationOpEqual, + Value: "true", + Effect: v1.TaintEffectNoSchedule, + }) + } + + return +} + +func fillResource(c *v1.Container) { + if c.Resources.Limits == nil { + c.Resources.Limits = map[v1.ResourceName]resource.Quantity{} + } + + if _, ok := c.Resources.Limits[v1.ResourceCPU]; !ok { + c.Resources.Limits[v1.ResourceCPU] = *resource.NewQuantity(2, resource.BinarySI) + } + if _, ok := c.Resources.Limits[v1.ResourceMemory]; !ok { + c.Resources.Limits[v1.ResourceMemory] = *resource.NewQuantity(2*1000*1000*1000, resource.BinarySI) + } + if _, ok := c.Resources.Limits[v1.ResourceEphemeralStorage]; !ok { + c.Resources.Limits[v1.ResourceEphemeralStorage] = *resource.NewQuantity(10*1000*1000*1000, resource.BinarySI) + } + + if c.Resources.Requests == nil { + c.Resources.Requests = map[v1.ResourceName]resource.Quantity{} + } + + if _, ok := c.Resources.Requests[v1.ResourceCPU]; !ok { + c.Resources.Requests[v1.ResourceCPU] = *resource.NewQuantity(2, resource.BinarySI) + } + if _, ok := c.Resources.Requests[v1.ResourceMemory]; !ok { + c.Resources.Requests[v1.ResourceMemory] = *resource.NewQuantity(2*1000*1000*1000, resource.BinarySI) + } + if _, ok := c.Resources.Requests[v1.ResourceEphemeralStorage]; !ok { + c.Resources.Requests[v1.ResourceEphemeralStorage] = *resource.NewQuantity(10*1000*1000*1000, resource.BinarySI) + } +} + +func fillLabelIfNotExist(bkPod *v1.Pod, key, value string) { + if _, ok := bkPod.Labels[key]; !ok { + bkPod.Labels[key] = value + } +} + +func fillAnnotationIfNotExist(bkPod *v1.Pod, key, value string) { + if _, ok := bkPod.Annotations[key]; !ok { + bkPod.Annotations[key] = value + } +} + +func init() { + RegisterBackendPlugin("sigma", &sigma{}) +} diff --git a/pkg/agent/provider/pod/kubebackend/sigma_test.go b/pkg/agent/provider/pod/kubebackend/sigma_test.go new file mode 100644 index 000000000..296c2c65e --- /dev/null +++ b/pkg/agent/provider/pod/kubebackend/sigma_test.go @@ -0,0 +1,68 @@ +// OPENSOURCE-CLEANUP DELETE_FILE + +// Copyright 2023 Ant Group Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kubebackend + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "gopkg.in/yaml.v3" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/secretflow/kuscia/pkg/agent/config" +) + +func TestSigma_PreSyncPod(t *testing.T) { + s := &sigma{} + + configYaml := ` +backend: + name: sigma + config: + allowOverQuota: true +` + providerCfg := &config.K8sProviderCfg{} + assert.NoError(t, yaml.Unmarshal([]byte(configYaml), providerCfg)) + + backendConfig := providerCfg.Backend.Config + assert.NoError(t, s.Init(&backendConfig)) + + pod := &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod01", + Namespace: "test-namespace", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "ctr01", + Command: []string{"sleep 60"}, + Image: "aa/bb:001", + }, + }, + }, + } + + s.PreSyncPod(pod) + + assert.Equal(t, "L3", pod.Labels[labelSigmaMigrationLevel]) + assert.Equal(t, "true", pod.Labels[labelEnableDefaultRoute]) + assert.Equal(t, "true", pod.Annotations[annotationSigmaAutoEviction]) + assert.Equal(t, 1, len(pod.Spec.Tolerations)) + assert.True(t, pod.Spec.Affinity != nil) +} diff --git a/pkg/client/.OPENSOURCE-CLEANUP-RMDIR b/pkg/client/.OPENSOURCE-CLEANUP-RMDIR new file mode 100644 index 000000000..e69de29bb diff --git a/pkg/client/kusciastorage/rest/client.go b/pkg/client/kusciastorage/rest/client.go new file mode 100644 index 000000000..b9e801a26 --- /dev/null +++ b/pkg/client/kusciastorage/rest/client.go @@ -0,0 +1,180 @@ +// Copyright 2023 Ant Group Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rest + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "time" +) + +// Client defines a client which is used to access to kuscia storage service. +type Client struct { + config *config + um *urlMaker + httpClient *http.Client +} + +// New is used to create a client instance. +func New(endpoint string, options ...Option) (*Client, error) { + conf := getDefaultConfig() + + for _, option := range options { + option(conf) + } + + um := &urlMaker{ + serverAPIPrefix: conf.serverAPIPrefix, + } + err := um.init(endpoint) + if err != nil { + return nil, err + } + + client := &Client{ + config: conf, + um: um, + httpClient: &http.Client{}, + } + + client.init() + return client, nil +} + +// init is used to initialize client. +func (c *Client) init() { + transport := &http.Transport{ + DialContext: (&net.Dialer{ + Timeout: c.config.connectTimeout, + KeepAlive: c.config.keepAliveTimeout, + }).DialContext, + } + + c.httpClient.Transport = transport + c.httpClient.Timeout = c.config.readWriteTimeout +} + +// CreateResource is used to create resource to kuscia storage service. +func (c *Client) CreateResource( + domain, kind, kindInstName, resourceName string, + normalReqBody *NormalRequestBody, + refReqBody *ReferenceRequestBody) error { + + if (refReqBody == nil && normalReqBody == nil) || (refReqBody != nil && normalReqBody != nil) { + return fmt.Errorf("reference request body or normal request body must exit") + } + + var isRefRequest bool + if refReqBody != nil { + isRefRequest = true + } + + host := c.um.getHost() + url, err := c.um.buildURL(domain, kind, kindInstName, resourceName, isRefRequest) + if err != nil { + return err + } + + var body []byte + if refReqBody != nil { + body, err = json.Marshal(refReqBody) + } else { + body, err = json.Marshal(normalReqBody) + } + if err != nil { + return err + } + + _, err = c.do(host, http.MethodPost, url, body) + return err +} + +// GetResource is used to get resource from kuscia storage service. +func (c *Client) GetResource(domain, kind, kindInstName, resourceName string) (string, error) { + host := c.um.getHost() + url, err := c.um.buildURL(domain, kind, kindInstName, resourceName, false) + if err != nil { + return "", err + } + + resp, err := c.do(host, http.MethodGet, url, nil) + if err != nil { + return "", err + } + + content, ok := resp.Content.(string) + if !ok { + return "", fmt.Errorf("response content can't convert to string type") + } + return content, nil +} + +// DeleteResource is used to delete resource in the kuscia storage service +func (c *Client) DeleteResource(domain, kind, kindInstName, resourceName string) error { + host := c.um.getHost() + url, err := c.um.buildURL(domain, kind, kindInstName, resourceName, false) + if err != nil { + return err + } + + _, err = c.do(host, http.MethodDelete, url, nil) + return err +} + +func (c *Client) do(host, method, url string, body []byte) (*response, error) { + req, err := http.NewRequest(method, url, bytes.NewReader(body)) + if err != nil { + return nil, err + } + req.Host = host + req.Header.Set("Content-Type", "application/json") + + var resp *http.Response + for i := 0; ; i++ { + resp, err = c.httpClient.Do(req) + if err == nil { + break + } + + if i >= c.config.retryTimes { + break + } + time.Sleep(200 * time.Millisecond) + } + if err != nil { + return nil, err + } + defer resp.Body.Close() + + rawRespBody, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + rp := &response{} + err = json.Unmarshal(rawRespBody, rp) + if err != nil { + return nil, err + } + + if rp.Status != nil && rp.Status.Code != statusCodeForSuccess { + return nil, fmt.Errorf("request failed, status code: %v, message: %q", rp.Status.Code, rp.Status.Message) + } + return rp, nil +} diff --git a/pkg/client/kusciastorage/rest/client_test.go b/pkg/client/kusciastorage/rest/client_test.go new file mode 100644 index 000000000..9e06d057d --- /dev/null +++ b/pkg/client/kusciastorage/rest/client_test.go @@ -0,0 +1,249 @@ +// Copyright 2023 Ant Group Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rest + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/secretflow/kuscia/proto/api/v1alpha1" +) + +func mockHTTPServer() *httptest.Server { + return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + status := &v1alpha1.Status{ + Code: 200, + } + + content := "test" + + var resp interface{} + switch r.Method { + case http.MethodGet: + resp = &response{ + Status: status, + Content: content, + } + case http.MethodPost, http.MethodDelete: + resp = &response{ + Status: status, + } + } + + br, _ := json.Marshal(resp) + _, _ = io.WriteString(w, string(br)) + })) +} + +func TestNewClient(t *testing.T) { + testCases := []struct { + name string + endpoint string + expectedErr bool + }{ + { + name: "endpoint is empty", + endpoint: "", + expectedErr: true, + }, + { + name: "endpoint is localhost", + endpoint: "localhost", + expectedErr: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + _, err := New(tc.endpoint) + if tc.expectedErr { + assert.NotNil(t, err) + } + }) + } +} + +func TestCreateResource(t *testing.T) { + svr := mockHTTPServer() + defer svr.Close() + client, _ := New(svr.URL) + + type args struct { + domain string + kind string + kindInstName string + resourceName string + normalReqBody *NormalRequestBody + refReqBody *ReferenceRequestBody + } + + testCases := []struct { + name string + args args + expectedErr bool + }{ + { + name: "request body is empty", + args: args{ + domain: "alice", + kind: "pods", + kindInstName: "test", + resourceName: "config", + }, + expectedErr: true, + }, + { + name: "get url failed", + args: args{ + domain: "alice", + kindInstName: "test", + resourceName: "config", + }, + expectedErr: true, + }, + { + name: "create resource success", + args: args{ + domain: "alice", + kind: "pods", + kindInstName: "test", + resourceName: "config", + normalReqBody: &NormalRequestBody{ + Content: "test", + }, + }, + expectedErr: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + err := client.CreateResource(tc.args.domain, tc.args.kind, tc.args.kindInstName, tc.args.resourceName, tc.args.normalReqBody, tc.args.refReqBody) + if tc.expectedErr { + assert.NotNil(t, err) + } else { + assert.Nil(t, err) + } + }) + } +} + +func TestGetResource(t *testing.T) { + svr := mockHTTPServer() + defer svr.Close() + client, _ := New(svr.URL) + + type args struct { + domain string + kind string + kindInstName string + resourceName string + } + + testCases := []struct { + name string + args args + expected string + expectedErr bool + }{ + { + name: "get url failed", + args: args{ + domain: "alice", + kindInstName: "test", + resourceName: "config", + }, + expectedErr: true, + }, + { + name: "get resource success", + args: args{ + domain: "alice", + kind: "pods", + kindInstName: "test", + resourceName: "config", + }, + expected: "test", + expectedErr: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + get, err := client.GetResource(tc.args.domain, tc.args.kind, tc.args.kindInstName, tc.args.resourceName) + if tc.expectedErr { + assert.NotNil(t, err) + } else { + assert.Equal(t, tc.expected, get) + assert.Nil(t, err) + } + }) + } +} + +func TestDeleteResource(t *testing.T) { + svr := mockHTTPServer() + defer svr.Close() + client, _ := New(svr.URL) + + type args struct { + domain string + kind string + kindInstName string + resourceName string + } + + testCases := []struct { + name string + args args + expectedErr bool + }{ + { + name: "get url failed", + args: args{ + domain: "alice", + kindInstName: "test", + resourceName: "config", + }, + expectedErr: true, + }, + { + name: "delete resource success", + args: args{ + domain: "alice", + kind: "pods", + kindInstName: "test", + resourceName: "config", + }, + expectedErr: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + err := client.DeleteResource(tc.args.domain, tc.args.kind, tc.args.kindInstName, tc.args.resourceName) + if tc.expectedErr { + assert.NotNil(t, err) + } else { + assert.Nil(t, err) + } + }) + } +} diff --git a/pkg/client/kusciastorage/rest/config.go b/pkg/client/kusciastorage/rest/config.go new file mode 100644 index 000000000..8bce137b1 --- /dev/null +++ b/pkg/client/kusciastorage/rest/config.go @@ -0,0 +1,78 @@ +// Copyright 2023 Ant Group Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rest + +import ( + "time" +) + +// Option is an alias of config func. +type Option func(c *config) + +// config defines client configuration. +type config struct { + serverAPIPrefix string + retryTimes int + connectTimeout time.Duration + readWriteTimeout time.Duration + keepAliveTimeout time.Duration +} + +// getDefaultConfig gets the default configurations. +func getDefaultConfig() *config { + c := &config{} + + c.retryTimes = 3 + c.connectTimeout = 30 * time.Second + c.readWriteTimeout = 300 * time.Second + c.keepAliveTimeout = 60 * time.Second + + return c +} + +// SetServerAPIPrefix sets the server api prefix. +func SetServerAPIPrefix(prefix string) Option { + return func(c *config) { + c.serverAPIPrefix = prefix + } +} + +// SetRetryTimes sets the retry times for failure request. +func SetRetryTimes(retryTimes int) Option { + return func(c *config) { + c.retryTimes = retryTimes + } +} + +// SetConnectTimeout sets the client connect timeout. +func SetConnectTimeout(timeout int) Option { + return func(c *config) { + c.connectTimeout = time.Duration(timeout) * time.Second + } +} + +// SetReadWriteTimeout sets the client read and write timeout. +func SetReadWriteTimeout(timeout int) Option { + return func(c *config) { + c.readWriteTimeout = time.Duration(timeout) * time.Second + } +} + +// SetKeepAliveTimeout sets the http client keep alive timeout. +func SetKeepAliveTimeout(timeout int) Option { + return func(c *config) { + c.keepAliveTimeout = time.Duration(timeout) * time.Second + } +} diff --git a/pkg/client/kusciastorage/rest/config_test.go b/pkg/client/kusciastorage/rest/config_test.go new file mode 100644 index 000000000..3ff64ff87 --- /dev/null +++ b/pkg/client/kusciastorage/rest/config_test.go @@ -0,0 +1,142 @@ +// Copyright 2023 Ant Group Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rest + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestSetServerAPIPrefix(t *testing.T) { + c := getDefaultConfig() + + testCases := []struct { + name string + prefix string + expected string + }{ + { + name: "set prefix to kuscia-storage", + prefix: "kuscia-storage", + expected: "kuscia-storage", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + opt := SetServerAPIPrefix(tc.prefix) + opt(c) + assert.Equal(t, tc.expected, c.serverAPIPrefix) + }) + } +} + +func TestSetRetryTimes(t *testing.T) { + c := getDefaultConfig() + + testCases := []struct { + name string + times int + expected int + }{ + { + name: "set retry times to 10", + times: 10, + expected: 10, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + opt := SetRetryTimes(tc.times) + opt(c) + assert.Equal(t, tc.expected, c.retryTimes) + }) + } +} + +func TestSetConnectTimeout(t *testing.T) { + c := getDefaultConfig() + + testCases := []struct { + name string + timeout int + expected time.Duration + }{ + { + name: "set timeout to 10", + timeout: 10, + expected: 10 * time.Second, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + opt := SetConnectTimeout(tc.timeout) + opt(c) + assert.Equal(t, tc.expected, c.connectTimeout) + }) + } +} + +func TestSetReadWriteTimeout(t *testing.T) { + c := getDefaultConfig() + + testCases := []struct { + name string + timeout int + expected time.Duration + }{ + { + name: "set timeout to 10", + timeout: 10, + expected: 10 * time.Second, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + opt := SetReadWriteTimeout(tc.timeout) + opt(c) + assert.Equal(t, tc.expected, c.readWriteTimeout) + }) + } +} + +func TestSetKeepAliveTimeout(t *testing.T) { + c := getDefaultConfig() + + testCases := []struct { + name string + timeout int + expected time.Duration + }{ + { + name: "set timeout to 10", + timeout: 10, + expected: 10 * time.Second, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + opt := SetKeepAliveTimeout(tc.timeout) + opt(c) + assert.Equal(t, tc.expected, c.keepAliveTimeout) + }) + } +} diff --git a/pkg/client/kusciastorage/rest/utils.go b/pkg/client/kusciastorage/rest/utils.go new file mode 100644 index 000000000..4053d0cd2 --- /dev/null +++ b/pkg/client/kusciastorage/rest/utils.go @@ -0,0 +1,136 @@ +// Copyright 2023 Ant Group Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rest + +import ( + "fmt" + "net" + "net/url" + "strings" + + "github.com/secretflow/kuscia/proto/api/v1alpha1" +) + +const ( + httpPrefix = "http://" + httpsPrefix = "https://" +) + +const ( + normalDomainRequestURL = "/api/v1/namespaces/%s/%s/%s/%s" + referenceDomainRequestURL = normalDomainRequestURL + "/reference" + normalClusterRequestURL = "/api/v1/%s/%s/%s" + referenceClusterRequestURL = normalClusterRequestURL + "/reference" +) + +const ( + statusCodeForSuccess = int32(200) +) + +// ReferenceRequestBody defines reference request body. +type ReferenceRequestBody struct { + Domain string `json:"domain,omitempty"` + Kind string `json:"kind"` + KindInstName string `json:"kind_instance_name"` + ResourceName string `json:"resource_name"` +} + +// NormalRequestBody defines normal request body. +type NormalRequestBody struct { + Content string `json:"content"` +} + +// response defines response info of kuscia storage service. +type response struct { + Status *v1alpha1.Status `json:"status"` + Content interface{} `json:"content,omitempty"` +} + +// urlMaker is used to build URL. +type urlMaker struct { + scheme string + netLoc string + host string + serverAPIPrefix string +} + +// init is used to initialize urlMaker. +func (um *urlMaker) init(endpoint string) error { + if endpoint == "" { + return fmt.Errorf("endpoint can't be empty") + } + + if strings.HasPrefix(endpoint, httpPrefix) { + um.scheme = "http" + um.netLoc = endpoint[len(httpPrefix):] + } else if strings.HasPrefix(endpoint, httpsPrefix) { + um.scheme = "https" + um.netLoc = endpoint[len(httpsPrefix):] + } else { + um.scheme = "http" + um.netLoc = endpoint + } + + rawURL := um.scheme + "://" + um.netLoc + parseURL, err := url.Parse(rawURL) + if err != nil { + return err + } + + um.netLoc = parseURL.Host + host, _, err := net.SplitHostPort(um.netLoc) + if err != nil { + host = um.netLoc + if len(host) > 0 && host[0] == '[' && host[len(host)-1] == ']' { + host = host[1 : len(host)-1] + } + } + um.host = host + return nil +} + +// buildURL is used to get request url. +func (um *urlMaker) buildURL(domain, kind, kindInstName, resourceName string, isRefRequest bool) (string, error) { + if kind == "" || kindInstName == "" || resourceName == "" { + return "", fmt.Errorf("kind, kind instance name and resource name can't be empty") + } + + urlPrefix := um.scheme + "://" + um.netLoc + if um.serverAPIPrefix != "" { + urlPrefix = urlPrefix + "/" + um.serverAPIPrefix + } + + apiPath := "" + switch isRefRequest { + case true: + if domain != "" { + apiPath = fmt.Sprintf(referenceDomainRequestURL, domain, kind, kindInstName, resourceName) + } else { + apiPath = fmt.Sprintf(referenceClusterRequestURL, kind, kindInstName, resourceName) + } + default: + if domain != "" { + apiPath = fmt.Sprintf(normalDomainRequestURL, domain, kind, kindInstName, resourceName) + } else { + apiPath = fmt.Sprintf(normalClusterRequestURL, kind, kindInstName, resourceName) + } + } + return urlPrefix + apiPath, nil +} + +// getHost is used to get host. +func (um *urlMaker) getHost() string { + return um.host +} diff --git a/pkg/client/kusciastorage/rest/utils_test.go b/pkg/client/kusciastorage/rest/utils_test.go new file mode 100644 index 000000000..47a3c5722 --- /dev/null +++ b/pkg/client/kusciastorage/rest/utils_test.go @@ -0,0 +1,256 @@ +// Copyright 2023 Ant Group Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package rest + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestURLMakerInit(t *testing.T) { + um := &urlMaker{} + testCases := []struct { + name string + endpoint string + expectedErr bool + }{ + { + name: "empty endpoint", + endpoint: "", + expectedErr: true, + }, + { + name: "http endpoint", + endpoint: "http://localhost:8080", + expectedErr: false, + }, + { + name: "https endpoint", + endpoint: "https://localhost/kuscia-storage", + expectedErr: false, + }, + { + name: "endpoint without scheme", + endpoint: "127.0.0.1", + expectedErr: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + err := um.init(tc.endpoint) + if tc.expectedErr { + assert.NotNil(t, err) + } else { + assert.Nil(t, err) + } + }) + } +} + +func TestURLMakerGetURL(t *testing.T) { + type args struct { + domain string + kind string + kindInstName string + resourceName string + isRefRequest bool + } + + testCases := []struct { + um *urlMaker + name string + args args + apiPath string + expectedErr bool + }{ + { + name: "empty kind", + um: &urlMaker{ + scheme: "http", + netLoc: "localhost", + }, + args: args{ + domain: "alice", + kindInstName: "test", + resourceName: "config", + }, + expectedErr: true, + }, + { + name: "empty kind instance name", + um: &urlMaker{ + scheme: "http", + netLoc: "localhost", + }, + args: args{ + domain: "alice", + kind: "pods", + resourceName: "config", + }, + expectedErr: true, + }, + { + name: "empty resource name", + um: &urlMaker{ + scheme: "http", + netLoc: "localhost", + }, + args: args{ + domain: "alice", + kind: "pods", + kindInstName: "test", + }, + expectedErr: true, + }, + { + name: "url for domain resource", + um: &urlMaker{ + scheme: "http", + netLoc: "localhost", + }, + args: args{ + domain: "alice", + kind: "pods", + kindInstName: "test", + resourceName: "config", + }, + apiPath: normalDomainRequestURL, + expectedErr: false, + }, + { + name: "url for reference domain resource", + um: &urlMaker{ + scheme: "http", + netLoc: "localhost", + }, + args: args{ + domain: "alice", + kind: "pods", + kindInstName: "test", + resourceName: "config", + isRefRequest: true, + }, + apiPath: referenceDomainRequestURL, + expectedErr: false, + }, + { + name: "url for cluster resource", + um: &urlMaker{ + scheme: "http", + netLoc: "localhost", + }, + args: args{ + kind: "appimages", + kindInstName: "test", + resourceName: "config", + }, + apiPath: normalClusterRequestURL, + expectedErr: false, + }, + { + name: "url for reference cluster resource", + um: &urlMaker{ + scheme: "http", + netLoc: "localhost", + }, + args: args{ + kind: "appimages", + kindInstName: "test", + resourceName: "config", + isRefRequest: true, + }, + apiPath: referenceClusterRequestURL, + expectedErr: false, + }, + { + name: "set server api prefix for norma request", + um: &urlMaker{ + scheme: "http", + netLoc: "localhost", + serverAPIPrefix: "kuscia-storage", + }, + args: args{ + kind: "appimages", + kindInstName: "test", + resourceName: "config", + }, + apiPath: normalClusterRequestURL, + expectedErr: false, + }, + { + name: "set server api prefix for reference request", + um: &urlMaker{ + scheme: "http", + netLoc: "localhost", + serverAPIPrefix: "kuscia-storage", + }, + args: args{ + kind: "appimages", + kindInstName: "test", + resourceName: "config", + isRefRequest: true, + }, + apiPath: referenceClusterRequestURL, + expectedErr: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + get, err := tc.um.buildURL(tc.args.domain, tc.args.kind, tc.args.kindInstName, tc.args.resourceName, tc.args.isRefRequest) + urlPrefix := tc.um.scheme + "://" + tc.um.netLoc + if tc.expectedErr { + assert.NotNil(t, err) + } else { + expected := fmt.Sprintf(tc.apiPath, tc.args.domain, tc.args.kind, tc.args.kindInstName, tc.args.resourceName) + if tc.args.domain == "" { + expected = fmt.Sprintf(tc.apiPath, tc.args.kind, tc.args.kindInstName, tc.args.resourceName) + } + + if tc.um.serverAPIPrefix != "" { + expected = "/" + tc.um.serverAPIPrefix + expected + } + expected = urlPrefix + expected + + assert.Nil(t, err) + assert.Equal(t, expected, get) + } + }) + } +} + +func TestURLMakerGetHost(t *testing.T) { + um := &urlMaker{ + host: "localhost", + } + + testCases := []struct { + name string + expected string + }{ + { + name: "host name is localhost", + expected: "localhost", + }, + } + + for _, tc := range testCases { + get := um.getHost() + assert.Equal(t, tc.expected, get) + } +} diff --git a/pkg/confmanager/commands/root.go b/pkg/confmanager/commands/root.go index 848d2c948..069b5ebb6 100644 --- a/pkg/confmanager/commands/root.go +++ b/pkg/confmanager/commands/root.go @@ -26,6 +26,9 @@ import ( // register driver _ "github.com/secretflow/kuscia/pkg/secretbackend/mem" + // OPENSOURCE-CLEANUP REMOVE 2 + // register mist driver + _ "github.com/secretflow/kuscia/pkg/secretbackend/mist" ) func Run(ctx context.Context, conf *config.ConfManagerConfig) error { diff --git a/pkg/controllers/kusciajob/handler/scheduler.go b/pkg/controllers/kusciajob/handler/scheduler.go index ddf52f600..d23c5300b 100644 --- a/pkg/controllers/kusciajob/handler/scheduler.go +++ b/pkg/controllers/kusciajob/handler/scheduler.go @@ -736,6 +736,8 @@ func isInterConnJob(kusciaJob *kusciaapisv1alpha1.KusciaJob) bool { } // kusciaJobHasTaskCycle check whether kusciaJob's tasks has cycles. +// OPENSOURCE-CLEANUP REMOVE 1 +// Topological sorting, Kahn: https://en.wikipedia.org/wiki/Topological_sorting func kusciaJobHasTaskCycle(kusciaJob *kusciaapisv1alpha1.KusciaJob) error { // check dependencies cycle. copyKusciaJob := kusciaJob.DeepCopy() diff --git a/pkg/gateway/utils/transit_test.go b/pkg/gateway/utils/transit_test.go index b733b4ac6..59e510368 100644 --- a/pkg/gateway/utils/transit_test.go +++ b/pkg/gateway/utils/transit_test.go @@ -20,29 +20,10 @@ import ( "github.com/secretflow/kuscia/pkg/crd/apis/kuscia/v1alpha1" ) -type args struct { - transit *v1alpha1.Transit -} - -var ( - argsWithReverseTunnel = args{ - transit: &v1alpha1.Transit{ - TransitMethod: v1alpha1.TransitMethodReverseTunnel, - }, - } - argsWithThirdDomain = args{ - transit: &v1alpha1.Transit{ - TransitMethod: v1alpha1.TransitMethodThirdDomain, - }, - } - argsWithEmpty = args{ - transit: &v1alpha1.Transit{}, - } - argsWithNil = args{nil} -) - func TestIsThirdPartyTransit(t *testing.T) { - + type args struct { + transit *v1alpha1.Transit + } tests := []struct { name string args args @@ -51,22 +32,34 @@ func TestIsThirdPartyTransit(t *testing.T) { // TODO: Add test cases. { name: "case 0", - args: argsWithReverseTunnel, + args: args{ + &v1alpha1.Transit{ + TransitMethod: v1alpha1.TransitMethodReverseTunnel, + }, + }, want: false, }, { name: "case 1", - args: argsWithThirdDomain, + args: args{ + &v1alpha1.Transit{ + TransitMethod: v1alpha1.TransitMethodThirdDomain, + }, + }, want: true, }, { name: "case 2", - args: argsWithNil, + args: args{ + nil, + }, want: false, }, { name: "case 3", - args: argsWithEmpty, + args: args{ + &v1alpha1.Transit{}, + }, want: true, }, } @@ -80,7 +73,9 @@ func TestIsThirdPartyTransit(t *testing.T) { } func TestIsReverseTunnelTransit(t *testing.T) { - + type args struct { + transit *v1alpha1.Transit + } tests := []struct { name string args args @@ -89,22 +84,34 @@ func TestIsReverseTunnelTransit(t *testing.T) { // TODO: Add test cases. { name: "case 0", - args: argsWithReverseTunnel, + args: args{ + &v1alpha1.Transit{ + TransitMethod: v1alpha1.TransitMethodReverseTunnel, + }, + }, want: true, }, { name: "case 1", - args: argsWithThirdDomain, + args: args{ + &v1alpha1.Transit{ + TransitMethod: v1alpha1.TransitMethodThirdDomain, + }, + }, want: false, }, { name: "case 2", - args: argsWithNil, + args: args{ + nil, + }, want: false, }, { name: "case 3", - args: argsWithEmpty, + args: args{ + &v1alpha1.Transit{}, + }, want: false, }, } @@ -118,7 +125,9 @@ func TestIsReverseTunnelTransit(t *testing.T) { } func TestIsTransit(t *testing.T) { - + type args struct { + transit *v1alpha1.Transit + } tests := []struct { name string args args @@ -127,22 +136,34 @@ func TestIsTransit(t *testing.T) { // TODO: Add test cases. { name: "case 0", - args: argsWithReverseTunnel, + args: args{ + &v1alpha1.Transit{ + TransitMethod: v1alpha1.TransitMethodReverseTunnel, + }, + }, want: true, }, { name: "case 1", - args: argsWithThirdDomain, + args: args{ + &v1alpha1.Transit{ + TransitMethod: v1alpha1.TransitMethodThirdDomain, + }, + }, want: true, }, { name: "case 2", - args: argsWithNil, + args: args{ + nil, + }, want: false, }, { name: "case 3", - args: argsWithEmpty, + args: args{ + &v1alpha1.Transit{}, + }, want: false, }, } diff --git a/pkg/kusciastorage/.OPENSOURCE-CLEANUP-RMDIR b/pkg/kusciastorage/.OPENSOURCE-CLEANUP-RMDIR new file mode 100644 index 000000000..e69de29bb diff --git a/pkg/kusciastorage/beans/beans.go b/pkg/kusciastorage/beans/beans.go new file mode 100644 index 000000000..2a78f791c --- /dev/null +++ b/pkg/kusciastorage/beans/beans.go @@ -0,0 +1,45 @@ +// Copyright 2023 Ant Group Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package beans + +import ( + "fmt" + + "github.com/secretflow/kuscia/pkg/kusciastorage/common" + "github.com/secretflow/kuscia/pkg/kusciastorage/domain/service" + "github.com/secretflow/kuscia/pkg/kusciastorage/repository/mysql" + "github.com/secretflow/kuscia/pkg/kusciastorage/server" + "github.com/secretflow/kuscia/pkg/web/framework/engine" +) + +// InjectBeans inject beans which used to init dependent function. +func InjectBeans(engine *engine.Engine) (err error) { + err = engine.UseBeanWithConfig(common.BeanNameForServer, server.NewServer()) + if err != nil { + return fmt.Errorf("inject bean %s failed: %v", common.BeanNameForServer, err.Error()) + } + + err = engine.UseBeanWithConfig(common.BeanNameForMysqlRepository, mysql.NewMysqlRepo()) + if err != nil { + return fmt.Errorf("inject bean %s failed: %v", common.BeanNameForMysqlRepository, err.Error()) + } + + err = engine.UseBean(common.BeanNameForResourceService, nil, service.NewResourceService()) + if err != nil { + return fmt.Errorf("inject bean %s failed: %v", common.BeanNameForResourceService, err.Error()) + } + + return nil +} diff --git a/pkg/kusciastorage/common/common.go b/pkg/kusciastorage/common/common.go new file mode 100644 index 000000000..fbdb29847 --- /dev/null +++ b/pkg/kusciastorage/common/common.go @@ -0,0 +1,50 @@ +// Copyright 2023 Ant Group Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package common + +const ( + BeanNameForServer = "ks-server" + BeanNameForMysqlRepository = "mysql-repository" + BeanNameForResourceService = "resource-service" +) + +// DomainKindList defines supported domain kind list Which used to validate the request kind. +var DomainKindList = []string{ + "dataobjects", + "datasources", + "datatables", + "domainappimages", + "domainnodes", + "pods", + "configmaps", + "endpoints", + "secrets", + "services", + "daemonsets", + "deployments", + "replicasets", + "statefulsets", + "cronjobs", + "jobs", + "leases", +} + +// ClusterKindList defines supported cluster kind list Which used to validate the request kind. +var ClusterKindList = []string{ + "nodes", + "appimages", + "domains", + "kusciatasks", +} diff --git a/pkg/kusciastorage/common/error_code.go b/pkg/kusciastorage/common/error_code.go new file mode 100644 index 000000000..f66215def --- /dev/null +++ b/pkg/kusciastorage/common/error_code.go @@ -0,0 +1,43 @@ +// Copyright 2023 Ant Group Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package common + +const ( + ErrorCodeForSuccess int32 = 200 +) + +const ( + ErrorCodeForServerInternalErr int32 = 71001001 + iota //71001001 + ErrorCodeForRequestIsInvalid //71001002 + ErrorCodeForRefRequestCreate //71001004 + ErrorCodeForNormalRequestCreate //71001005 + ErrorCodeForDeleteResource //71001006 + ErrorCodeForQueryResource //71001007 +) + +var errorCodeToMsg = map[int32]string{ + ErrorCodeForSuccess: "success", + ErrorCodeForServerInternalErr: "server internal error", + ErrorCodeForRequestIsInvalid: "request is invalid", + ErrorCodeForRefRequestCreate: "failed to create resource for reference request", + ErrorCodeForNormalRequestCreate: "failed to create resource for normal request", + ErrorCodeForDeleteResource: "failed to delete resource", + ErrorCodeForQueryResource: "failed to query resource", +} + +// GetMsg is used to get error message. +func GetMsg(code int32) string { + return errorCodeToMsg[code] +} diff --git a/pkg/kusciastorage/controller/common.go b/pkg/kusciastorage/controller/common.go new file mode 100644 index 000000000..e298dba8e --- /dev/null +++ b/pkg/kusciastorage/controller/common.go @@ -0,0 +1,139 @@ +// Copyright 2023 Ant Group Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controller + +import ( + "encoding/json" + "fmt" + "strings" + + "github.com/secretflow/kuscia/pkg/kusciastorage/common" + "github.com/secretflow/kuscia/pkg/web/api" + "github.com/secretflow/kuscia/proto/api/v1alpha1" +) + +const ( + Domain = "domain" + Kind = "kind" + KindInstanceName = "kind_instance_name" + ResourceName = "resource_name" +) + +const ( + errUnsupportedKind = "request kind %v is invalid, supported kind list: %v" + errUnsupportedRefKind = "request reference kind %v is invalid, supported kind list: %v" + errRequestBodyTooLarge = "request body is larger than the limit size: %d" + errEmptyRequestBody = "request body can't be empty" + errInvalidRequestParams = "request params is invalid" +) + +// refRequestBody defines reference request body. +type refRequestBody struct { + Domain string `json:"domain"` + Kind string `json:"kind"` + KindInstanceName string `json:"kind_instance_name"` + ResourceName string `json:"resource_name"` +} + +// String returns reference request body with string format. +func (b *refRequestBody) String() string { + body, _ := json.Marshal(b) + return string(body) +} + +// normalRequestBody defines normal request body. +type normalRequestBody struct { + Content string `json:"content"` +} + +// String returns normal request body with string format. +func (b *normalRequestBody) String() string { + body, _ := json.Marshal(b) + return string(body) +} + +// Response defines response for request. +type Response struct { + Status *v1alpha1.Status `json:"status"` + Content string `json:"content,omitempty"` +} + +// String is used to get response with string format. +func (r *Response) String() string { + resp, _ := json.Marshal(r) + return string(resp) +} + +// buildResponse is used to build response. +func buildResponse(content string, status *v1alpha1.Status) api.ProtoResponse { + resp := &Response{ + Status: status, + Content: content, + } + return &api.AnyStringProto{Content: resp.String()} +} + +// buildStatus is used to build status. +func buildStatus(code int32, msg string) *v1alpha1.Status { + return &v1alpha1.Status{ + Code: code, + Message: msg, + } +} + +// isDomainResourceRequest is used to check whether it's domain resource request. +func isDomainResourceRequest(path string) bool { + return strings.HasPrefix(path, "/api/v1/namespaces") +} + +// isReferenceRequest is used to check whether it's reference request. +func isReferenceRequest(path string) bool { + fields := strings.Split(path, "/") + return fields[len(fields)-1] == "reference" +} + +// validateRequestKind is used to check request kind. +func validateRequestKind(kind, apiPath string) error { + if isDomainResourceRequest(apiPath) { + if ok := validateDomainKind(kind); !ok { + return fmt.Errorf(errUnsupportedKind, kind, common.DomainKindList) + } + } else { + if ok := validateClusterKind(kind); !ok { + return fmt.Errorf(errUnsupportedKind, kind, common.ClusterKindList) + } + } + return nil +} + +// validateDomainKind is used to validate domain kind. +func validateDomainKind(kind string) bool { + for _, k := range common.DomainKindList { + if k == kind { + return true + } + } + return false +} + +// validateClusterKind is used to validate cluster kind. +func validateClusterKind(kind string) bool { + for _, k := range common.ClusterKindList { + if k == kind { + return true + } + } + return false +} diff --git a/pkg/kusciastorage/controller/common_test.go b/pkg/kusciastorage/controller/common_test.go new file mode 100644 index 000000000..c2e91325c --- /dev/null +++ b/pkg/kusciastorage/controller/common_test.go @@ -0,0 +1,208 @@ +// Copyright 2023 Ant Group Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controller + +import ( + "context" + "fmt" + "net/http/httptest" + "strings" + "testing" + + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/assert" + + "github.com/secretflow/kuscia/pkg/kusciastorage/domain/model" + "github.com/secretflow/kuscia/pkg/web/framework" +) + +const ( + validReqForDomainResourcePath = "/api/v1/namespaces/alice-test-9d060f86b01acbb3/pods/secretflow-5655d668fc-g8j8j/test" + invalidReqForDomainResourcePath = "/api/v1/namespaces/alice-test-9d060f86b01acbb3/kusciatasks/secretflow-cdge35jv23fid2snn96g/test" + validRefReqForDomainResourcePath = "/api/v1/namespaces/alice-test-9d060f86b01acbb3/pods/secretflow-5655d668fc-g8j8j/test/reference" + invalidRefReqForDomainResourcePath = "/api/v1/namespaces/alice-test-9d060f86b01acbb3/pods/secretflow-5655d668fc-g8j8j/test/reference-test" + validReqForClusterResourcePath = "/api/v1/kusciatasks/secretflow-cdge35jv23fid2snn96g/task-input-config" + invalidReqForClusterResourcePath = "/api/v1/pods/secretflow-cdge35jv23fid2snn96g/task-input-config" +) + +type mockServiceBean struct { + framework.Bean + framework.Config + + createErr bool + createForRefResource bool + findErr bool + deleteErr bool +} + +func (m *mockServiceBean) GetBeanByName(name string) (framework.Bean, bool) { return m, false } +func (m *mockServiceBean) GetConfigByName(name string) (framework.Config, bool) { + return nil, false + +} +func (m *mockServiceBean) Init(e framework.ConfBeanRegistry) error { return nil } +func (m *mockServiceBean) Start(ctx context.Context, e framework.ConfBeanRegistry) error { return nil } + +func (m *mockServiceBean) Create(resource *model.Resource, data *model.Data) error { + if m.createErr { + return fmt.Errorf("create failed") + } + return nil +} + +func (m *mockServiceBean) CreateForRefResource(domain, kind, name string, resource *model.Resource) error { + if m.createForRefResource { + return fmt.Errorf("create for reference resource failed") + } + return nil +} + +func (m *mockServiceBean) Find(domain, kind, name string) (*model.Data, error) { + if m.findErr { + return nil, fmt.Errorf("find failed") + } + return &model.Data{ + Content: "mock test", + }, nil +} + +func (m *mockServiceBean) Delete(domain, kind, name string) error { + if m.deleteErr { + return fmt.Errorf("delete failed") + } + return nil +} + +func setGinParams(ctx *gin.Context, params map[string]string) { + var ginParams gin.Params + for key, value := range params { + param := gin.Param{ + Key: key, + Value: value, + } + ginParams = append(ginParams, param) + } + ctx.Params = ginParams +} + +func setHTTPRequest(ctx *gin.Context, method, path, body string) { + if body == "" { + ctx.Request = httptest.NewRequest(method, path, nil) + return + } + ctx.Request = httptest.NewRequest(method, path, strings.NewReader(body)) +} + +func TestIsDomainResourceRequest(t *testing.T) { + testCases := []struct { + name string + path string + expected bool + }{ + { + name: "request path is for domain resource", + path: validReqForDomainResourcePath, + expected: true, + }, + { + name: "request path isn't for domain resource", + path: validReqForClusterResourcePath, + expected: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ok := isDomainResourceRequest(tc.path) + assert.Equal(t, tc.expected, ok) + }) + } +} + +func TestIsReferenceRequest(t *testing.T) { + testCases := []struct { + name string + path string + expected bool + }{ + { + name: "invalid reference request", + path: invalidRefReqForDomainResourcePath, + expected: false, + }, + { + name: "reference request", + path: validRefReqForDomainResourcePath, + expected: true, + }, + { + name: "normal request", + path: validReqForDomainResourcePath, + expected: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ok := isReferenceRequest(tc.path) + assert.Equal(t, tc.expected, ok) + }) + } +} + +func TestValidateRequestKind(t *testing.T) { + testCases := []struct { + name string + path string + kind string + expectedErr bool + }{ + { + name: "valid domain resource kind", + path: validReqForDomainResourcePath, + kind: "pods", + expectedErr: false, + }, + { + name: "invalid domain resource kind", + path: invalidReqForDomainResourcePath, + kind: "kusciatasks", + expectedErr: true, + }, + { + name: "valid cluster resource kind", + path: validReqForClusterResourcePath, + kind: "kusciatasks", + expectedErr: false, + }, + { + name: "invalid cluster resource kind", + path: invalidReqForClusterResourcePath, + kind: "pods", + expectedErr: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + err := validateRequestKind(tc.kind, tc.path) + if tc.expectedErr { + assert.NotNil(t, err) + } else { + assert.Nil(t, err) + } + }) + } +} diff --git a/pkg/kusciastorage/controller/resource_create.go b/pkg/kusciastorage/controller/resource_create.go new file mode 100644 index 000000000..cca586348 --- /dev/null +++ b/pkg/kusciastorage/controller/resource_create.go @@ -0,0 +1,171 @@ +// Copyright 2023 Ant Group Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controller + +import ( + "encoding/json" + "fmt" + "reflect" + + "github.com/tidwall/gjson" + + "github.com/secretflow/kuscia/pkg/kusciastorage/common" + "github.com/secretflow/kuscia/pkg/kusciastorage/domain/model" + "github.com/secretflow/kuscia/pkg/kusciastorage/domain/service" + "github.com/secretflow/kuscia/pkg/utils/nlog" + "github.com/secretflow/kuscia/pkg/web/api" + "github.com/secretflow/kuscia/pkg/web/errorcode" +) + +// resourceCreateHandler defines the handler info for creating resource request. +type resourceCreateHandler struct { + resourceService service.IService + + domain string + kind string + kindInstanceName string + resourceName string + + reqBodyMaxSize int + normalRequestBody *normalRequestBody + refRequestBody *refRequestBody +} + +// NewResourceCreateHandler returns a resourceCreateHandler instance. +func NewResourceCreateHandler(reqBodyMaxSize int) api.ProtoHandler { + return &resourceCreateHandler{ + reqBodyMaxSize: reqBodyMaxSize, + } +} + +func (h *resourceCreateHandler) Validate(ctx *api.BizContext, request api.ProtoRequest, errs *errorcode.Errs) { + kind, _ := ctx.Params.Get(Kind) + requestURI := ctx.Request.RequestURI + + err := validateRequestKind(kind, requestURI) + if err != nil { + errs.AppendErr(err) + return + } + + req, ok := request.(*api.AnyStringProto) + if !ok || req == nil { + errs.AppendErr(fmt.Errorf("request is invalid")) + return + } + + if isReferenceRequest(requestURI) { + if len(req.String()) > h.reqBodyMaxSize { + errs.AppendErr(fmt.Errorf(errRequestBodyTooLarge, h.reqBodyMaxSize)) + return + } + + var refReqBody refRequestBody + err = json.Unmarshal([]byte(req.String()), &refReqBody) + if err != nil { + errs.AppendErr(err) + return + } + + if refReqBody.Kind == "" || refReqBody.KindInstanceName == "" || refReqBody.ResourceName == "" { + errs.AppendErr(fmt.Errorf(errInvalidRequestParams)) + return + } + + if refReqBody.Domain != "" { + if ok = validateDomainKind(refReqBody.Kind); !ok { + errs.AppendErr(fmt.Errorf(fmt.Sprintf(errUnsupportedRefKind, kind, common.DomainKindList))) + return + } + } else { + if ok = validateClusterKind(refReqBody.Kind); !ok { + errs.AppendErr(fmt.Errorf(fmt.Sprintf(errUnsupportedRefKind, kind, common.ClusterKindList))) + return + } + } + + h.refRequestBody = &refReqBody + } else { + content := gjson.Get(req.String(), "content").String() + if content == "" { + errs.AppendErr(fmt.Errorf(errEmptyRequestBody)) + return + } + + h.normalRequestBody = &normalRequestBody{ + Content: content, + } + } +} + +// Handle is used to handle request to create resource. +func (h *resourceCreateHandler) Handle(ctx *api.BizContext, request api.ProtoRequest) api.ProtoResponse { + h.domain, _ = ctx.Params.Get(Domain) + h.kind, _ = ctx.Params.Get(Kind) + h.kindInstanceName, _ = ctx.Params.Get(KindInstanceName) + h.resourceName, _ = ctx.Params.Get(ResourceName) + + resourceServiceBean, _ := ctx.ConfBeanRegistry.GetBeanByName(common.BeanNameForResourceService) + h.resourceService, _ = resourceServiceBean.(service.IService) + + switch h.refRequestBody != nil { + case true: + err := h.createForReferenceRequest() + if err != nil { + nlog.Errorf("Create resource failed: %v", err.Error()) + return buildResponse("", buildStatus(common.ErrorCodeForRefRequestCreate, err.Error())) + } + default: + err := h.createForNormalRequest() + if err != nil { + nlog.Errorf("Create resource failed: %v", err.Error()) + return buildResponse("", buildStatus(common.ErrorCodeForNormalRequestCreate, err.Error())) + } + } + + return buildResponse("", buildStatus(common.ErrorCodeForSuccess, common.GetMsg(common.ErrorCodeForSuccess))) +} + +// GetType is used to get request and response type. +func (h *resourceCreateHandler) GetType() (reqType, respType reflect.Type) { + return reflect.TypeOf(api.AnyStringProto{}), reflect.TypeOf(api.AnyStringProto{}) +} + +// createForReferenceRequest is used to create resource for reference request. +func (h *resourceCreateHandler) createForReferenceRequest() error { + resource := &model.Resource{ + Domain: h.domain, + Kind: h.kind, + Name: h.kindInstanceName + "/" + h.resourceName, + } + + return h.resourceService.CreateForRefResource( + h.refRequestBody.Domain, + h.refRequestBody.Kind, + h.refRequestBody.KindInstanceName+"/"+h.refRequestBody.ResourceName, + resource) +} + +// createForNormalRequest is used to create resource for normal request. +func (h *resourceCreateHandler) createForNormalRequest() error { + resource := &model.Resource{ + Domain: h.domain, + Kind: h.kind, + Name: h.kindInstanceName + "/" + h.resourceName, + } + + data := &model.Data{Content: h.normalRequestBody.Content} + return h.resourceService.Create(resource, data) +} diff --git a/pkg/kusciastorage/controller/resource_create_test.go b/pkg/kusciastorage/controller/resource_create_test.go new file mode 100644 index 000000000..0ca5983ce --- /dev/null +++ b/pkg/kusciastorage/controller/resource_create_test.go @@ -0,0 +1,305 @@ +// Copyright 2023 Ant Group Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controller + +import ( + "testing" + + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/assert" + "github.com/tidwall/gjson" + + "github.com/secretflow/kuscia/pkg/kusciastorage/common" + "github.com/secretflow/kuscia/pkg/web/api" + "github.com/secretflow/kuscia/pkg/web/errorcode" +) + +func TestValidateForResourceCreate(t *testing.T) { + h := resourceCreateHandler{reqBodyMaxSize: 1 << 27} + testCases := []struct { + name string + ctx *api.BizContext + params map[string]string + method string + path string + request *api.AnyStringProto + errs *errorcode.Errs + expectedErr bool + }{ + { + name: "request kind is invalid", + ctx: &api.BizContext{ + Context: &gin.Context{}, + }, + params: map[string]string{Kind: "kusciatasks"}, + method: "POST", + path: invalidReqForDomainResourcePath, + request: &api.AnyStringProto{ + Content: "test", + }, + errs: &errorcode.Errs{}, + expectedErr: true, + }, + { + name: "request body is nil", + ctx: &api.BizContext{ + Context: &gin.Context{}, + }, + params: map[string]string{"kind": "pods"}, + method: "POST", + path: validReqForDomainResourcePath, + errs: &errorcode.Errs{}, + expectedErr: true, + }, + { + name: "normal request body format is invalid", + ctx: &api.BizContext{ + Context: &gin.Context{}, + }, + params: map[string]string{Kind: "pods"}, + method: "POST", + path: validReqForDomainResourcePath, + request: &api.AnyStringProto{ + Content: "test", + }, + errs: &errorcode.Errs{}, + expectedErr: true, + }, + { + name: "reference request body format is invalid", + ctx: &api.BizContext{ + Context: &gin.Context{}, + }, + params: map[string]string{Kind: "pods"}, + method: "POST", + path: validRefReqForDomainResourcePath, + request: &api.AnyStringProto{ + Content: "test", + }, + errs: &errorcode.Errs{}, + expectedErr: true, + }, + { + name: "domain kind in reference request body is invalid", + ctx: &api.BizContext{ + Context: &gin.Context{}, + }, + params: map[string]string{Kind: "pods"}, + method: "POST", + path: validRefReqForDomainResourcePath, + request: &api.AnyStringProto{ + Content: (&refRequestBody{ + Domain: "bob-test-b912bea48f6f5630", + Kind: "kusciatasks", + KindInstanceName: "secretflow-cdgeljbv23fid2snn9hg", + ResourceName: "test", + }).String(), + }, + errs: &errorcode.Errs{}, + expectedErr: true, + }, + { + name: "cluster kind in reference request body is invalid", + ctx: &api.BizContext{ + Context: &gin.Context{}, + }, + params: map[string]string{Kind: "pods"}, + method: "POST", + path: validRefReqForDomainResourcePath, + request: &api.AnyStringProto{ + Content: (&refRequestBody{ + Kind: "pods", + KindInstanceName: "secretflow-cdgeljbv23fid2snn9hg", + ResourceName: "test", + }).String(), + }, + errs: &errorcode.Errs{}, + expectedErr: true, + }, + { + name: "reference request is valid", + ctx: &api.BizContext{ + Context: &gin.Context{}, + }, + params: map[string]string{Kind: "pods"}, + method: "POST", + path: validRefReqForDomainResourcePath, + request: &api.AnyStringProto{ + Content: (&refRequestBody{ + Domain: "bob-test-b912bea48f6f5630", + Kind: "pods", + KindInstanceName: "secretflow-cdgeljbv23fid2snn9hg", + ResourceName: "test", + }).String(), + }, + errs: &errorcode.Errs{}, + expectedErr: false, + }, + { + name: "normal request is valid", + ctx: &api.BizContext{ + Context: &gin.Context{}, + }, + params: map[string]string{Kind: "pods"}, + method: "POST", + path: validReqForDomainResourcePath, + request: &api.AnyStringProto{ + Content: (&normalRequestBody{ + Content: "test", + }).String(), + }, + errs: &errorcode.Errs{}, + expectedErr: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + setGinParams(tc.ctx.Context, tc.params) + if tc.request != nil { + setHTTPRequest(tc.ctx.Context, tc.method, tc.path, tc.request.String()) + } else { + setHTTPRequest(tc.ctx.Context, tc.method, tc.path, "") + } + + h.Validate(tc.ctx, tc.request, tc.errs) + if tc.expectedErr { + assert.Equal(t, 1, len(*tc.errs)) + } else { + assert.Equal(t, 0, len(*tc.errs)) + } + }) + } +} + +func TestHandleForResourceCreate(t *testing.T) { + domainResourceParams := map[string]string{ + Domain: "alice-test-9d060f86b01acbb3", + Kind: "pods", + KindInstanceName: "secretflow-5655d668fc-g8j8j", + ResourceName: "test", + } + + testCases := []struct { + name string + ctx *api.BizContext + params map[string]string + h *resourceCreateHandler + request *api.AnyStringProto + expectedErr bool + }{ + { + name: "create resource failed for normal request", + ctx: &api.BizContext{ + ConfBeanRegistry: &mockServiceBean{ + createErr: true, + }, + Context: &gin.Context{}, + }, + params: domainResourceParams, + h: &resourceCreateHandler{ + normalRequestBody: &normalRequestBody{ + Content: "test", + }, + }, + request: &api.AnyStringProto{ + Content: "test", + }, + expectedErr: true, + }, + { + name: "create resource failed for reference request", + ctx: &api.BizContext{ + ConfBeanRegistry: &mockServiceBean{ + createForRefResource: true, + }, + Context: &gin.Context{}, + }, + params: domainResourceParams, + h: &resourceCreateHandler{ + refRequestBody: &refRequestBody{ + Domain: "bob-test-b912bea48f6f5630", + Kind: "kusciatasks", + KindInstanceName: "secretflow-cdgeljbv23fid2snn9hg", + ResourceName: "test", + }, + }, + request: &api.AnyStringProto{ + Content: (&refRequestBody{ + Domain: "bob-test-b912bea48f6f5630", + Kind: "kusciatasks", + KindInstanceName: "secretflow-cdgeljbv23fid2snn9hg", + ResourceName: "test", + }).String(), + }, + expectedErr: true, + }, + { + name: "normal request is successful", + ctx: &api.BizContext{ + ConfBeanRegistry: &mockServiceBean{}, + Context: &gin.Context{}, + }, + params: domainResourceParams, + h: &resourceCreateHandler{ + normalRequestBody: &normalRequestBody{ + Content: "test", + }, + }, + request: &api.AnyStringProto{ + Content: "test", + }, + expectedErr: false, + }, + { + name: "reference request is successful", + ctx: &api.BizContext{ + ConfBeanRegistry: &mockServiceBean{}, + Context: &gin.Context{}, + }, + params: domainResourceParams, + h: &resourceCreateHandler{ + refRequestBody: &refRequestBody{ + Domain: "bob-test-b912bea48f6f5630", + Kind: "kusciatasks", + KindInstanceName: "secretflow-cdgeljbv23fid2snn9hg", + ResourceName: "test", + }, + }, + request: &api.AnyStringProto{ + Content: (&refRequestBody{ + Domain: "bob-test-b912bea48f6f5630", + Kind: "kusciatasks", + KindInstanceName: "secretflow-cdgeljbv23fid2snn9hg", + ResourceName: "test", + }).String(), + }, + expectedErr: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + setGinParams(tc.ctx.Context, tc.params) + resp := tc.h.Handle(tc.ctx, tc.request) + code := gjson.Get(resp.(*api.AnyStringProto).String(), "status.code").Int() + if tc.expectedErr { + assert.NotEqual(t, common.ErrorCodeForSuccess, int32(code)) + } else { + assert.Equal(t, common.ErrorCodeForSuccess, int32(code)) + } + }) + } +} diff --git a/pkg/kusciastorage/controller/resource_delete.go b/pkg/kusciastorage/controller/resource_delete.go new file mode 100644 index 000000000..120ba126a --- /dev/null +++ b/pkg/kusciastorage/controller/resource_delete.go @@ -0,0 +1,68 @@ +// Copyright 2023 Ant Group Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controller + +import ( + "reflect" + + "github.com/secretflow/kuscia/pkg/kusciastorage/common" + "github.com/secretflow/kuscia/pkg/kusciastorage/domain/service" + "github.com/secretflow/kuscia/pkg/utils/nlog" + "github.com/secretflow/kuscia/pkg/web/api" + "github.com/secretflow/kuscia/pkg/web/errorcode" +) + +// resourceDeleteHandler defines the handler info for deleting resource request. +type resourceDeleteHandler struct{} + +// NewResourceDeleteHandler returns a resourceDeleteHandler instance. +func NewResourceDeleteHandler() api.ProtoHandler { + return &resourceDeleteHandler{} +} + +// Validate is used to validate request to delete resource. +func (h *resourceDeleteHandler) Validate(ctx *api.BizContext, request api.ProtoRequest, errs *errorcode.Errs) { + kind, _ := ctx.Params.Get(Kind) + requestURI := ctx.Request.RequestURI + err := validateRequestKind(kind, requestURI) + if err != nil { + errs.AppendErr(err) + } +} + +// Handle is used to handle request to delete resource. +func (h *resourceDeleteHandler) Handle(ctx *api.BizContext, request api.ProtoRequest) api.ProtoResponse { + domain, _ := ctx.Params.Get(Domain) + kind, _ := ctx.Params.Get(Kind) + kindInstanceName, _ := ctx.Params.Get(KindInstanceName) + resourceName, _ := ctx.Params.Get(ResourceName) + resourceNameInDB := kindInstanceName + "/" + resourceName + + resourceServiceBean, _ := ctx.ConfBeanRegistry.GetBeanByName(common.BeanNameForResourceService) + rs, _ := resourceServiceBean.(service.IService) + + err := rs.Delete(domain, kind, resourceNameInDB) + if err != nil { + nlog.Errorf("Delete resource failed: %v", err.Error()) + return buildResponse("", buildStatus(common.ErrorCodeForDeleteResource, err.Error())) + } + + return buildResponse("", buildStatus(common.ErrorCodeForSuccess, common.GetMsg(common.ErrorCodeForSuccess))) +} + +// GetType is used to get request and response type. +func (h *resourceDeleteHandler) GetType() (reqType, respType reflect.Type) { + return reflect.TypeOf(api.AnyStringProto{}), reflect.TypeOf(api.AnyStringProto{}) +} diff --git a/pkg/kusciastorage/controller/resource_delete_test.go b/pkg/kusciastorage/controller/resource_delete_test.go new file mode 100644 index 000000000..3bf56e51f --- /dev/null +++ b/pkg/kusciastorage/controller/resource_delete_test.go @@ -0,0 +1,159 @@ +// Copyright 2023 Ant Group Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controller + +import ( + "testing" + + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/assert" + "github.com/tidwall/gjson" + + "github.com/secretflow/kuscia/pkg/kusciastorage/common" + "github.com/secretflow/kuscia/pkg/web/api" + "github.com/secretflow/kuscia/pkg/web/errorcode" +) + +func TestValidateForResourceDelete(t *testing.T) { + h := NewResourceDeleteHandler() + testCases := []struct { + name string + ctx *api.BizContext + params map[string]string + method string + path string + request *api.AnyStringProto + errs *errorcode.Errs + expectedErr bool + }{ + { + name: "request domain kind is invalid", + ctx: &api.BizContext{ + Context: &gin.Context{}, + }, + params: map[string]string{Kind: "kusciatasks"}, + method: "DELETE", + path: invalidReqForDomainResourcePath, + request: &api.AnyStringProto{}, + errs: &errorcode.Errs{}, + expectedErr: true, + }, + { + name: "request cluster kind is invalid", + ctx: &api.BizContext{ + Context: &gin.Context{}, + }, + params: map[string]string{Kind: "pods"}, + method: "DELETE", + path: invalidReqForClusterResourcePath, + request: &api.AnyStringProto{}, + errs: &errorcode.Errs{}, + expectedErr: true, + }, + { + name: "request domain kind is valid", + ctx: &api.BizContext{ + Context: &gin.Context{}, + }, + params: map[string]string{Kind: "pods"}, + method: "DELETE", + path: validReqForDomainResourcePath, + request: &api.AnyStringProto{}, + errs: &errorcode.Errs{}, + expectedErr: false, + }, + { + name: "request cluster kind is valid", + ctx: &api.BizContext{ + Context: &gin.Context{}, + }, + params: map[string]string{Kind: "kusciatasks"}, + method: "DELETE", + path: validReqForClusterResourcePath, + request: &api.AnyStringProto{}, + errs: &errorcode.Errs{}, + expectedErr: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + setGinParams(tc.ctx.Context, tc.params) + setHTTPRequest(tc.ctx.Context, tc.method, tc.path, "") + h.Validate(tc.ctx, tc.request, tc.errs) + if tc.expectedErr { + assert.Equal(t, 1, len(*tc.errs)) + } else { + assert.Equal(t, 0, len(*tc.errs)) + } + }) + } +} + +func TestHandleForResourceDelete(t *testing.T) { + domainResourceParams := map[string]string{ + Domain: "alice-test-9d060f86b01acbb3", + Kind: "pods", + KindInstanceName: "secretflow-5655d668fc-g8j8j", + ResourceName: "test", + } + + testCases := []struct { + name string + ctx *api.BizContext + params map[string]string + h *resourceDeleteHandler + request *api.AnyStringProto + expectedErr bool + }{ + { + name: "delete resource failed", + ctx: &api.BizContext{ + ConfBeanRegistry: &mockServiceBean{ + deleteErr: true, + }, + Context: &gin.Context{}, + }, + params: domainResourceParams, + h: &resourceDeleteHandler{}, + request: &api.AnyStringProto{}, + expectedErr: true, + }, + { + name: "request is successful", + ctx: &api.BizContext{ + ConfBeanRegistry: &mockServiceBean{}, + Context: &gin.Context{}, + }, + params: domainResourceParams, + h: &resourceDeleteHandler{}, + request: &api.AnyStringProto{}, + expectedErr: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + setGinParams(tc.ctx.Context, tc.params) + resp := tc.h.Handle(tc.ctx, tc.request) + code := gjson.Get(resp.(*api.AnyStringProto).String(), "status.code").Int() + if tc.expectedErr { + assert.NotEqual(t, common.ErrorCodeForSuccess, int32(code)) + } else { + assert.Equal(t, common.ErrorCodeForSuccess, int32(code)) + } + }) + } +} diff --git a/pkg/kusciastorage/controller/resource_query.go b/pkg/kusciastorage/controller/resource_query.go new file mode 100644 index 000000000..18e6c25c1 --- /dev/null +++ b/pkg/kusciastorage/controller/resource_query.go @@ -0,0 +1,68 @@ +// Copyright 2023 Ant Group Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controller + +import ( + "reflect" + + "github.com/secretflow/kuscia/pkg/kusciastorage/common" + "github.com/secretflow/kuscia/pkg/kusciastorage/domain/service" + "github.com/secretflow/kuscia/pkg/utils/nlog" + "github.com/secretflow/kuscia/pkg/web/api" + "github.com/secretflow/kuscia/pkg/web/errorcode" +) + +// resourceQueryHandler defines the handler info for getting resource request. +type resourceQueryHandler struct{} + +// NewResourceQueryHandler returns a resourceQueryHandler instance. +func NewResourceQueryHandler() api.ProtoHandler { + return &resourceQueryHandler{} +} + +// Validate is used to validate request to get resource. +func (h *resourceQueryHandler) Validate(ctx *api.BizContext, request api.ProtoRequest, errs *errorcode.Errs) { + kind, _ := ctx.Params.Get(Kind) + requestURI := ctx.Request.RequestURI + err := validateRequestKind(kind, requestURI) + if err != nil { + errs.AppendErr(err) + } +} + +// Handle is used to handle request to get resource. +func (h *resourceQueryHandler) Handle(ctx *api.BizContext, request api.ProtoRequest) api.ProtoResponse { + domain, _ := ctx.Params.Get(Domain) + kind, _ := ctx.Params.Get(Kind) + kindInstanceName, _ := ctx.Params.Get(KindInstanceName) + resourceName, _ := ctx.Params.Get(ResourceName) + resourceNameInDB := kindInstanceName + "/" + resourceName + + resourceServiceBean, _ := ctx.ConfBeanRegistry.GetBeanByName(common.BeanNameForResourceService) + rs, _ := resourceServiceBean.(service.IService) + + data, err := rs.Find(domain, kind, resourceNameInDB) + if err != nil { + nlog.Errorf("Query resource failed: %v", err.Error()) + return buildResponse("", buildStatus(common.ErrorCodeForQueryResource, err.Error())) + } + + return buildResponse(data.Content, buildStatus(common.ErrorCodeForSuccess, common.GetMsg(common.ErrorCodeForSuccess))) +} + +// GetType is used to get request and response type. +func (h *resourceQueryHandler) GetType() (reqType, respType reflect.Type) { + return reflect.TypeOf(api.AnyStringProto{}), reflect.TypeOf(api.AnyStringProto{}) +} diff --git a/pkg/kusciastorage/controller/resource_query_test.go b/pkg/kusciastorage/controller/resource_query_test.go new file mode 100644 index 000000000..8abcf786e --- /dev/null +++ b/pkg/kusciastorage/controller/resource_query_test.go @@ -0,0 +1,159 @@ +// Copyright 2023 Ant Group Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package controller + +import ( + "testing" + + "github.com/gin-gonic/gin" + "github.com/stretchr/testify/assert" + "github.com/tidwall/gjson" + + "github.com/secretflow/kuscia/pkg/kusciastorage/common" + "github.com/secretflow/kuscia/pkg/web/api" + "github.com/secretflow/kuscia/pkg/web/errorcode" +) + +func TestValidateForResourceQuery(t *testing.T) { + h := NewResourceQueryHandler() + testCases := []struct { + name string + ctx *api.BizContext + params map[string]string + method string + path string + request *api.AnyStringProto + errs *errorcode.Errs + expectedErr bool + }{ + { + name: "request domain kind is invalid", + ctx: &api.BizContext{ + Context: &gin.Context{}, + }, + params: map[string]string{Kind: "kusciatasks"}, + method: "GET", + path: invalidReqForDomainResourcePath, + request: &api.AnyStringProto{}, + errs: &errorcode.Errs{}, + expectedErr: true, + }, + { + name: "request cluster kind is invalid", + ctx: &api.BizContext{ + Context: &gin.Context{}, + }, + params: map[string]string{Kind: "pods"}, + method: "GET", + path: invalidReqForClusterResourcePath, + request: &api.AnyStringProto{}, + errs: &errorcode.Errs{}, + expectedErr: true, + }, + { + name: "request domain kind is valid", + ctx: &api.BizContext{ + Context: &gin.Context{}, + }, + params: map[string]string{Kind: "pods"}, + method: "GET", + path: validReqForDomainResourcePath, + request: &api.AnyStringProto{}, + errs: &errorcode.Errs{}, + expectedErr: false, + }, + { + name: "request cluster kind is valid", + ctx: &api.BizContext{ + Context: &gin.Context{}, + }, + params: map[string]string{Kind: "kusciatasks"}, + method: "GET", + path: validReqForClusterResourcePath, + request: &api.AnyStringProto{}, + errs: &errorcode.Errs{}, + expectedErr: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + setGinParams(tc.ctx.Context, tc.params) + setHTTPRequest(tc.ctx.Context, tc.method, tc.path, "") + h.Validate(tc.ctx, tc.request, tc.errs) + if tc.expectedErr { + assert.Equal(t, 1, len(*tc.errs)) + } else { + assert.Equal(t, 0, len(*tc.errs)) + } + }) + } +} + +func TestHandleForResourceQuery(t *testing.T) { + domainResourceParams := map[string]string{ + Domain: "alice-test-9d060f86b01acbb3", + Kind: "pods", + KindInstanceName: "secretflow-5655d668fc-g8j8j", + ResourceName: "test", + } + + testCases := []struct { + name string + ctx *api.BizContext + params map[string]string + h *resourceQueryHandler + request *api.AnyStringProto + expectedErr bool + }{ + { + name: "find failed", + ctx: &api.BizContext{ + ConfBeanRegistry: &mockServiceBean{ + findErr: true, + }, + Context: &gin.Context{}, + }, + params: domainResourceParams, + h: &resourceQueryHandler{}, + request: &api.AnyStringProto{}, + expectedErr: true, + }, + { + name: "request is successful", + ctx: &api.BizContext{ + ConfBeanRegistry: &mockServiceBean{}, + Context: &gin.Context{}, + }, + params: domainResourceParams, + h: &resourceQueryHandler{}, + request: &api.AnyStringProto{}, + expectedErr: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + setGinParams(tc.ctx.Context, tc.params) + resp := tc.h.Handle(tc.ctx, tc.request) + code := gjson.Get(resp.(*api.AnyStringProto).String(), "status.code").Int() + if tc.expectedErr { + assert.NotEqual(t, common.ErrorCodeForSuccess, int32(code)) + } else { + assert.Equal(t, common.ErrorCodeForSuccess, int32(code)) + } + }) + } +} diff --git a/pkg/kusciastorage/domain/model/model.go b/pkg/kusciastorage/domain/model/model.go new file mode 100644 index 000000000..e561c6099 --- /dev/null +++ b/pkg/kusciastorage/domain/model/model.go @@ -0,0 +1,63 @@ +// Copyright 2023 Ant Group Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package model + +import "time" + +var ( + resourceTableName string + dataTableName string +) + +// Resource defines the resource table schema. +type Resource struct { + ID uint `gorm:"primaryKey"` + CreatedAt time.Time + UpdatedAt time.Time + + Domain string `gorm:"column:domain"` + Kind string `gorm:"column:kind"` + Name string `gorm:"column:name"` + DataID *uint `gorm:"column:data_id"` +} + +// TableName return the resource table name. +func (Resource) TableName() string { + return resourceTableName +} + +// SetResourceTableName is used to set resource table name. +func SetResourceTableName(name string) { + resourceTableName = name +} + +// Data defines the data table schema. +type Data struct { + ID uint `gorm:"primaryKey"` + CreatedAt time.Time + UpdatedAt time.Time + + Content string `gorm:"column:content"` +} + +// TableName return the data table name.ddd +func (Data) TableName() string { + return dataTableName +} + +// SetDataTableName is used to set data table name. +func SetDataTableName(name string) { + dataTableName = name +} diff --git a/pkg/kusciastorage/domain/service/service.go b/pkg/kusciastorage/domain/service/service.go new file mode 100644 index 000000000..5be527d30 --- /dev/null +++ b/pkg/kusciastorage/domain/service/service.go @@ -0,0 +1,87 @@ +// Copyright 2023 Ant Group Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package service + +import ( + "context" + + "github.com/secretflow/kuscia/pkg/kusciastorage/common" + "github.com/secretflow/kuscia/pkg/kusciastorage/domain/model" + "github.com/secretflow/kuscia/pkg/kusciastorage/repository" + "github.com/secretflow/kuscia/pkg/web/framework" +) + +var _ IService = (*resourceService)(nil) + +// IService defines an interface which used to handle resource. +type IService interface { + framework.Bean + + Create(resource *model.Resource, data *model.Data) error + CreateForRefResource(domain, kind, name string, resource *model.Resource) error + Find(domain, kind, name string) (*model.Data, error) + Delete(domain, kind, name string) error +} + +// resourceService defines resource service info which used to handle resource. +type resourceService struct { + mysqlRepo repository.IRepository +} + +// NewResourceService returns a resource service instance. +func NewResourceService() IService { + return &resourceService{} +} + +// Init is used to initialize resource service. +func (s *resourceService) Init(registry framework.ConfBeanRegistry) error { + mysqlRepo, _ := registry.GetBeanByName(common.BeanNameForMysqlRepository) + s.mysqlRepo, _ = mysqlRepo.(repository.IRepository) + return nil +} + +// Start is used to implement the Start method of framework.Bean interface. +func (s *resourceService) Start(ctx context.Context, e framework.ConfBeanRegistry) error { + return nil +} + +// Create is used to create resource and data. +func (s *resourceService) Create(resource *model.Resource, data *model.Data) error { + if err := s.mysqlRepo.Create(resource, data); err != nil { + return err + } + return nil +} + +// CreateForRefResource is used to create reference resource. +func (s *resourceService) CreateForRefResource(domain, kind, name string, resource *model.Resource) error { + if err := s.mysqlRepo.CreateForRefResource(domain, kind, name, resource); err != nil { + return err + } + return nil +} + +// Find is used to find data. +func (s *resourceService) Find(domain, kind, name string) (*model.Data, error) { + return s.mysqlRepo.Find(domain, kind, name) +} + +// Delete is used to delete resource and data. +func (s *resourceService) Delete(domain, kind, name string) error { + if err := s.mysqlRepo.Delete(domain, kind, name); err != nil { + return err + } + return nil +} diff --git a/pkg/kusciastorage/domain/service/service_test.go b/pkg/kusciastorage/domain/service/service_test.go new file mode 100644 index 000000000..b05f5d52c --- /dev/null +++ b/pkg/kusciastorage/domain/service/service_test.go @@ -0,0 +1,271 @@ +// Copyright 2023 Ant Group Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package service + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/secretflow/kuscia/pkg/kusciastorage/domain/model" + "github.com/secretflow/kuscia/pkg/web/framework" +) + +type mockRegistry struct { + framework.Bean + framework.Config + + createErr bool + createForRefResourceErr bool + findErr bool + deleteErr bool +} + +func (m mockRegistry) Init(e framework.ConfBeanRegistry) error { return nil } +func (m mockRegistry) Start(ctx context.Context, e framework.ConfBeanRegistry) error { return nil } +func (m mockRegistry) GetBeanByName(name string) (framework.Bean, bool) { return m, true } +func (m mockRegistry) GetConfigByName(name string) (framework.Config, bool) { return nil, false } + +func (m mockRegistry) Create(resource *model.Resource, data *model.Data) error { + if m.createErr { + return fmt.Errorf("create failed") + } + return nil +} + +func (m mockRegistry) CreateForRefResource(domain, kind, name string, resource *model.Resource) error { + if m.createForRefResourceErr { + return fmt.Errorf("create for referece resource failed") + } + return nil +} + +func (m mockRegistry) Find(domain, kind, name string) (*model.Data, error) { + if m.findErr { + return nil, fmt.Errorf("find failed") + } + return &model.Data{Content: "test"}, nil +} + +func (m mockRegistry) Delete(domain, kind, name string) error { + if m.deleteErr { + return fmt.Errorf("delete failed") + } + return nil +} + +func TestResourceServiceInit(t *testing.T) { + testCases := []struct { + name string + rs IService + registry framework.ConfBeanRegistry + expectedErr bool + }{ + { + name: "init success", + rs: NewResourceService(), + registry: mockRegistry{}, + expectedErr: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + err := tc.rs.Init(tc.registry) + t.Logf("err: %v", err) + if tc.expectedErr { + assert.NotNil(t, err) + } else { + assert.Nil(t, err) + } + }) + } +} + +func TestResourceServiceCreate(t *testing.T) { + id := uint(1) + resource := &model.Resource{ + Domain: "alice-test-9d060f86b01acbb3", + Kind: "pods", + Name: "secretflow-5655d668fc-g8j8j/test", + DataID: &id, + } + + data := &model.Data{ID: id, Content: "test"} + + testCases := []struct { + name string + rs IService + registry framework.ConfBeanRegistry + resource *model.Resource + data *model.Data + expectedErr bool + }{ + { + name: "create failed", + rs: NewResourceService(), + registry: mockRegistry{ + createErr: true, + }, + expectedErr: true, + }, + { + name: "create success", + rs: NewResourceService(), + registry: mockRegistry{}, + expectedErr: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + _ = tc.rs.Init(tc.registry) + err := tc.rs.Create(resource, data) + if tc.expectedErr { + assert.NotNil(t, err) + } else { + assert.Nil(t, err) + } + }) + } +} + +func TestResourceServiceCreateForRefResource(t *testing.T) { + var ( + domain = "alice-test-9d060f86b01acbb3" + kind = "pods" + name = "secretflow-5655d668fc-g8j8j/test" + ) + + testCases := []struct { + name string + rs IService + registry framework.ConfBeanRegistry + expectedErr bool + }{ + { + name: "create failed", + rs: NewResourceService(), + registry: mockRegistry{ + createForRefResourceErr: true, + }, + expectedErr: true, + }, + { + name: "create success", + rs: NewResourceService(), + registry: mockRegistry{}, + expectedErr: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + _ = tc.rs.Init(tc.registry) + err := tc.rs.CreateForRefResource(domain, kind, name, nil) + if tc.expectedErr { + assert.NotNil(t, err) + } else { + assert.Nil(t, err) + } + }) + } +} + +func TestResourceServiceFind(t *testing.T) { + var ( + domain = "alice-test-9d060f86b01acbb3" + kind = "pods" + name = "secretflow-5655d668fc-g8j8j/test" + ) + + testCases := []struct { + name string + rs IService + registry framework.ConfBeanRegistry + expectedErr bool + }{ + { + name: "find failed", + rs: NewResourceService(), + registry: mockRegistry{ + findErr: true, + }, + expectedErr: true, + }, + { + name: "find success", + rs: NewResourceService(), + registry: mockRegistry{}, + expectedErr: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + _ = tc.rs.Init(tc.registry) + _, err := tc.rs.Find(domain, kind, name) + if tc.expectedErr { + assert.NotNil(t, err) + } else { + assert.Nil(t, err) + } + }) + } +} + +func TestResourceServiceDelete(t *testing.T) { + var ( + domain = "alice-test-9d060f86b01acbb3" + kind = "pods" + name = "secretflow-5655d668fc-g8j8j/test" + ) + + testCases := []struct { + name string + rs IService + registry framework.ConfBeanRegistry + expectedErr bool + }{ + { + name: "find failed", + rs: NewResourceService(), + registry: mockRegistry{ + deleteErr: true, + }, + expectedErr: true, + }, + { + name: "find success", + rs: NewResourceService(), + registry: mockRegistry{}, + expectedErr: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + _ = tc.rs.Init(tc.registry) + err := tc.rs.Delete(domain, kind, name) + if tc.expectedErr { + assert.NotNil(t, err) + } else { + assert.Nil(t, err) + } + }) + } +} diff --git a/pkg/kusciastorage/kuscia_storage.go b/pkg/kusciastorage/kuscia_storage.go new file mode 100644 index 000000000..753feb39d --- /dev/null +++ b/pkg/kusciastorage/kuscia_storage.go @@ -0,0 +1,39 @@ +// Copyright 2023 Ant Group Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package kusciastorage + +import ( + "github.com/secretflow/kuscia/pkg/kusciastorage/beans" + "github.com/secretflow/kuscia/pkg/utils/meta" + "github.com/secretflow/kuscia/pkg/utils/signals" + "github.com/secretflow/kuscia/pkg/web/framework" + "github.com/secretflow/kuscia/pkg/web/framework/engine" +) + +// RunKusciaStorage runs the kuscia storage service. +func RunKusciaStorage() error { + appEngine := engine.New(&framework.AppConfig{ + Name: "kuscia-storage", + Usage: "store large resource under k8s cluster", + Version: meta.KusciaVersionString(), + }) + + err := beans.InjectBeans(appEngine) + if err != nil { + return err + } + + return appEngine.RunCommand(signals.NewKusciaContextWithStopCh(signals.SetupSignalHandler())) +} diff --git a/pkg/kusciastorage/repository/mysql/mysql_repo.go b/pkg/kusciastorage/repository/mysql/mysql_repo.go new file mode 100644 index 000000000..5d1dee731 --- /dev/null +++ b/pkg/kusciastorage/repository/mysql/mysql_repo.go @@ -0,0 +1,179 @@ +// Copyright 2023 Ant Group Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mysql + +import ( + "context" + "fmt" + + "gorm.io/driver/mysql" + "gorm.io/gorm" + + "github.com/secretflow/kuscia/pkg/kusciastorage/domain/model" + "github.com/secretflow/kuscia/pkg/kusciastorage/repository" + "github.com/secretflow/kuscia/pkg/utils/nlog" + "github.com/secretflow/kuscia/pkg/web/errorcode" + "github.com/secretflow/kuscia/pkg/web/framework" + "github.com/secretflow/kuscia/pkg/web/framework/config" +) + +var ( + resourceTableScheme = `CREATE TABLE IF NOT EXISTS %s + ( + id BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, + name VARCHAR(256) NOT NULL, + kind VARCHAR(256) NOT NULL, + domain VARCHAR(256), + data_id BIGINT(20) NOT NULL, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (id), + INDEX idx_data_id (data_id), + UNIQUE INDEX uk_name_kind_domain (name, kind, domain) + );` + dataTableScheme = `CREATE TABLE IF NOT EXISTS %s + ( + id BIGINT(20) UNSIGNED NOT NULL AUTO_INCREMENT, + content longtext NOT NULL, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (id) + );` +) + +var _ repository.IRepository = (*mysqlRepo)(nil) + +// mysqlRepo defines a client which used to access to mysql database. +type mysqlRepo struct { + config.FlagEnvConfigLoader + + Name string `name:"db-name" usage:"db name which used to access"` + Host string `name:"db-host" usage:"db host which used to access"` + Port int `name:"db-port" usage:"db port which used to access" default:"3306"` + User string `name:"db-user" usage:"db user which used to access"` + Password string `name:"db-password" usage:"db password which used to access"` + ResourceTableName string `name:"db-resource-table-name" usage:"resource table name" default:"resource"` + DataTableName string `name:"db-data-table-name" usage:"data table name" default:"data"` + + Client *gorm.DB +} + +// NewMysqlRepo returns a mysql repo instance. +func NewMysqlRepo() repository.IRepository { + return &mysqlRepo{} +} + +// Validate is used to validate mysql repo config. +func (r *mysqlRepo) Validate(errs *errorcode.Errs) { + if r.Port <= 0 { + errs.AppendErr(fmt.Errorf("server public port %v is illegal", r.Port)) + } +} + +// Init is used to initialize mysql repo. +func (r *mysqlRepo) Init(registry framework.ConfBeanRegistry) error { + var err error + dsn := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?charset=utf8mb4&parseTime=True&loc=Local", + r.User, r.Password, r.Host, r.Port, r.Name) + r.Client, err = gorm.Open(mysql.Open(dsn), &gorm.Config{}) + if err != nil { + return err + } + + // set table name + model.SetResourceTableName(r.ResourceTableName) + model.SetDataTableName(r.DataTableName) + + // create table + err = r.Client.Exec(fmt.Sprintf(resourceTableScheme, r.ResourceTableName)).Error + if err != nil { + nlog.Warnf("Create table %s failed: %v", r.ResourceTableName, err.Error()) + } + + err = r.Client.Exec(fmt.Sprintf(dataTableScheme, r.DataTableName)).Error + if err != nil { + nlog.Warnf("Create table %s failed: %v", r.DataTableName, err.Error()) + } + + return nil +} + +// Start is used to implement the Start method of framework.Bean interface. +func (r *mysqlRepo) Start(ctx context.Context, e framework.ConfBeanRegistry) error { return nil } + +// Create is used to create resource and data. +func (r *mysqlRepo) Create(resource *model.Resource, data *model.Data) error { + return r.Client.Transaction(func(tx *gorm.DB) error { + if err := tx.Create(data).Error; err != nil { + return err + } + + resource.DataID = &data.ID + if err := tx.Create(resource).Error; err != nil { + return err + } + + return nil + }) +} + +// CreateForRefResource is used to create reference resource for exist resource. +func (r *mysqlRepo) CreateForRefResource(domain, kind, name string, resource *model.Resource) error { + return r.Client.Transaction(func(tx *gorm.DB) error { + rs := &model.Resource{} + if err := tx.Where("name = ? AND kind = ? AND domain = ?", name, kind, domain).First(rs).Error; err != nil { + return err + } + + resource.DataID = rs.DataID + return tx.Create(resource).Error + }) +} + +// Find is used to find resource data. +func (r *mysqlRepo) Find(domain, kind, name string) (*model.Data, error) { + data := &model.Data{} + resource := &model.Resource{} + sql := fmt.Sprintf("SELECT * FROM %s WHERE id = (SELECT data_id FROM %s WHERE name = ? AND kind = ? AND domain = ?)", + data.TableName(), resource.TableName()) + err := r.Client.Raw(sql, name, kind, domain).First(data).Error + return data, err +} + +// Delete is used to delete resource and data. +func (r *mysqlRepo) Delete(domain, kind, name string) error { + return r.Client.Transaction(func(tx *gorm.DB) error { + rs := &model.Resource{} + dataTblName := (&model.Data{}).TableName() + + queryIDSQL := fmt.Sprintf("SELECT data_id FROM %s WHERE name = ? AND kind = ? AND domain = ?", rs.TableName()) + if err := r.Client.Raw(queryIDSQL, name, kind, domain).First(rs).Error; err != nil { + return err + } + + dataID := rs.DataID + delResourceSQL := fmt.Sprintf("DELETE FROM %s WHERE name = ? AND kind = ? AND domain = ?", rs.TableName()) + if err := tx.Exec(delResourceSQL, name, kind, domain).Error; err != nil { + return err + } + + delDataSQL := "DELETE FROM %s WHERE id = ? and NOT EXISTS (SELECT 1 FROM %s WHERE data_id = ?)" + delDataSQL = fmt.Sprintf(delDataSQL, dataTblName, rs.TableName()) + if err := tx.Exec(delDataSQL, dataID, dataID).Error; err != nil { + return err + } + return nil + }) +} diff --git a/pkg/kusciastorage/repository/repository.go b/pkg/kusciastorage/repository/repository.go new file mode 100644 index 000000000..258bd1e8b --- /dev/null +++ b/pkg/kusciastorage/repository/repository.go @@ -0,0 +1,31 @@ +// Copyright 2023 Ant Group Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package repository + +import ( + "github.com/secretflow/kuscia/pkg/kusciastorage/domain/model" + "github.com/secretflow/kuscia/pkg/web/framework" +) + +// IRepository defines an interface which used to handle repository resource. +type IRepository interface { + framework.Config + framework.Bean + + Create(resource *model.Resource, data *model.Data) error + CreateForRefResource(domain, kind, name string, resource *model.Resource) error + Find(domain, kind, name string) (*model.Data, error) + Delete(domain, kind, name string) error +} diff --git a/pkg/kusciastorage/server/routers.go b/pkg/kusciastorage/server/routers.go new file mode 100644 index 000000000..851645082 --- /dev/null +++ b/pkg/kusciastorage/server/routers.go @@ -0,0 +1,103 @@ +// Copyright 2023 Ant Group Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "net/http" + + "github.com/gin-gonic/gin" + + "github.com/secretflow/kuscia/pkg/kusciastorage/common" + "github.com/secretflow/kuscia/pkg/kusciastorage/controller" + "github.com/secretflow/kuscia/pkg/web/api" + "github.com/secretflow/kuscia/pkg/web/decorator" + "github.com/secretflow/kuscia/pkg/web/framework/engine" + "github.com/secretflow/kuscia/pkg/web/framework/router" +) + +const ( + domainResourceAPIPath = "/namespaces/:domain/:kind/:kind_instance_name/:resource_name" + clusterResourceAPIPath = "/:kind/:kind_instance_name/:resource_name" + domainResourceRefAPIPath = "/namespaces/:domain/:kind/:kind_instance_name/:resource_name/reference" + clusterResourceRefAPIPath = "/:kind/:kind_instance_name/:resource_name/reference" +) + +// GetPublicRouterGroups returns public router groups. +func GetPublicRouterGroups(s *Server, engine *engine.Engine) router.GroupsRouters { + return []*router.GroupRouters{ + { + Group: "/api/v1", + GroupMiddleware: nil, + Routes: []*router.Router{ + { + HTTPMethod: http.MethodPost, + RelativePath: domainResourceAPIPath, + Handlers: []gin.HandlerFunc{protoDecorator(engine, controller.NewResourceCreateHandler(s.ReqBodyMaxSize))}, + }, + { + HTTPMethod: http.MethodGet, + RelativePath: domainResourceAPIPath, + Handlers: []gin.HandlerFunc{protoDecorator(engine, controller.NewResourceQueryHandler())}, + }, + { + HTTPMethod: http.MethodDelete, + RelativePath: domainResourceAPIPath, + Handlers: []gin.HandlerFunc{protoDecorator(engine, controller.NewResourceDeleteHandler())}, + }, + { + HTTPMethod: http.MethodPost, + RelativePath: clusterResourceAPIPath, + Handlers: []gin.HandlerFunc{protoDecorator(engine, controller.NewResourceCreateHandler(s.ReqBodyMaxSize))}, + }, + { + HTTPMethod: http.MethodGet, + RelativePath: clusterResourceAPIPath, + Handlers: []gin.HandlerFunc{protoDecorator(engine, controller.NewResourceQueryHandler())}, + }, + { + HTTPMethod: http.MethodDelete, + RelativePath: clusterResourceAPIPath, + Handlers: []gin.HandlerFunc{protoDecorator(engine, controller.NewResourceDeleteHandler())}, + }, + }, + }, + } +} + +// GetInternalRouterGroups returns internal router groups. +func GetInternalRouterGroups(s *Server, engine *engine.Engine) router.GroupsRouters { + routerGroups := []*router.GroupRouters{{ + Group: "/api/v1", + GroupMiddleware: nil, + Routes: []*router.Router{ + { + HTTPMethod: http.MethodPost, + RelativePath: domainResourceRefAPIPath, + Handlers: []gin.HandlerFunc{protoDecorator(engine, controller.NewResourceCreateHandler(s.ReqBodyMaxSize))}, + }, + { + HTTPMethod: http.MethodPost, + RelativePath: clusterResourceRefAPIPath, + Handlers: []gin.HandlerFunc{protoDecorator(engine, controller.NewResourceCreateHandler(s.ReqBodyMaxSize))}, + }, + }}} + + return append(routerGroups, GetPublicRouterGroups(s, engine)...) +} + +// protoDecorator is used to wrap handler. +func protoDecorator(engine *engine.Engine, handler api.ProtoHandler) gin.HandlerFunc { + return decorator.DefaultProtoDecoratorMaker(common.ErrorCodeForRequestIsInvalid, common.ErrorCodeForServerInternalErr)(engine, handler) +} diff --git a/pkg/kusciastorage/server/server.go b/pkg/kusciastorage/server/server.go new file mode 100644 index 000000000..8da2289c8 --- /dev/null +++ b/pkg/kusciastorage/server/server.go @@ -0,0 +1,188 @@ +// Copyright 2023 Ant Group Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package server + +import ( + "context" + "fmt" + "net/http" + "time" + + "github.com/gin-gonic/gin" + "github.com/prometheus/client_golang/prometheus/promhttp" + + "github.com/secretflow/kuscia/pkg/utils/nlog" + "github.com/secretflow/kuscia/pkg/utils/nlog/zlogwriter" + "github.com/secretflow/kuscia/pkg/web/errorcode" + "github.com/secretflow/kuscia/pkg/web/framework" + "github.com/secretflow/kuscia/pkg/web/framework/beans" + "github.com/secretflow/kuscia/pkg/web/framework/config" + "github.com/secretflow/kuscia/pkg/web/framework/engine" + "github.com/secretflow/kuscia/pkg/web/framework/router" + "github.com/secretflow/kuscia/pkg/web/logs" +) + +// Server defines the server info which provide service. +type Server struct { + config.FlagEnvConfigLoader + + Debug bool `name:"debug" usage:"server debug mode"` + + PublicPort int `name:"public-port" usage:"public server port" default:"8080"` + InternalPort int `name:"internal-port" usage:"internal server port" default:"8090"` + + ReadTimeout int `name:"read-timeout" usage:"server read timeout in seconds" default:"300"` + WriteTimeout int `name:"write-timeout" usage:"server write timeout in seconds" default:"300"` + IdleTimeout int `name:"idle-timeout" usage:"server idle timeout in seconds" default:"60"` + + LogLevel string `name:"log-level" usage:"log level. supported list[debug,info,warn,error,fatal]" default:"info"` + LogPath string `name:"log-path" usage:"log path" default:"./logs/kuscia-storage.log"` + LogMaxFileSize int `name:"log-max-file-size" usage:"the max size of log file in MB, " default:"100"` + LogMaxFiles int `name:"log-max-files" usage:"the max number of log files" default:"10"` + + ReqBodyMaxSize int `name:"request-body-max-size" usage:"the max size in bytes allowed for request body" default:"134217728"` + + PublicEngine *gin.Engine + InternalEngine *gin.Engine +} + +// NewServer returns a server instance. +func NewServer() *Server { + return &Server{} +} + +// Validate is used to validate server config. +func (s *Server) Validate(errs *errorcode.Errs) { + if s.PublicPort <= 0 { + errs.AppendErr(fmt.Errorf("server public port %v is illegal", s.PublicPort)) + } + + if s.InternalPort <= 0 { + errs.AppendErr(fmt.Errorf("server internal port %v is illegal", s.InternalPort)) + } + + if s.ReadTimeout < 0 { + errs.AppendErr(fmt.Errorf("server read timeout %v is illegal", s.ReadTimeout)) + } + + if s.WriteTimeout < 0 { + errs.AppendErr(fmt.Errorf("server write timeout %v is illegal", s.WriteTimeout)) + } + + if s.IdleTimeout < 0 { + errs.AppendErr(fmt.Errorf("server idle timeout %v is illegal", s.IdleTimeout)) + } + + if s.ReqBodyMaxSize <= 0 { + errs.AppendErr(fmt.Errorf("server request body max size %v is illegal", s.ReqBodyMaxSize)) + } +} + +// RegisterGroup is used to register group for server. +func RegisterGroup(engine *gin.Engine, groupRouters *router.GroupRouters) { + if groupRouters != nil { + group := engine.Group(groupRouters.Group, groupRouters.GroupMiddleware...) + for _, route := range groupRouters.Routes { + group.Handle(route.HTTPMethod, route.RelativePath, route.Handlers...) + } + } +} + +// Init is used to init server. +func (s *Server) Init(e framework.ConfBeanRegistry) error { + w, err := zlogwriter.New(&nlog.LogConfig{ + LogLevel: s.LogLevel, + MaxFileSizeMB: s.LogMaxFileSize, + LogPath: s.LogPath, + MaxFiles: s.LogMaxFiles, + Compress: true, + }) + if err != nil { + return err + } + + logs.Setup(nlog.SetWriter(w)) + nlog.Setup(nlog.SetWriter(w), nlog.SetFormatter(nlog.NewGinLogFormatter())) + + if s.Debug { + gin.SetMode(gin.DebugMode) + } else { + gin.SetMode(gin.ReleaseMode) + } + + eg, ok := e.(*engine.Engine) + if !ok { + return fmt.Errorf("unable convert e to engine.Engine") + } + + pubGinEngine := gin.New() + pubGinEngine.Use(gin.Recovery()) + pubGinEngine.Use(beans.GinLogger(nlog.DefaultLogger(), "public-server")) + s.PublicEngine = pubGinEngine + for _, groupRouters := range GetPublicRouterGroups(s, eg) { + RegisterGroup(s.PublicEngine, groupRouters) + } + + innerGinEngine := gin.New() + innerGinEngine.Use(gin.Recovery()) + innerGinEngine.Use(beans.GinLogger(nlog.DefaultLogger(), "internal-server")) + s.InternalEngine = innerGinEngine + + for _, groupRouters := range GetInternalRouterGroups(s, eg) { + RegisterGroup(s.InternalEngine, groupRouters) + } + return nil +} + +// Start is used to start server. +func (s *Server) Start(ctx context.Context, e framework.ConfBeanRegistry) error { + errChan := make(chan error) + // start public server + go func() { + mux := http.NewServeMux() + mux.Handle("/metrics", promhttp.Handler()) + mux.Handle("/", s.PublicEngine) + sr := &http.Server{ + Addr: fmt.Sprintf(":%d", s.PublicPort), + Handler: mux, + ReadTimeout: time.Duration(s.ReadTimeout) * time.Second, + WriteTimeout: time.Duration(s.WriteTimeout) * time.Second, + IdleTimeout: time.Duration(s.IdleTimeout) * time.Second, + MaxHeaderBytes: 1 << 20, + } + + nlog.Infof("server listen to public address 0.0.0.0:%d", s.PublicPort) + errChan <- sr.ListenAndServe() + }() + + // start internal server + go func() { + mux := http.NewServeMux() + mux.Handle("/metrics", promhttp.Handler()) + mux.Handle("/", s.InternalEngine) + sr := &http.Server{ + Addr: fmt.Sprintf(":%d", s.InternalPort), + Handler: mux, + ReadTimeout: time.Duration(s.ReadTimeout) * time.Second, + WriteTimeout: time.Duration(s.WriteTimeout) * time.Second, + IdleTimeout: time.Duration(s.IdleTimeout) * time.Second, + MaxHeaderBytes: 1 << 20, + } + + nlog.Infof("server listen to internal address 0.0.0.0:%d", s.InternalPort) + errChan <- sr.ListenAndServe() + }() + return <-errChan +} diff --git a/pkg/secretbackend/mist/.OPENSOURCE-CLEANUP-RMDIR b/pkg/secretbackend/mist/.OPENSOURCE-CLEANUP-RMDIR new file mode 100644 index 000000000..e69de29bb diff --git a/pkg/secretbackend/mist/mist.go b/pkg/secretbackend/mist/mist.go new file mode 100644 index 000000000..6c0e7c9f3 --- /dev/null +++ b/pkg/secretbackend/mist/mist.go @@ -0,0 +1,266 @@ +// Copyright 2023 Ant Group Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mist + +import ( + "encoding/json" + "fmt" + "strings" + + "github.com/secretflow/kuscia/pkg/secretbackend" + "github.com/secretflow/kuscia/pkg/utils/nlog" + + "github.com/mitchellh/mapstructure" + "gitlab.alipay-inc.com/mist-sdk/mist_sdk_go/mist" +) + +const ( + // SecretWrapperContentTypeSecret for short content, value will be string + SecretWrapperContentTypeSecret = "secret" + // SecretWrapperContentTypeCompose for long content, value will be mistMultiValue + SecretWrapperContentTypeCompose = "compose" + + mistAntVipURLForStable = "antvip-pool.stable.alipay.net" + mistAntVipURLForTest = "antvip-pool.test.alipay.net" + mistAntVipURLForPre = "antvip-pool.cz99s.alipay.com" + mistAntVipURLForGray = "antvip-pool.cz87w.alipay.com" + mistAntVipURLForProd = "antvip-pool.global.alipay.com" + + mistBkmiURLForTest = "bkmi-read-pool.stable.global.alipay.com" + mistBkmiURLForProd = "bkmi-read-pool.global.alipay.com" +) + +var mistEndpoints = map[string]mistEndpoint{ + "stable": { + antVipURL: mistAntVipURLForStable, + BkmiURL: mistBkmiURLForTest, + }, + "test": { + antVipURL: mistAntVipURLForTest, + BkmiURL: mistBkmiURLForTest, + }, + "pre": { + antVipURL: mistAntVipURLForPre, + BkmiURL: mistBkmiURLForProd, + }, + "gray": { + antVipURL: mistAntVipURLForGray, + BkmiURL: mistBkmiURLForProd, + }, + "prod": { + antVipURL: mistAntVipURLForProd, + BkmiURL: mistBkmiURLForProd, + }, +} + +type Mist struct { + config Config + client IMistClient +} + +type Config struct { + AppName string `mapstructure:"appName"` + Tenant string `mapstructure:"tenant"` + Env string `mapstructure:"env"` +} + +type wrapperValue struct { + Type string + Content string // for SecretWrapperContentTypeSecret + Part int // for SecretWrapperContentTypeCompose +} + +type mistMultiValue struct { + MultiPart *int `json:"_kuscia_multipart_,omitempty"` +} + +type IMistClient interface { + GetSecretInfo(secretName string) (string, string, string, string, error) + GetFactorSecretInfo(secretName string, factor string) (string, string, string, string, error) +} + +func NewMistSecretDriver(configMap map[string]any) (secretbackend.SecretDriver, error) { + config := Config{} + if err := mapstructure.Decode(configMap, &config); err != nil { + return nil, err + } + mistConfig := mist.NewMistConfig() + mistConfig.SetAppName(config.AppName) + mistConfig.SetTenant(config.Tenant) + mistConfig.SetMode(config.Env) + endpoint, exist := mistEndpoints[config.Env] + if !exist { + return nil, fmt.Errorf("can not find mist endpoint for Env=%s", config.Env) + } + mistConfig.SetAntVipUrl(endpoint.antVipURL) + mistConfig.SetBkmiUrl(endpoint.BkmiURL) + + nlog.Infof("NewMistSecretDriver config=%+v, endpoint=%+v", config, endpoint) + + return &Mist{ + config: config, + client: mist.NewMistClient(mistConfig), + }, nil +} + +// setMistClient only for package debug +func (m *Mist) setMistClient(c IMistClient) { + m.client = c +} + +func (m *Mist) Set(confID string, value string) error { + return fmt.Errorf("unsupported for mist set") +} + +func (m *Mist) Get(confID string) (string, error) { + return m.GetByParams(confID, nil) +} + +type getByParams struct { + Factor string `mapstructure:"factor"` + PartFactors []string `mapstructure:"part_factors"` +} + +func (m *Mist) GetByParams(confID string, params map[string]any) (string, error) { + var mistGetParams *getByParams + if params != nil { + mistGetParams = &getByParams{} + if err := mapstructure.Decode(params, mistGetParams); err != nil { + return "", err + } + } + mistKey := m.idToMistKey(confID) + var factor = "" + if mistGetParams != nil { + factor = mistGetParams.Factor + } + secretValue, err := m.getMistValue(mistKey, factor) + if err != nil { + return "", err + } + + switch secretValue.Type { + case SecretWrapperContentTypeSecret: + return secretValue.Content, nil + case SecretWrapperContentTypeCompose: + if mistGetParams != nil { + if secretValue.Part != len(mistGetParams.PartFactors) { + return "", fmt.Errorf("find multipart secret but partCount not equals part factors num: %d != %d", + secretValue.Part, len(mistGetParams.PartFactors)) + } + } + partChannels := make([]<-chan getMistKeyAsyncResult, 0) + for i := 0; i < secretValue.Part; i++ { + partFactor := "" + if mistGetParams != nil { + partFactor = mistGetParams.PartFactors[i] + } + partChannels = append(partChannels, m.getMistKeyAsync(m.idToMistPartKey(confID, i), partFactor)) + } + var value = strings.Builder{} + for _, channel := range partChannels { + partResult := <-channel + if partResult.err != nil { + return "", err + } + if partResult.value.Type != SecretWrapperContentTypeSecret { + return "", fmt.Errorf("unexpected value type for mist multipart key: %s", partResult.value.Type) + } + value.WriteString(partResult.value.Content) + } + return value.String(), nil + default: + return "", fmt.Errorf("unexpected value type for mist key") + } +} + +func (m *Mist) Close() error { + m.client = nil + return nil +} + +func (m *Mist) getMistValue(mistKey string, factor string) (*wrapperValue, error) { + var secretInfo string + var err error + if factor != "" { + _, secretInfo, _, _, err = m.client.GetFactorSecretInfo(mistKey, factor) + } else { + _, secretInfo, _, _, err = m.client.GetSecretInfo(mistKey) + } + if err != nil { + return nil, err + } + + // try to detect mist multipart count + if part := m.detectMistMultiPart(secretInfo); part != nil { + return &wrapperValue{ + Type: SecretWrapperContentTypeCompose, + Part: *part, + }, nil + } + + return &wrapperValue{ + Type: SecretWrapperContentTypeSecret, + Content: secretInfo, + }, nil +} + +func (m *Mist) detectMistMultiPart(secretInfo string) *int { + // try to Unmarshal to mistMultiValue + var secretValue mistMultiValue + var err = json.Unmarshal([]byte(secretInfo), &secretValue) + if err != nil { + return nil + } + if secretValue.MultiPart == nil || *secretValue.MultiPart <= 0 { + return nil + } + return secretValue.MultiPart +} + +type getMistKeyAsyncResult struct { + value *wrapperValue + err error +} + +func (m *Mist) getMistKeyAsync(mistKey string, factor string) <-chan getMistKeyAsyncResult { + resultChan := make(chan getMistKeyAsyncResult) + go func() { + value, err := m.getMistValue(mistKey, factor) + resultChan <- getMistKeyAsyncResult{ + value: value, + err: err, + } + }() + + return resultChan +} + +func (m *Mist) idToMistKey(confID string) string { + return fmt.Sprintf("other_manual_%s_%s", m.config.AppName, confID) +} + +func (m *Mist) idToMistPartKey(confID string, part int) string { + return fmt.Sprintf("other_manual_%s_%s_part_%d", m.config.AppName, confID, part) +} + +type mistEndpoint struct { + antVipURL string + BkmiURL string +} + +func init() { + secretbackend.Register("mist", NewMistSecretDriver) +} diff --git a/pkg/secretbackend/mist/mist_test.go b/pkg/secretbackend/mist/mist_test.go new file mode 100644 index 000000000..7b9020ade --- /dev/null +++ b/pkg/secretbackend/mist/mist_test.go @@ -0,0 +1,168 @@ +// Copyright 2023 Ant Group Co., Ltd. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package mist + +import ( + "github.com/secretflow/kuscia/pkg/secretbackend" + "testing" + + "github.com/secretflow/kuscia/pkg/utils/nlog" + "github.com/secretflow/kuscia/pkg/web/asserts" +) + +var ( + mistDriverParams = map[string]any{ + "app_name": "secretpad", + "tenant": "ALIPAY", + "env": "test", + } +) + +func newMistDriver() (secretbackend.SecretDriver, error) { + backend, err := secretbackend.NewSecretBackendWith("mist", mistDriverParams) + if err != nil { + nlog.Errorf("New secret backend with name=%s failed: %s", "mist", err) + return nil, err + } + return backend, nil +} + +func TestNewMistSecretDriver(t *testing.T) { + type args struct { + configMap map[string]any + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "init mist client should return no error", + args: args{configMap: mistDriverParams}, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := NewMistSecretDriver(tt.args.configMap) + if (err != nil) != tt.wantErr { + t.Errorf("NewMistSecretDriver() error = %v, wantErr %v", err, tt.wantErr) + return + } + }) + } +} + +func TestMist_Set(t *testing.T) { + type args struct { + confID string + value string + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "mist set should return unsupported error", + args: args{ + confID: "abc", + value: "abc", + }, + wantErr: true, + }, + } + driver, err := newMistDriver() + _ = asserts.IsNil(err, "new mist driver error should be nil") + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := driver.Set(tt.args.confID, tt.args.value); (err != nil) != tt.wantErr { + t.Errorf("Set() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func TestMist_Get(t *testing.T) { + type fields struct { + configMap map[string]any + } + type args struct { + confID string + } + tests := []struct { + name string + fields fields + args args + want string + wantErr bool + }{ + { + name: "mist get should for short secret value should success", + fields: fields{configMap: mistDriverParams}, + args: args{ + confID: "a", + }, + want: "aa", + wantErr: false, + }, + { + name: "mist get should for long compose value should success", + fields: fields{configMap: mistDriverParams}, + args: args{ + confID: "b", + }, + want: "b0b0b1b1", + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + m, err := NewMistSecretDriver(tt.fields.configMap) + if (err != nil) != tt.wantErr { + t.Errorf("NewMistSecretDriver() error = %v, wantErr %v", err, tt.wantErr) + return + } + m.(*Mist).setMistClient(&MockMistClient{ + store: map[string]string{ + "other_manual__a": "aa", + "other_manual__b": "{\"_kuscia_multipart_\": 2}", + "other_manual__b_part_0": "b0b0", + "other_manual__b_part_1": "b1b1", + }, + }) + got, err := m.Get(tt.args.confID) + if (err != nil) != tt.wantErr { + t.Errorf("Get() error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("Get() got = %v, want %v", got, tt.want) + } + }) + } +} + +type MockMistClient struct { + store map[string]string +} + +func (m *MockMistClient) GetSecretInfo(secretName string) (string, string, string, string, error) { + return "", m.store[secretName], "", "", nil +} + +func (m *MockMistClient) GetFactorSecretInfo(secretName string, factor string) (string, string, string, string, error) { + return "", "", "", "", nil +} diff --git a/scripts/templates/task_input_config.2pc_balanced_psi.json b/scripts/templates/task_input_config.2pc_balanced_psi.json index a77e67634..dda6c4b5a 100644 --- a/scripts/templates/task_input_config.2pc_balanced_psi.json +++ b/scripts/templates/task_input_config.2pc_balanced_psi.json @@ -27,8 +27,8 @@ "sf_node_eval_param": { "domain": "data_prep", "name": "psi", - "version": "0.0.2", - "attr_paths": ["input/receiver_input/key", "input/sender_input/key", "protocol", "precheck_input", "bucket_size", "curve_type"], + "version": "0.0.4", + "attr_paths": ["input/receiver_input/key", "input/sender_input/key", "protocol", "precheck_input", "bucket_size", "curve_type", "left_side"], "attrs": [{ "ss": ["id1"] }, { @@ -41,6 +41,9 @@ "i64": "1048576" }, { "s": "CURVE_FOURQ" + }, { + "is_na": false, + "ss": ["alice"] }] }, "sf_input_ids": ["alice-table","bob-table"], diff --git a/scripts/templates/task_input_config.2pc_balanced_psi_dp.json b/scripts/templates/task_input_config.2pc_balanced_psi_dp.json index 16e172d2f..4b4d98d37 100644 --- a/scripts/templates/task_input_config.2pc_balanced_psi_dp.json +++ b/scripts/templates/task_input_config.2pc_balanced_psi_dp.json @@ -27,8 +27,8 @@ "sf_node_eval_param": { "domain": "data_prep", "name": "psi", - "version": "0.0.2", - "attr_paths": ["input/receiver_input/key", "input/sender_input/key", "protocol", "precheck_input", "bucket_size", "curve_type"], + "version": "0.0.4", + "attr_paths": ["input/receiver_input/key", "input/sender_input/key", "protocol", "precheck_input", "bucket_size", "curve_type","left_side"], "attrs": [{ "ss": ["id1"] }, { @@ -41,6 +41,9 @@ "i64": "1048576" }, { "s": "CURVE_FOURQ" + }, { + "is_na": false, + "ss": ["alice"] }] }, "sf_input_ids": ["alice-dp-table","bob-dp-table"],