diff --git a/.github/actions/kfp-cluster/action.yml b/.github/actions/kfp-cluster/action.yml index 4abf265b404..c669a40a186 100644 --- a/.github/actions/kfp-cluster/action.yml +++ b/.github/actions/kfp-cluster/action.yml @@ -14,8 +14,8 @@ runs: - name: Build images shell: bash - run: ./scripts/deploy/github/build-images.sh + run: ./.github/resources/scripts/build-images.sh - name: Deploy KFP shell: bash - run: ./scripts/deploy/github/deploy-kfp.sh + run: ./.github/resources/scripts/deploy-kfp.sh diff --git a/.github/actions/kfp-tekton-cluster/action.yml b/.github/actions/kfp-tekton-cluster/action.yml index 08120570b93..4c396305e43 100644 --- a/.github/actions/kfp-tekton-cluster/action.yml +++ b/.github/actions/kfp-tekton-cluster/action.yml @@ -14,8 +14,8 @@ runs: - name: Build images shell: bash - run: ./scripts/deploy/github/build-images.sh + run: ./.github/resources/scripts/build-images.sh - name: Deploy KFP shell: bash - run: ./scripts/deploy/github/deploy-kfp-tekton.sh + run: ./.github/resources/scripts/deploy-kfp-tekton.sh diff --git a/.github/resources/manifests/argo/kustomization.yaml b/.github/resources/manifests/argo/kustomization.yaml new file mode 100644 index 00000000000..cd2c6bdb0b3 --- /dev/null +++ b/.github/resources/manifests/argo/kustomization.yaml @@ -0,0 +1,19 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization + +resources: +- ../../../../manifests/kustomize/env/platform-agnostic + +images: +- name: gcr.io/ml-pipeline/api-server + newName: kind-registry:5000/apiserver + newTag: latest +- name: gcr.io/ml-pipeline/persistenceagent + newName: kind-registry:5000/persistenceagent + newTag: latest +- name: gcr.io/ml-pipeline/scheduledworkflow + newName: kind-registry:5000/scheduledworkflow + newTag: latest + +patchesStrategicMerge: +- overlays/apiserver-env.yaml diff --git a/.github/resources/manifests/argo/overlays/apiserver-env.yaml b/.github/resources/manifests/argo/overlays/apiserver-env.yaml new file mode 100644 index 00000000000..1aefd4aa43d --- /dev/null +++ b/.github/resources/manifests/argo/overlays/apiserver-env.yaml @@ -0,0 +1,14 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: ml-pipeline +spec: + template: + spec: + containers: + - name: ml-pipeline-api-server + env: + - name: V2_DRIVER_IMAGE + value: kind-registry:5000/driver + - name: V2_LAUNCHER_IMAGE + value: kind-registry:5000/launcher diff --git a/scripts/deploy/github/manifests/kustomization.yaml b/.github/resources/manifests/tekton/kustomization.yaml similarity index 100% rename from scripts/deploy/github/manifests/kustomization.yaml rename to .github/resources/manifests/tekton/kustomization.yaml diff --git a/scripts/deploy/github/build-images.sh b/.github/resources/scripts/build-images.sh similarity index 100% rename from scripts/deploy/github/build-images.sh rename to .github/resources/scripts/build-images.sh diff --git a/scripts/deploy/github/deploy-kfp-tekton.sh b/.github/resources/scripts/deploy-kfp-tekton.sh similarity index 93% rename from scripts/deploy/github/deploy-kfp-tekton.sh rename to .github/resources/scripts/deploy-kfp-tekton.sh index 17df6278df4..502c96b5045 100755 --- a/scripts/deploy/github/deploy-kfp-tekton.sh +++ b/.github/resources/scripts/deploy-kfp-tekton.sh @@ -33,7 +33,8 @@ then fi # Deploy manifest -kubectl apply -k "scripts/deploy/github/manifests" || EXIT_CODE=$? +TEST_MANIFESTS=".github/resources/manifests/tekton" +kubectl apply -k "${TEST_MANIFESTS}" || EXIT_CODE=$? if [[ $EXIT_CODE -ne 0 ]] then echo "Deploy unsuccessful. Failure applying $KUSTOMIZE_DIR." diff --git a/scripts/deploy/github/deploy-kfp.sh b/.github/resources/scripts/deploy-kfp.sh similarity index 52% rename from scripts/deploy/github/deploy-kfp.sh rename to .github/resources/scripts/deploy-kfp.sh index 6206b92fe58..f4675a3196c 100755 --- a/scripts/deploy/github/deploy-kfp.sh +++ b/.github/resources/scripts/deploy-kfp.sh @@ -32,27 +32,15 @@ then exit $EXIT_CODE fi -PLATFORM_AGNOSTIC_MANIFESTS="manifests/kustomize/env/platform-agnostic" - -kubectl apply -k "${PLATFORM_AGNOSTIC_MANIFESTS}" || EXIT_CODE=$? +# Deploy manifest +TEST_MANIFESTS=".github/resources/manifests/argo" +kubectl apply -k "${TEST_MANIFESTS}" || EXIT_CODE=$? if [[ $EXIT_CODE -ne 0 ]] then - echo "Deploy unsuccessful. Failure applying ${PLATFORM_AGNOSTIC_MANIFESTS}." + echo "Deploy unsuccessful. Failure applying ${TEST_MANIFESTS}." exit 1 fi -echo "Patching deployments to use built docker images..." -# Patch API server -kubectl patch deployment ml-pipeline -p '{"spec": {"template": {"spec": {"containers": [{"name": "ml-pipeline-api-server", "image": "kind-registry:5000/apiserver"}]}}}}' -n kubeflow -# Patch persistence agent -kubectl patch deployment.apps/ml-pipeline-persistenceagent -p '{"spec": {"template": {"spec": {"containers": [{"name": "ml-pipeline-persistenceagent", "image": "kind-registry:5000/persistenceagent"}]}}}}' -n kubeflow -# Patch scheduled workflow -kubectl patch deployment.apps/ml-pipeline-scheduledworkflow -p '{"spec": {"template": {"spec": {"containers": [{"name": "ml-pipeline-scheduledworkflow", "image": "kind-registry:5000/scheduledworkflow"}]}}}}' -n kubeflow - -# Update environment variables to override driver / launcher -kubectl set env deployments/ml-pipeline V2_DRIVER_IMAGE=kind-registry:5000/driver -n kubeflow -kubectl set env deployments/ml-pipeline V2_LAUNCHER_IMAGE=kind-registry:5000/launcher -n kubeflow - # Check if all pods are running - (10 minutes) wait_for_pods || EXIT_CODE=$? if [[ $EXIT_CODE -ne 0 ]] diff --git a/scripts/deploy/github/e2e-test.sh b/.github/resources/scripts/e2e-test.sh similarity index 100% rename from scripts/deploy/github/e2e-test.sh rename to .github/resources/scripts/e2e-test.sh diff --git a/scripts/deploy/github/forward-port.sh b/.github/resources/scripts/forward-port.sh similarity index 100% rename from scripts/deploy/github/forward-port.sh rename to .github/resources/scripts/forward-port.sh diff --git a/scripts/deploy/github/helper-functions.sh b/.github/resources/scripts/helper-functions.sh similarity index 100% rename from scripts/deploy/github/helper-functions.sh rename to .github/resources/scripts/helper-functions.sh diff --git a/scripts/deploy/github/kfp-readiness/wait_for_pods.py b/.github/resources/scripts/kfp-readiness/wait_for_pods.py similarity index 100% rename from scripts/deploy/github/kfp-readiness/wait_for_pods.py rename to .github/resources/scripts/kfp-readiness/wait_for_pods.py diff --git a/scripts/deploy/github/re-tag.sh b/.github/resources/scripts/re-tag.sh similarity index 100% rename from scripts/deploy/github/re-tag.sh rename to .github/resources/scripts/re-tag.sh diff --git a/scripts/deploy/github/test-dynamic-loop.sh b/.github/resources/scripts/test-dynamic-loop.sh similarity index 100% rename from scripts/deploy/github/test-dynamic-loop.sh rename to .github/resources/scripts/test-dynamic-loop.sh diff --git a/scripts/deploy/github/test-env.sh b/.github/resources/scripts/test-env.sh similarity index 100% rename from scripts/deploy/github/test-env.sh rename to .github/resources/scripts/test-env.sh diff --git a/scripts/deploy/github/test-flip-coin.sh b/.github/resources/scripts/test-flip-coin.sh similarity index 100% rename from scripts/deploy/github/test-flip-coin.sh rename to .github/resources/scripts/test-flip-coin.sh diff --git a/scripts/deploy/github/test-pipeline.sh b/.github/resources/scripts/test-pipeline.sh similarity index 100% rename from scripts/deploy/github/test-pipeline.sh rename to .github/resources/scripts/test-pipeline.sh diff --git a/scripts/deploy/github/test-secret-as-env.sh b/.github/resources/scripts/test-secret-as-env.sh similarity index 100% rename from scripts/deploy/github/test-secret-as-env.sh rename to .github/resources/scripts/test-secret-as-env.sh diff --git a/scripts/deploy/github/test-secret-as-volume.sh b/.github/resources/scripts/test-secret-as-volume.sh similarity index 100% rename from scripts/deploy/github/test-secret-as-volume.sh rename to .github/resources/scripts/test-secret-as-volume.sh diff --git a/scripts/deploy/github/test-static-loop.sh b/.github/resources/scripts/test-static-loop.sh similarity index 100% rename from scripts/deploy/github/test-static-loop.sh rename to .github/resources/scripts/test-static-loop.sh diff --git a/scripts/deploy/github/test-volume.sh b/.github/resources/scripts/test-volume.sh similarity index 100% rename from scripts/deploy/github/test-volume.sh rename to .github/resources/scripts/test-volume.sh diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index faef43a0042..27f1cd21f4a 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -10,7 +10,7 @@ on: - '.github/workflows/backend.yml' - 'go.mod' - 'backend/**' - - 'scripts/deploy/github/**' + - '.github/resources/**' - 'manifests/kustomize/**' env: @@ -48,23 +48,23 @@ jobs: - name: "flip coin test" run: | . .venv/bin/activate - TEST_SCRIPT="test-flip-coin.sh" ./scripts/deploy/github/e2e-test.sh + TEST_SCRIPT="test-flip-coin.sh" ./.github/resources/scripts/e2e-test.sh - name: "static loop test" run: | . .venv/bin/activate - TEST_SCRIPT="test-static-loop.sh" ./scripts/deploy/github/e2e-test.sh + TEST_SCRIPT="test-static-loop.sh" ./.github/resources/scripts/e2e-test.sh - name: "dynamic loop test" run: | . .venv/bin/activate - TEST_SCRIPT="test-dynamic-loop.sh" ./scripts/deploy/github/e2e-test.sh + TEST_SCRIPT="test-dynamic-loop.sh" ./.github/resources/scripts/e2e-test.sh - name: "use env" run: | . .venv/bin/activate - TEST_SCRIPT="test-env.sh" ./scripts/deploy/github/e2e-test.sh + TEST_SCRIPT="test-env.sh" ./.github/resources/scripts/e2e-test.sh - name: "use volume" run: | . .venv/bin/activate - TEST_SCRIPT="test-volume.sh" ./scripts/deploy/github/e2e-test.sh + TEST_SCRIPT="test-volume.sh" ./.github/resources/scripts/e2e-test.sh - name: Collect test results if: always() uses: actions/upload-artifact@v4 diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index bc7783bafda..4273b9b2fbb 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -7,7 +7,7 @@ on: pull_request: paths: - '.github/workflows/e2e-test.yml' - - 'scripts/deploy/github/**' + - '.github/resources/**' - 'go.mod' - 'go.sum' - 'backend/**' @@ -32,7 +32,7 @@ jobs: uses: ./.github/actions/kfp-cluster - name: Forward API port - run: ./scripts/deploy/github/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888 + run: ./.github/resources/scripts/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888 - name: Initialization tests v1 working-directory: ./backend/test/initialization @@ -60,7 +60,7 @@ jobs: uses: ./.github/actions/kfp-cluster - name: Forward API port - run: ./scripts/deploy/github/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888 + run: ./.github/resources/scripts/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888 - name: Initialization tests v2 working-directory: ./backend/test/v2/initialization @@ -88,7 +88,7 @@ jobs: uses: ./.github/actions/kfp-cluster - name: Forward API port - run: ./scripts/deploy/github/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888 + run: ./.github/resources/scripts/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888 - name: API integration tests v1 working-directory: ./backend/test/integration @@ -116,7 +116,7 @@ jobs: uses: ./.github/actions/kfp-cluster - name: Forward API port - run: ./scripts/deploy/github/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888 + run: ./.github/resources/scripts/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888 - name: API integration tests v2 working-directory: ./backend/test/v2/integration @@ -144,10 +144,10 @@ jobs: uses: ./.github/actions/kfp-cluster - name: Forward API port - run: ./scripts/deploy/github/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888 + run: ./.github/resources/scripts/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888 - name: Forward Frontend port - run: ./scripts/deploy/github/forward-port.sh "kubeflow" "ml-pipeline-ui" 3000 3000 + run: ./.github/resources/scripts/forward-port.sh "kubeflow" "ml-pipeline-ui" 3000 3000 - name: Build frontend integration tests image working-directory: ./test/frontend-integration-test @@ -178,7 +178,7 @@ jobs: uses: ./.github/actions/kfp-cluster - name: Forward API port - run: ./scripts/deploy/github/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888 + run: ./.github/resources/scripts/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888 - name: Install prerequisites run: pip3 install -r ./test/sample-test/requirements.txt diff --git a/.github/workflows/kfp-kubernetes-execution-tests.yml b/.github/workflows/kfp-kubernetes-execution-tests.yml index 92c60ccf43d..d2521a7a317 100644 --- a/.github/workflows/kfp-kubernetes-execution-tests.yml +++ b/.github/workflows/kfp-kubernetes-execution-tests.yml @@ -7,7 +7,7 @@ on: pull_request: paths: - '.github/workflows/kfp-kubernetes-execution-tests.yml' - - 'scripts/deploy/github/**' + - '.github/resources/**' - 'sdk/python/**' - 'api/v2alpha1/**' - 'kubernetes_platform/**' @@ -28,7 +28,7 @@ jobs: uses: ./.github/actions/kfp-cluster - name: Forward API port - run: ./scripts/deploy/github/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888 + run: ./.github/resources/scripts/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888 - name: apt-get update run: sudo apt-get update diff --git a/.github/workflows/kfp-samples.yml b/.github/workflows/kfp-samples.yml index e63619f2574..63569884682 100644 --- a/.github/workflows/kfp-samples.yml +++ b/.github/workflows/kfp-samples.yml @@ -6,7 +6,7 @@ on: - master pull_request: paths: - - 'scripts/deploy/github/**' + - '.github/resources/**' - 'samples/**' - 'backend/src/v2/**' - '.github/workflows/kfp-samples.yml' @@ -28,7 +28,7 @@ jobs: uses: ./.github/actions/kfp-cluster - name: Forward API port - run: ./scripts/deploy/github/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888 + run: ./.github/resources/scripts/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888 - name: Run Samples Tests run: | diff --git a/.github/workflows/kubeflow-pipelines-integration-v2.yml b/.github/workflows/kubeflow-pipelines-integration-v2.yml index cef8c26d764..5062e98e4d9 100644 --- a/.github/workflows/kubeflow-pipelines-integration-v2.yml +++ b/.github/workflows/kubeflow-pipelines-integration-v2.yml @@ -7,7 +7,7 @@ on: pull_request: paths: - '.github/workflows/kubeflow-pipelines-integration-v2.yml' - - 'scripts/deploy/github/**' + - '.github/resources/**' - 'samples/core/parameterized_tfx_oss/**' - 'samples/core/dataflow/**' - 'backend/**' @@ -32,7 +32,7 @@ jobs: uses: ./.github/actions/kfp-cluster - name: Forward API port - run: ./scripts/deploy/github/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888 + run: ./.github/resources/scripts/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888 - name: Run the Integration Tests run: | diff --git a/.github/workflows/sdk-execution.yml b/.github/workflows/sdk-execution.yml index b646534fa68..9f4b93906ab 100644 --- a/.github/workflows/sdk-execution.yml +++ b/.github/workflows/sdk-execution.yml @@ -7,7 +7,7 @@ on: pull_request: paths: - '.github/workflows/sdk-execution.yml' - - 'scripts/deploy/github/**' + - '.github/resources/**' - 'sdk/python/**' - 'api/v2alpha1/**' @@ -27,7 +27,7 @@ jobs: uses: ./.github/actions/kfp-cluster - name: Forward API port - run: ./scripts/deploy/github/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888 + run: ./.github/resources/scripts/forward-port.sh "kubeflow" "ml-pipeline" 8888 8888 - name: apt-get update run: sudo apt-get update diff --git a/.github/workflows/upgrade-test.yml b/.github/workflows/upgrade-test.yml index bd40b868549..090f23c1794 100644 --- a/.github/workflows/upgrade-test.yml +++ b/.github/workflows/upgrade-test.yml @@ -7,7 +7,7 @@ on: pull_request: paths: - '.github/workflows/upgrade-test.yml' - - 'scripts/deploy/github/**' + - '.github/resources/**' - 'backend/**' - 'manifests/kustomize/**' diff --git a/scripts/OWNERS b/scripts/OWNERS deleted file mode 100644 index cd28cb484b8..00000000000 --- a/scripts/OWNERS +++ /dev/null @@ -1,5 +0,0 @@ -approvers: - - hbelmiro - - DharmitD -reviewers: - - rimolive