From c0bbb8dfb45d7d1da1da55d6ed7e43bd4c154982 Mon Sep 17 00:00:00 2001 From: Kyle White Date: Thu, 25 May 2023 15:59:23 -0400 Subject: [PATCH 01/12] Update machine image and remove old deps --- .circleci/config.yml | 9 +++++++-- etc/testing/circle/install.sh | 9 --------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 19775678..5706a643 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,5 +1,10 @@ version: 2.1 +parameters: + machine_image: + type: string + default: ubuntu-2204:2023.04.2 + commands: deploy-pachyderm: steps: @@ -49,7 +54,7 @@ jobs: python-version: type: string machine: - image: ubuntu-2004:202111-01 + image: << pipeline.parameters.machine_image >> steps: - checkout - deploy-pachyderm @@ -77,7 +82,7 @@ jobs: python-version: type: string machine: - image: ubuntu-2004:202111-01 + image: << pipeline.parameters.machine_image >> steps: - checkout - deploy-pachyderm diff --git a/etc/testing/circle/install.sh b/etc/testing/circle/install.sh index 49355000..bc4317bc 100755 --- a/etc/testing/circle/install.sh +++ b/etc/testing/circle/install.sh @@ -4,15 +4,6 @@ set -ex mkdir -p cached-deps -# Install deps -sudo apt update -y -sudo apt-get install -y -qq \ - pkg-config \ - conntrack \ - pv \ - jq \ - socat - # Install Helm curl https://baltocdn.com/helm/signing.asc | sudo apt-key add - sudo apt-get install apt-transport-https --yes From 912bcded3f36af9d211c0eefba9cc43c272ed209 Mon Sep 17 00:00:00 2001 From: Kyle White Date: Thu, 25 May 2023 16:01:53 -0400 Subject: [PATCH 02/12] Update helm install --- etc/testing/circle/install.sh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/etc/testing/circle/install.sh b/etc/testing/circle/install.sh index bc4317bc..b04bab13 100755 --- a/etc/testing/circle/install.sh +++ b/etc/testing/circle/install.sh @@ -4,12 +4,13 @@ set -ex mkdir -p cached-deps -# Install Helm -curl https://baltocdn.com/helm/signing.asc | sudo apt-key add - -sudo apt-get install apt-transport-https --yes -echo "deb https://baltocdn.com/helm/stable/debian/ all main" | sudo tee /etc/apt/sources.list.d/helm-stable-debian.list -sudo apt-get update -sudo apt-get install helm +# Install helm +if [ ! -f cached-deps/helm ]; then + HELM_VERSION=3.5.4 + curl -L https://get.helm.sh/helm-v${HELM_VERSION}-linux-${ARCH}.tar.gz \ + | tar xzf - linux-${ARCH}/helm + mv ./linux-${ARCH}/helm cached-deps/helm +fi # Install kubectl # To get the latest kubectl version: From f6f2ea94e4d4684129f66c91d3a4323c2b46d4e3 Mon Sep 17 00:00:00 2001 From: Kyle White Date: Thu, 25 May 2023 16:04:21 -0400 Subject: [PATCH 03/12] Add architecture + pipefail --- etc/testing/circle/install.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/etc/testing/circle/install.sh b/etc/testing/circle/install.sh index b04bab13..51961a92 100755 --- a/etc/testing/circle/install.sh +++ b/etc/testing/circle/install.sh @@ -1,8 +1,9 @@ #!/bin/bash -set -ex +set -euxo pipefail -mkdir -p cached-deps +ARCH=amd64 +if [ "$(uname -m)" = "aarch64" ]; then ARCH=arm64; fi # Install helm if [ ! -f cached-deps/helm ]; then From 2fd2817925120d6a6b38394c561f52ffdfde6c5d Mon Sep 17 00:00:00 2001 From: Kyle White Date: Thu, 25 May 2023 16:07:52 -0400 Subject: [PATCH 04/12] Add to path --- .circleci/config.yml | 4 ++++ etc/testing/circle/install.sh | 2 ++ 2 files changed, 6 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5706a643..40f5ef73 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -11,6 +11,10 @@ commands: - run: name: Install dependencies command: etc/testing/circle/install.sh + - run: + name: Add deps to path + command: | + echo 'export PATH=/home/circleci/project/cached-deps:$PATH' >> $BASH_ENV - run: name: Start minikube command: etc/testing/circle/start-minikube.sh diff --git a/etc/testing/circle/install.sh b/etc/testing/circle/install.sh index 51961a92..84c3b3a7 100755 --- a/etc/testing/circle/install.sh +++ b/etc/testing/circle/install.sh @@ -5,6 +5,8 @@ set -euxo pipefail ARCH=amd64 if [ "$(uname -m)" = "aarch64" ]; then ARCH=arm64; fi +mkdir -p cached-deps + # Install helm if [ ! -f cached-deps/helm ]; then HELM_VERSION=3.5.4 From ef79c6306c7ab2d4fe748f609675bfbd98c7af3f Mon Sep 17 00:00:00 2001 From: Kyle White Date: Fri, 26 May 2023 08:55:33 -0400 Subject: [PATCH 05/12] Remove start minikube script, update minikube, add circle command for starting minikube --- .circleci/config.yml | 13 +++++++--- etc/testing/circle/start-minikube.sh | 38 ---------------------------- version.json | 2 +- 3 files changed, 11 insertions(+), 42 deletions(-) delete mode 100755 etc/testing/circle/start-minikube.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 40f5ef73..e86bee44 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -6,6 +6,15 @@ parameters: default: ubuntu-2204:2023.04.2 commands: + start-minikube: + steps: + - run: + name: Start minikube + command: | + KUBE_VERSION="$(jq -r .kubernetes < version.json)" + minikube start --vm-driver=docker --kubernetes-version=${KUBE_VERSION} --cpus=7 --memory=12Gi --wait=all + # TODO: Add back in minikube wait + #background: true deploy-pachyderm: steps: - run: @@ -15,9 +24,7 @@ commands: name: Add deps to path command: | echo 'export PATH=/home/circleci/project/cached-deps:$PATH' >> $BASH_ENV - - run: - name: Start minikube - command: etc/testing/circle/start-minikube.sh + - start-minikube - run: name: Deploy pachyderm command: etc/testing/circle/deploy-pachyderm.sh diff --git a/etc/testing/circle/start-minikube.sh b/etc/testing/circle/start-minikube.sh deleted file mode 100755 index f476a7a9..00000000 --- a/etc/testing/circle/start-minikube.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash - -set -ex - -export PATH=$(pwd):$(pwd)/cached-deps:$GOPATH/bin:$PATH - -# Parse flags -KUBE_VERSION="$( jq -r .kubernetes <"$(git rev-parse --show-toplevel)/version.json" )" -minikube_args=( - "--vm-driver=docker" - "--kubernetes-version=${KUBE_VERSION}" -) -while getopts ":v" opt; do - case "${opt}" in - v) - KUBE_VERSION="v${OPTARG}" - ;; - \?) - echo "Invalid argument: ${opt}" - exit 1 - ;; - esac -done - -minikube start "${minikube_args[@]}" - -# Try to connect for three minutes -for _ in $(seq 36); do - if kubectl version &>/dev/null; then - exit 0 - fi - sleep 5 -done - -# Give up--kubernetes isn't coming up -minikube delete -sleep 30 # Wait for minikube to go completely down -exit 1 diff --git a/version.json b/version.json index 0570fed8..58c55904 100644 --- a/version.json +++ b/version.json @@ -1,5 +1,5 @@ { "pachyderm": "2.6.0", "kubernetes": "1.22.8", - "minikube": "1.25.2" + "minikube": "1.30.1" } From 3202466498e7a2e495fd434f446f4b3a6c8beaa6 Mon Sep 17 00:00:00 2001 From: Kyle White Date: Fri, 26 May 2023 08:57:42 -0400 Subject: [PATCH 06/12] Change CPU and mem for minikube --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index e86bee44..6e9a54eb 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,7 +12,7 @@ commands: name: Start minikube command: | KUBE_VERSION="$(jq -r .kubernetes < version.json)" - minikube start --vm-driver=docker --kubernetes-version=${KUBE_VERSION} --cpus=7 --memory=12Gi --wait=all + minikube start --vm-driver=docker --kubernetes-version=${KUBE_VERSION} --cpus=4 --memory=8Gi --wait=all # TODO: Add back in minikube wait #background: true deploy-pachyderm: From 9a99b69fd46c4532648323d12734e6672bcc9235 Mon Sep 17 00:00:00 2001 From: Kyle White Date: Fri, 26 May 2023 09:05:32 -0400 Subject: [PATCH 07/12] Remove CPU and memory setting from minikube, Remove deploy pachyderm script, move wait for pach until after install and setup of python (for concurrency) --- .circleci/config.yml | 16 ++++++++++++++-- etc/testing/circle/deploy-pachyderm.sh | 13 ------------- 2 files changed, 14 insertions(+), 15 deletions(-) delete mode 100755 etc/testing/circle/deploy-pachyderm.sh diff --git a/.circleci/config.yml b/.circleci/config.yml index 6e9a54eb..be2cc27b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -12,9 +12,15 @@ commands: name: Start minikube command: | KUBE_VERSION="$(jq -r .kubernetes < version.json)" - minikube start --vm-driver=docker --kubernetes-version=${KUBE_VERSION} --cpus=4 --memory=8Gi --wait=all + minikube start --vm-driver=docker --kubernetes-version=${KUBE_VERSION} --wait=all # TODO: Add back in minikube wait #background: true + wait-for-pach: + steps: + - run: + name: Wait for pachyderm + command: | + kubectl wait --for=condition=available deployment -l app=pachd --timeout=5m deploy-pachyderm: steps: - run: @@ -27,7 +33,11 @@ commands: - start-minikube - run: name: Deploy pachyderm - command: etc/testing/circle/deploy-pachyderm.sh + command: | + PACHYDERM_VERSION="$(jq -r .pachyderm version.json)" + helm repo add pachyderm https://pachyderm.github.io/helmchart + helm repo update + helm install pachd pachyderm/pachyderm --set deployTarget=LOCAL --version ${PACHYDERM_VERSION} install-python: parameters: @@ -73,6 +83,7 @@ jobs: version: <> - install-poetry - create-venv + - wait-for-pach - run: name: Run unit tests command: | @@ -101,6 +112,7 @@ jobs: version: <> - install-poetry - create-venv + - wait-for-pach - run: name: Run opencv example command: | diff --git a/etc/testing/circle/deploy-pachyderm.sh b/etc/testing/circle/deploy-pachyderm.sh deleted file mode 100755 index 8b38a10f..00000000 --- a/etc/testing/circle/deploy-pachyderm.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -set -ex - -export PATH=$(pwd):$(pwd)/cached-deps:$GOPATH/bin:$PATH -export PACHYDERM_VERSION="$(jq -r .pachyderm version.json)" - -helm repo add pachyderm https://pachyderm.github.io/helmchart -helm repo update -helm install pachd pachyderm/pachyderm --set deployTarget=LOCAL --version ${PACHYDERM_VERSION} - -kubectl wait --for=condition=available deployment -l app=pachd --timeout=5m -pachctl version \ No newline at end of file From 7e2142934741b8279f6176d1451715b20de91a2c Mon Sep 17 00:00:00 2001 From: Kyle White Date: Fri, 26 May 2023 09:08:32 -0400 Subject: [PATCH 08/12] Add timeout --- .circleci/config.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index be2cc27b..4d689ee3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -90,6 +90,7 @@ jobs: pachctl port-forward & mkdir -p test-results poetry run pytest -v --junitxml=test-results/junit.xml tests + no_output_timeout: 20m - store_test_results: path: test-results - store_artifacts: @@ -118,6 +119,7 @@ jobs: command: | pachctl port-forward & poetry run python ./examples/opencv/opencv.py + no_output_timeout: 20m - run: when: on_fail command: etc/testing/circle/kube_debug.sh From 75dedbd71229cd5440beee4667e85011f5e2dff6 Mon Sep 17 00:00:00 2001 From: Kyle White Date: Fri, 26 May 2023 09:53:32 -0400 Subject: [PATCH 09/12] cache python version (#431) * cache python version * Fix up cache key * Skip python version if already installed, check location of python --- .circleci/config.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4d689ee3..c4f4e7ff 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -44,13 +44,19 @@ commands: version: type: string steps: + - restore_cache: + key: python-pach-pyenv-v1-<< parameters.version >> - run: name: Install python command: | git -C "/opt/circleci/.pyenv/" checkout master -q git -C "/opt/circleci/.pyenv/" pull -q - pyenv install <> + pyenv install -s <> pyenv global <> + - save_cache: + key: python-pach-pyenv-v1-<< parameters.version >> + paths: + - /opt/circleci/.pyenv/versions/<< parameters.version >> install-poetry: steps: @@ -58,6 +64,8 @@ commands: name: Install poetry command: | curl -fsS -o install-poetry.py https://install.python-poetry.org + which python + which python3 python3 install-poetry.py --version 1.2.2 echo 'export PATH=$HOME/.local/bin:$PATH' >> $BASH_ENV rm install-poetry.py From 35936457f1b9f01482aae095ad21da1c7b061f9a Mon Sep 17 00:00:00 2001 From: Kyle White Date: Fri, 26 May 2023 10:28:09 -0400 Subject: [PATCH 10/12] Cleanup --- .circleci/config.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index c4f4e7ff..5a25c4f2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -13,8 +13,7 @@ commands: command: | KUBE_VERSION="$(jq -r .kubernetes < version.json)" minikube start --vm-driver=docker --kubernetes-version=${KUBE_VERSION} --wait=all - # TODO: Add back in minikube wait - #background: true + wait-for-pach: steps: - run: @@ -64,8 +63,6 @@ commands: name: Install poetry command: | curl -fsS -o install-poetry.py https://install.python-poetry.org - which python - which python3 python3 install-poetry.py --version 1.2.2 echo 'export PATH=$HOME/.local/bin:$PATH' >> $BASH_ENV rm install-poetry.py From 459a7636f206d36bee2a980e6b0e69fe72604fd0 Mon Sep 17 00:00:00 2001 From: Kyle White Date: Thu, 25 May 2023 12:21:04 -0400 Subject: [PATCH 11/12] Use NodePort for Proxy --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 5a25c4f2..78d15ecf 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -36,7 +36,7 @@ commands: PACHYDERM_VERSION="$(jq -r .pachyderm version.json)" helm repo add pachyderm https://pachyderm.github.io/helmchart helm repo update - helm install pachd pachyderm/pachyderm --set deployTarget=LOCAL --version ${PACHYDERM_VERSION} + helm install pachd pachyderm/pachyderm --set deployTarget=LOCAL,proxy.service.type=NodePort --version ${PACHYDERM_VERSION} install-python: parameters: From b4d08558293dab4c6d54c42719cb3a2bc386860a Mon Sep 17 00:00:00 2001 From: Kyle White Date: Thu, 25 May 2023 15:25:43 -0400 Subject: [PATCH 12/12] Remove port forward and login to pach at proxy address --- .circleci/config.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 78d15ecf..11443308 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -37,6 +37,10 @@ commands: helm repo add pachyderm https://pachyderm.github.io/helmchart helm repo update helm install pachd pachyderm/pachyderm --set deployTarget=LOCAL,proxy.service.type=NodePort --version ${PACHYDERM_VERSION} + - run: + name: Login to Pach + command: | + pachctl connect grpc://$(minikube ip):30080 install-python: parameters: @@ -92,7 +96,6 @@ jobs: - run: name: Run unit tests command: | - pachctl port-forward & mkdir -p test-results poetry run pytest -v --junitxml=test-results/junit.xml tests no_output_timeout: 20m @@ -122,7 +125,6 @@ jobs: - run: name: Run opencv example command: | - pachctl port-forward & poetry run python ./examples/opencv/opencv.py no_output_timeout: 20m - run: