From 70492f0c0107ca096a4275453779e2b1225babab Mon Sep 17 00:00:00 2001 From: ciaran28 Date: Tue, 31 Jan 2023 14:09:45 +0000 Subject: [PATCH 1/7] KV Environment Location From UK to US --- .github/MLOps_Engineer/Utilities/Bash/utilsSetEnvVariables.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/MLOps_Engineer/Utilities/Bash/utilsSetEnvVariables.sh b/.github/MLOps_Engineer/Utilities/Bash/utilsSetEnvVariables.sh index 78b9d9a3..f7af412c 100644 --- a/.github/MLOps_Engineer/Utilities/Bash/utilsSetEnvVariables.sh +++ b/.github/MLOps_Engineer/Utilities/Bash/utilsSetEnvVariables.sh @@ -16,6 +16,7 @@ DATABRICKS_INSTANCE="$(az databricks workspace list -g $RESOURCE_GROUP_NAME --qu WORKSPACE_ID=$(az databricks workspace list -g $RESOURCE_GROUP_NAME --query "[].id" -o tsv) AZ_KEYVAULT_NAME=$(az keyvault list -g $RESOURCE_GROUP_NAME --query "[].name" -o tsv) SUBSCRIPTION_ID=$( az account show --query id -o tsv ) +echo $SUBSCRIPTION_ID #DATABRICKS_TOKEN=$(az keyvault secret show --name "dbkstoken" --vault-name $AZ_KEYVAULT_NAME --query "value" -o tsv) From 7198ecde8de126c4ca586626d14aa57ea76100b6 Mon Sep 17 00:00:00 2001 From: ciaran28 Date: Tue, 31 Jan 2023 14:16:01 +0000 Subject: [PATCH 2/7] Bug Fix --- .../MLOps_Engineer/Utilities/Bash/utilsSetEnvVariables.sh | 2 +- .../MLOps_Engineer/Utilities/Bash/utilsSetEnvVariables.sh | 4 ++++ .github/workflows/taskDatabricks.yaml | 8 ++++---- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.azureDevOps/MLOps_Engineer/Utilities/Bash/utilsSetEnvVariables.sh b/.azureDevOps/MLOps_Engineer/Utilities/Bash/utilsSetEnvVariables.sh index 4e046495..08c72ed7 100644 --- a/.azureDevOps/MLOps_Engineer/Utilities/Bash/utilsSetEnvVariables.sh +++ b/.azureDevOps/MLOps_Engineer/Utilities/Bash/utilsSetEnvVariables.sh @@ -15,7 +15,7 @@ DATABRICKS_ORDGID=$(az databricks workspace list -g $RESOURCE_GROUP_NAME --query DATABRICKS_INSTANCE="$(az databricks workspace list -g $RESOURCE_GROUP_NAME --query "[].workspaceUrl" -o tsv)" WORKSPACE_ID=$(az databricks workspace list -g $RESOURCE_GROUP_NAME --query "[].id" -o tsv) AZ_KEYVAULT_NAME=$(az keyvault list -g $RESOURCE_GROUP_NAME --query "[].name" -o tsv) -SUBSCRIPTION_ID=$( az account show --query id -o tsv ) +SUBSCRIPTION_ID=$( az account show --query id ) #DATABRICKS_TOKEN=$(az keyvault secret show --name "dbkstoken" --vault-name $AZ_KEYVAULT_NAME --query "value" -o tsv) diff --git a/.github/MLOps_Engineer/Utilities/Bash/utilsSetEnvVariables.sh b/.github/MLOps_Engineer/Utilities/Bash/utilsSetEnvVariables.sh index f7af412c..2455334f 100644 --- a/.github/MLOps_Engineer/Utilities/Bash/utilsSetEnvVariables.sh +++ b/.github/MLOps_Engineer/Utilities/Bash/utilsSetEnvVariables.sh @@ -17,6 +17,10 @@ WORKSPACE_ID=$(az databricks workspace list -g $RESOURCE_GROUP_NAME --query "[]. AZ_KEYVAULT_NAME=$(az keyvault list -g $RESOURCE_GROUP_NAME --query "[].name" -o tsv) SUBSCRIPTION_ID=$( az account show --query id -o tsv ) echo $SUBSCRIPTION_ID +echo $DATABRICKS_ORDGID +echo $WORKSPACE_ID +echo $AZ_KEYVAULT_NAME +echo $SUBSCRIPTION_ID #DATABRICKS_TOKEN=$(az keyvault secret show --name "dbkstoken" --vault-name $AZ_KEYVAULT_NAME --query "value" -o tsv) diff --git a/.github/workflows/taskDatabricks.yaml b/.github/workflows/taskDatabricks.yaml index 1cb91e9d..c4afdc54 100644 --- a/.github/workflows/taskDatabricks.yaml +++ b/.github/workflows/taskDatabricks.yaml @@ -72,10 +72,10 @@ jobs: #################################/ ## Deploy Azure Infrastructure #################################/ - - name: Deploy DBX CICD Azure Resources - run: bash .github/MLOps_Engineer/Utilities/Bash/utilsCreateAzureResources.sh - env: - ENVIRONMENT: ${{ inputs.ENVIRONMENT }} + #- name: Deploy DBX CICD Azure Resources + # run: bash .github/MLOps_Engineer/Utilities/Bash/utilsCreateAzureResources.sh + # env: + # ENVIRONMENT: ${{ inputs.ENVIRONMENT }} #################################/ From 5f0154e85fba46cc8156883f914a432f778ad334 Mon Sep 17 00:00:00 2001 From: ciaran28 Date: Tue, 31 Jan 2023 14:17:16 +0000 Subject: [PATCH 3/7] Bug Fix --- .github/MLOps_Engineer/Utilities/Bash/utilsSetEnvVariables.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/MLOps_Engineer/Utilities/Bash/utilsSetEnvVariables.sh b/.github/MLOps_Engineer/Utilities/Bash/utilsSetEnvVariables.sh index 2455334f..1b4b460e 100644 --- a/.github/MLOps_Engineer/Utilities/Bash/utilsSetEnvVariables.sh +++ b/.github/MLOps_Engineer/Utilities/Bash/utilsSetEnvVariables.sh @@ -15,7 +15,7 @@ DATABRICKS_ORDGID=$(az databricks workspace list -g $RESOURCE_GROUP_NAME --query DATABRICKS_INSTANCE="$(az databricks workspace list -g $RESOURCE_GROUP_NAME --query "[].workspaceUrl" -o tsv)" WORKSPACE_ID=$(az databricks workspace list -g $RESOURCE_GROUP_NAME --query "[].id" -o tsv) AZ_KEYVAULT_NAME=$(az keyvault list -g $RESOURCE_GROUP_NAME --query "[].name" -o tsv) -SUBSCRIPTION_ID=$( az account show --query id -o tsv ) +SUBSCRIPTION_ID=$( az account show --query id ) echo $SUBSCRIPTION_ID echo $DATABRICKS_ORDGID echo $WORKSPACE_ID From 940636aa0855dcab461df18d4dff7229422aa3aa Mon Sep 17 00:00:00 2001 From: ciaran28 Date: Tue, 31 Jan 2023 14:20:32 +0000 Subject: [PATCH 4/7] Bug Fix --- .../MLOps_Engineer/Utilities/Bash/utilsSetEnvVariables.sh | 2 +- .github/MLOps_Engineer/Utilities/Bash/utilsSetEnvVariables.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.azureDevOps/MLOps_Engineer/Utilities/Bash/utilsSetEnvVariables.sh b/.azureDevOps/MLOps_Engineer/Utilities/Bash/utilsSetEnvVariables.sh index 08c72ed7..4e046495 100644 --- a/.azureDevOps/MLOps_Engineer/Utilities/Bash/utilsSetEnvVariables.sh +++ b/.azureDevOps/MLOps_Engineer/Utilities/Bash/utilsSetEnvVariables.sh @@ -15,7 +15,7 @@ DATABRICKS_ORDGID=$(az databricks workspace list -g $RESOURCE_GROUP_NAME --query DATABRICKS_INSTANCE="$(az databricks workspace list -g $RESOURCE_GROUP_NAME --query "[].workspaceUrl" -o tsv)" WORKSPACE_ID=$(az databricks workspace list -g $RESOURCE_GROUP_NAME --query "[].id" -o tsv) AZ_KEYVAULT_NAME=$(az keyvault list -g $RESOURCE_GROUP_NAME --query "[].name" -o tsv) -SUBSCRIPTION_ID=$( az account show --query id ) +SUBSCRIPTION_ID=$( az account show --query id -o tsv ) #DATABRICKS_TOKEN=$(az keyvault secret show --name "dbkstoken" --vault-name $AZ_KEYVAULT_NAME --query "value" -o tsv) diff --git a/.github/MLOps_Engineer/Utilities/Bash/utilsSetEnvVariables.sh b/.github/MLOps_Engineer/Utilities/Bash/utilsSetEnvVariables.sh index 1b4b460e..2455334f 100644 --- a/.github/MLOps_Engineer/Utilities/Bash/utilsSetEnvVariables.sh +++ b/.github/MLOps_Engineer/Utilities/Bash/utilsSetEnvVariables.sh @@ -15,7 +15,7 @@ DATABRICKS_ORDGID=$(az databricks workspace list -g $RESOURCE_GROUP_NAME --query DATABRICKS_INSTANCE="$(az databricks workspace list -g $RESOURCE_GROUP_NAME --query "[].workspaceUrl" -o tsv)" WORKSPACE_ID=$(az databricks workspace list -g $RESOURCE_GROUP_NAME --query "[].id" -o tsv) AZ_KEYVAULT_NAME=$(az keyvault list -g $RESOURCE_GROUP_NAME --query "[].name" -o tsv) -SUBSCRIPTION_ID=$( az account show --query id ) +SUBSCRIPTION_ID=$( az account show --query id -o tsv ) echo $SUBSCRIPTION_ID echo $DATABRICKS_ORDGID echo $WORKSPACE_ID From 2e2d0a20e22d68fc9cd3db2dcdb4983aa0a55401 Mon Sep 17 00:00:00 2001 From: ciaran28 Date: Tue, 31 Jan 2023 14:26:39 +0000 Subject: [PATCH 5/7] Bug Fix --- .github/workflows/taskDatabricks.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/taskDatabricks.yaml b/.github/workflows/taskDatabricks.yaml index c4afdc54..1cb91e9d 100644 --- a/.github/workflows/taskDatabricks.yaml +++ b/.github/workflows/taskDatabricks.yaml @@ -72,10 +72,10 @@ jobs: #################################/ ## Deploy Azure Infrastructure #################################/ - #- name: Deploy DBX CICD Azure Resources - # run: bash .github/MLOps_Engineer/Utilities/Bash/utilsCreateAzureResources.sh - # env: - # ENVIRONMENT: ${{ inputs.ENVIRONMENT }} + - name: Deploy DBX CICD Azure Resources + run: bash .github/MLOps_Engineer/Utilities/Bash/utilsCreateAzureResources.sh + env: + ENVIRONMENT: ${{ inputs.ENVIRONMENT }} #################################/ From 23edd311bf0832e68e804d76748b5c6e317f2d93 Mon Sep 17 00:00:00 2001 From: ciaran28 Date: Wed, 1 Feb 2023 17:15:23 +0000 Subject: [PATCH 6/7] Updated Readme - Feature Store Info --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index 0bb82165..dd790b26 100644 --- a/README.md +++ b/README.md @@ -152,6 +152,20 @@ https://learn.microsoft.com/en-us/azure/databricks/machine-learning/mlops/deploy --- --- +## Feature Store Integration + +In an organization, thousands of features are buried in different scripts and in different formats; they are not captured, organized, or preserved, and thus cannot be reused and leveraged by teams other than those who generated them. + +Because feature engineering is so important for machine learning models and features cannot be shared, data scientists must duplicate their feature engineering efforts across teams. + +To solve those problems, a concept called feature store was developed, so that: + +- Features are centralized in an organization and can be reused +- Features can be served in real-time with low latency + +--- +--- + ## Create Repository
From f34d58e32feb0c45fc9c4d66b33312f3bc785917 Mon Sep 17 00:00:00 2001 From: Ciaran Hamill Diamond <108273509+ciaran28@users.noreply.github.com> Date: Wed, 1 Feb 2023 17:17:33 +0000 Subject: [PATCH 7/7] Update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index dd790b26..cbfb815f 100644 --- a/README.md +++ b/README.md @@ -163,6 +163,8 @@ To solve those problems, a concept called feature store was developed, so that: - Features are centralized in an organization and can be reused - Features can be served in real-time with low latency +![image](https://user-images.githubusercontent.com/108273509/216114586-0c4dea68-a98c-4cf6-938a-ceecf11b12a8.png) + --- ---