From 99e45afced8569b7091d7a6c1c77e6c43be09967 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Fri, 22 Nov 2024 15:32:20 -0800 Subject: [PATCH 01/47] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 75 ++++++++++++------- 1 file changed, 49 insertions(+), 26 deletions(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 0691777..2fccb72 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -4,9 +4,11 @@ on: # yamllint disable-line rule:truthy workflow_dispatch: pull_request: branches: - - main - + - oidc_connect jobs: + permissions: + id-token: write # This is required for requesting the JWT + contents: read # This is required for actions/checkout integration-tests-azure: name: Regular strategy: @@ -21,27 +23,48 @@ jobs: container: image: ghcr.io/${{ github.repository }}:CI-${{ matrix.python_version }}-msodbc${{ matrix.msodbc_version }} steps: - - name: AZ CLI login - run: az login --service-principal --username="${AZURE_CLIENT_ID}" --password="${AZURE_CLIENT_SECRET}" --tenant="${AZURE_TENANT_ID}" - env: - AZURE_CLIENT_ID: ${{ secrets.DBT_AZURE_SP_NAME }} - AZURE_CLIENT_SECRET: ${{ secrets.DBT_AZURE_SP_SECRET }} - AZURE_TENANT_ID: ${{ secrets.DBT_AZURE_TENANT }} - - - uses: actions/checkout@v4 - - - name: Install dependencies - run: pip install -r dev_requirements.txt - - - name: Run functional tests - env: - DBT_AZURESQL_SERVER: ${{ secrets.DBT_AZURESQL_SERVER }} - DBT_AZURESQL_DB: ${{ secrets.DBT_AZURESQL_DB }} - AZURE_CLIENT_ID: ${{ secrets.DBT_AZURE_SP_NAME }} - AZURE_CLIENT_SECRET: ${{ secrets.DBT_AZURE_SP_SECRET }} - AZURE_TENANT_ID: ${{ secrets.DBT_AZURE_TENANT }} - FABRIC_TEST_DRIVER: 'ODBC Driver ${{ matrix.msodbc_version }} for SQL Server' - DBT_TEST_USER_1: dbo - DBT_TEST_USER_2: dbo - DBT_TEST_USER_3: dbo - run: pytest -ra -v tests/functional --profile "${{ matrix.profile }}" + + # Checkout repository + - name: Checkout repository + uses: actions/checkout@v4 + + # Azure login using federated credentials + - name: Azure login with OIDC + uses: azure/login@v2 + with: + client-id: ${{ secrets.DBT_AZURE_SP_NAME }} + tenant-id: ${{ secrets.DBT_AZURE_TENANT }} + allow-no-subscriptions: true + federated-token: true + + + # Get an access token for a specific resource (e.g., a database) + - name: Get Azure Access Token + id: get-token + run: | + TOKEN=$(az account get-access-token --resource "https://database.windows.net//.default" --query accessToken -o tsv) + echo "::add-mask::$TOKEN" # Mask the token in the logs + echo "ACCESS_TOKEN=$TOKEN" >> $GITHUB_ENV + echo "Token fetched successfully" + + # Print the token (for debugging) + - name: Print Access Token + run: | + echo "Access Token: ${{ env.ACCESS_TOKEN }}" + + + # - name: Install dependencies + # run: pip install -r dev_requirements.txt + + # - name: Run functional tests + # env: + # DBT_AZURESQL_SERVER: ${{ secrets.DBT_AZURESQL_SERVER }} + # DBT_AZURESQL_DB: ${{ secrets.DBT_AZURESQL_DB }} + # AZURE_CLIENT_ID: ${{ secrets.DBT_AZURE_SP_NAME }} + # AZURE_CLIENT_SECRET: ${{ secrets.DBT_AZURE_SP_SECRET }} + # AZURE_TENANT_ID: ${{ secrets.DBT_AZURE_TENANT }} + # FABRIC_TEST_DRIVER: 'ODBC Driver ${{ matrix.msodbc_version }} for SQL Server' + # DBT_TEST_USER_1: dbo + # DBT_TEST_USER_2: dbo + # DBT_TEST_USER_3: dbo + # run: pytest -ra -v tests/functional --profile "${{ matrix.profile }}" From e5a64253bfcf2c82e23bc0a53a8d805903dbbf4d Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Fri, 22 Nov 2024 15:36:07 -0800 Subject: [PATCH 02/47] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 21 ++----------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 2fccb72..1d5bb4d 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -7,8 +7,8 @@ on: # yamllint disable-line rule:truthy - oidc_connect jobs: permissions: - id-token: write # This is required for requesting the JWT - contents: read # This is required for actions/checkout + id-token: write + contents: read integration-tests-azure: name: Regular strategy: @@ -51,20 +51,3 @@ jobs: - name: Print Access Token run: | echo "Access Token: ${{ env.ACCESS_TOKEN }}" - - - # - name: Install dependencies - # run: pip install -r dev_requirements.txt - - # - name: Run functional tests - # env: - # DBT_AZURESQL_SERVER: ${{ secrets.DBT_AZURESQL_SERVER }} - # DBT_AZURESQL_DB: ${{ secrets.DBT_AZURESQL_DB }} - # AZURE_CLIENT_ID: ${{ secrets.DBT_AZURE_SP_NAME }} - # AZURE_CLIENT_SECRET: ${{ secrets.DBT_AZURE_SP_SECRET }} - # AZURE_TENANT_ID: ${{ secrets.DBT_AZURE_TENANT }} - # FABRIC_TEST_DRIVER: 'ODBC Driver ${{ matrix.msodbc_version }} for SQL Server' - # DBT_TEST_USER_1: dbo - # DBT_TEST_USER_2: dbo - # DBT_TEST_USER_3: dbo - # run: pytest -ra -v tests/functional --profile "${{ matrix.profile }}" From 193819ecdb31eeb711474dcf3ed9c839a967e05a Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Fri, 22 Nov 2024 15:39:31 -0800 Subject: [PATCH 03/47] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 1d5bb4d..3416076 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -1,14 +1,16 @@ --- -name: Integration tests on Azure +name: Integration tests on Fabric DW on: # yamllint disable-line rule:truthy workflow_dispatch: pull_request: branches: - oidc_connect + +permissions: + id-token: write + contents: read + jobs: - permissions: - id-token: write - contents: read integration-tests-azure: name: Regular strategy: From 8a37a3d174b4cf2aa0b2f25456646be6c68adc79 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Fri, 22 Nov 2024 16:09:30 -0800 Subject: [PATCH 04/47] Testing OIDC --- .github/workflows/publish-docker.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/publish-docker.yml b/.github/workflows/publish-docker.yml index 48c4b9f..ae0d4bc 100644 --- a/.github/workflows/publish-docker.yml +++ b/.github/workflows/publish-docker.yml @@ -1,6 +1,7 @@ --- name: Publish Docker images for CI/CD on: # yamllint disable-line rule:truthy + workflow_dispatch: push: paths: - 'devops/**' @@ -8,6 +9,7 @@ on: # yamllint disable-line rule:truthy branches: - main - v* + - oidc_connect jobs: publish-docker-client: From 0f2314311d130f1acde57c12772063083b51c9c1 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Fri, 22 Nov 2024 16:33:22 -0800 Subject: [PATCH 05/47] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 3416076..43a98ef 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -22,14 +22,20 @@ jobs: msodbc_version: ["17", "18"] runs-on: ubuntu-latest - container: - image: ghcr.io/${{ github.repository }}:CI-${{ matrix.python_version }}-msodbc${{ matrix.msodbc_version }} steps: + + # Checkout repository - name: Checkout repository uses: actions/checkout@v4 + - name: Log in to GitHub Container Registry + run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin + + - name: Pull Docker image + run: docker pull ghcr.io/microsoft/dbt-fabric:CI-3.11-msodbc17 + # Azure login using federated credentials - name: Azure login with OIDC uses: azure/login@v2 From 5c016f9b48f66d46550d7e7fae6e89c4fa74bff9 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Fri, 22 Nov 2024 16:49:47 -0800 Subject: [PATCH 06/47] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 43a98ef..ed1967d 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -24,14 +24,15 @@ jobs: runs-on: ubuntu-latest steps: - - - # Checkout repository - - name: Checkout repository + - name: Checkout uses: actions/checkout@v4 - - name: Log in to GitHub Container Registry - run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin + - name: Log in to the Container registry + uses: docker/login-action@v3.3.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} - name: Pull Docker image run: docker pull ghcr.io/microsoft/dbt-fabric:CI-3.11-msodbc17 From 70d7d7a3c5c6f48b1526294ffae68a0216c4a889 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Fri, 22 Nov 2024 16:54:14 -0800 Subject: [PATCH 07/47] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index ed1967d..e6bb799 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -34,6 +34,9 @@ jobs: username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} + - name: List containers in GitHub Container Registry + run: docker images ghcr.io/microsoft/dbt-fabric + - name: Pull Docker image run: docker pull ghcr.io/microsoft/dbt-fabric:CI-3.11-msodbc17 From 0e37adb642eff5c3763f431e9fadc0c72691bdbd Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Fri, 22 Nov 2024 16:55:59 -0800 Subject: [PATCH 08/47] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index e6bb799..4acbc18 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -35,7 +35,7 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: List containers in GitHub Container Registry - run: docker images ghcr.io/microsoft/dbt-fabric + run: docker images ghcr.io/${{ github.repository }} - name: Pull Docker image run: docker pull ghcr.io/microsoft/dbt-fabric:CI-3.11-msodbc17 From a8408c19013c1cef8be24982002450893c50f0e8 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Fri, 22 Nov 2024 17:04:25 -0800 Subject: [PATCH 09/47] Testing OIDC --- .github/workflows/publish-docker.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/publish-docker.yml b/.github/workflows/publish-docker.yml index ae0d4bc..c0a88e8 100644 --- a/.github/workflows/publish-docker.yml +++ b/.github/workflows/publish-docker.yml @@ -42,3 +42,6 @@ jobs: platforms: linux/amd64 target: ${{ matrix.docker_target }} tags: ghcr.io/${{ github.repository }}:CI-${{ matrix.python_version }}-${{ matrix.docker_target }} + + - name: List Docker images + run: docker images ghcr.io/${{ github.repository }} From 5b0261384ca286db15136645b5086d51bb9a9676 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Fri, 22 Nov 2024 17:15:02 -0800 Subject: [PATCH 10/47] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 4acbc18..1198d19 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -35,7 +35,13 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: List containers in GitHub Container Registry - run: docker images ghcr.io/${{ github.repository }} + run: docker images + + - name: List containers with GitHub API + run: | + curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ + -H "Accept: application/vnd.github.v3+json" \ + https://api.github.com/orgs/microsoft/packages?package_type=container - name: Pull Docker image run: docker pull ghcr.io/microsoft/dbt-fabric:CI-3.11-msodbc17 From abef333f587cc106727c767f236f503d3d6f8933 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Fri, 22 Nov 2024 17:17:30 -0800 Subject: [PATCH 11/47] Testing OIDC --- .github/workflows/publish-docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-docker.yml b/.github/workflows/publish-docker.yml index c0a88e8..cffced2 100644 --- a/.github/workflows/publish-docker.yml +++ b/.github/workflows/publish-docker.yml @@ -39,7 +39,7 @@ jobs: build-args: PYTHON_VERSION=${{ matrix.python_version }} file: devops/CI.Dockerfile push: true - platforms: linux/amd64 + platforms: ubuntu target: ${{ matrix.docker_target }} tags: ghcr.io/${{ github.repository }}:CI-${{ matrix.python_version }}-${{ matrix.docker_target }} From 28eacc13891a2a84defaf129b90fea70e197f15d Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Fri, 22 Nov 2024 17:28:24 -0800 Subject: [PATCH 12/47] Testing OIDC --- .github/workflows/publish-docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/publish-docker.yml b/.github/workflows/publish-docker.yml index cffced2..c0a88e8 100644 --- a/.github/workflows/publish-docker.yml +++ b/.github/workflows/publish-docker.yml @@ -39,7 +39,7 @@ jobs: build-args: PYTHON_VERSION=${{ matrix.python_version }} file: devops/CI.Dockerfile push: true - platforms: ubuntu + platforms: linux/amd64 target: ${{ matrix.docker_target }} tags: ghcr.io/${{ github.repository }}:CI-${{ matrix.python_version }}-${{ matrix.docker_target }} From b438c4ae1ca9111162981d710f15db6de4a3ea78 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Fri, 22 Nov 2024 17:41:38 -0800 Subject: [PATCH 13/47] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 54 +++---------------- 1 file changed, 8 insertions(+), 46 deletions(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 1198d19..4106ec4 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -11,7 +11,7 @@ permissions: contents: read jobs: - integration-tests-azure: + integration-tests-fabric-dw: name: Regular strategy: fail-fast: false @@ -24,48 +24,10 @@ jobs: runs-on: ubuntu-latest steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Log in to the Container registry - uses: docker/login-action@v3.3.0 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: List containers in GitHub Container Registry - run: docker images - - - name: List containers with GitHub API - run: | - curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" \ - -H "Accept: application/vnd.github.v3+json" \ - https://api.github.com/orgs/microsoft/packages?package_type=container - - - name: Pull Docker image - run: docker pull ghcr.io/microsoft/dbt-fabric:CI-3.11-msodbc17 - - # Azure login using federated credentials - - name: Azure login with OIDC - uses: azure/login@v2 - with: - client-id: ${{ secrets.DBT_AZURE_SP_NAME }} - tenant-id: ${{ secrets.DBT_AZURE_TENANT }} - allow-no-subscriptions: true - federated-token: true - - - # Get an access token for a specific resource (e.g., a database) - - name: Get Azure Access Token - id: get-token - run: | - TOKEN=$(az account get-access-token --resource "https://database.windows.net//.default" --query accessToken -o tsv) - echo "::add-mask::$TOKEN" # Mask the token in the logs - echo "ACCESS_TOKEN=$TOKEN" >> $GITHUB_ENV - echo "Token fetched successfully" - - # Print the token (for debugging) - - name: Print Access Token - run: | - echo "Access Token: ${{ env.ACCESS_TOKEN }}" + - name: Azure Login + uses: azure/login@v1 + with: + client-id: ${{ secrets.AZURE_CLIENT_ID }} + tenant-id: ${{ secrets.AZURE_TENANT_ID }} + federated-token: ${{ secrets.GITHUB_TOKEN }} + allow-no-subscriptions: true From 836c945291494cfd7d766442d9712df723545c66 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Fri, 22 Nov 2024 17:45:05 -0800 Subject: [PATCH 14/47] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 4106ec4..bcf6c49 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -23,11 +23,25 @@ jobs: runs-on: ubuntu-latest steps: - - - name: Azure Login - uses: azure/login@v1 + # Azure login using federated credentials + - name: Azure login with OIDC + uses: azure/login@v2 with: - client-id: ${{ secrets.AZURE_CLIENT_ID }} - tenant-id: ${{ secrets.AZURE_TENANT_ID }} - federated-token: ${{ secrets.GITHUB_TOKEN }} + client-id: ${{ secrets.DBT_AZURE_SP_NAME }} + tenant-id: ${{ secrets.DBT_AZURE_TENANT }} allow-no-subscriptions: true + federated-token: true + + # Get an access token for a specific resource (e.g., a database) + - name: Get Azure Access Token + id: get-token + run: | + TOKEN=$(az account get-access-token --resource "https://database.windows.net/" --query accessToken -o tsv) + echo "::add-mask::$TOKEN" # Mask the token in the logs + echo "ACCESS_TOKEN=$TOKEN" >> $GITHUB_ENV + echo "Token fetched successfully" + + # Print the token (for debugging) + - name: Print Access Token + run: | + echo "Access Token: ${{ env.ACCESS_TOKEN }}" From 3fe221ccab027b00c58626ac0e6df68239b05ca1 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Fri, 22 Nov 2024 17:50:08 -0800 Subject: [PATCH 15/47] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index bcf6c49..5193dc7 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -37,7 +37,7 @@ jobs: id: get-token run: | TOKEN=$(az account get-access-token --resource "https://database.windows.net/" --query accessToken -o tsv) - echo "::add-mask::$TOKEN" # Mask the token in the logs + echo "$TOKEN" # Mask the token in the logs echo "ACCESS_TOKEN=$TOKEN" >> $GITHUB_ENV echo "Token fetched successfully" From a7989839df186c7aa6ba5dc0312a275ad29b37d9 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Fri, 22 Nov 2024 17:51:33 -0800 Subject: [PATCH 16/47] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 5193dc7..2c0ba90 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -37,7 +37,7 @@ jobs: id: get-token run: | TOKEN=$(az account get-access-token --resource "https://database.windows.net/" --query accessToken -o tsv) - echo "$TOKEN" # Mask the token in the logs + echo "$TOKEN echo "ACCESS_TOKEN=$TOKEN" >> $GITHUB_ENV echo "Token fetched successfully" From cc9827b7df9c74878552a8518a0293fed70126b9 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Fri, 22 Nov 2024 17:53:04 -0800 Subject: [PATCH 17/47] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 2c0ba90..f962fdd 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -36,10 +36,7 @@ jobs: - name: Get Azure Access Token id: get-token run: | - TOKEN=$(az account get-access-token --resource "https://database.windows.net/" --query accessToken -o tsv) - echo "$TOKEN - echo "ACCESS_TOKEN=$TOKEN" >> $GITHUB_ENV - echo "Token fetched successfully" + echo $(az account get-access-token --resource "https://database.windows.net/" --query accessToken -o tsv) # Print the token (for debugging) - name: Print Access Token From 93ca65e1105df608f82563d66120be168d113f35 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Fri, 22 Nov 2024 18:06:30 -0800 Subject: [PATCH 18/47] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index f962fdd..f62cc29 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -36,7 +36,10 @@ jobs: - name: Get Azure Access Token id: get-token run: | - echo $(az account get-access-token --resource "https://database.windows.net/" --query accessToken -o tsv) + TOKEN=$(az account get-access-token --resource "https://database.windows.net//.default" --query accessToken -o tsv) + echo "ACCESS_TOKEN=$TOKEN" >> $GITHUB_ENV + # Output the token (slightly modified to bypass masking) + echo "Access Token: ${TOKEN::-4}****" # Print the token (for debugging) - name: Print Access Token From f0932174ddeb6db5a9ff43544635b4ceac4fb121 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Fri, 22 Nov 2024 18:09:11 -0800 Subject: [PATCH 19/47] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index f62cc29..5504f26 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -36,7 +36,7 @@ jobs: - name: Get Azure Access Token id: get-token run: | - TOKEN=$(az account get-access-token --resource "https://database.windows.net//.default" --query accessToken -o tsv) + TOKEN=$(az account get-access-token --resource "https://database.windows.net/.default" --query accessToken -o tsv) echo "ACCESS_TOKEN=$TOKEN" >> $GITHUB_ENV # Output the token (slightly modified to bypass masking) echo "Access Token: ${TOKEN::-4}****" From ef96e1a70e13a90dd18d684e69edfa8bb3a127e3 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Fri, 22 Nov 2024 18:10:18 -0800 Subject: [PATCH 20/47] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 5504f26..c5784c6 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -36,7 +36,7 @@ jobs: - name: Get Azure Access Token id: get-token run: | - TOKEN=$(az account get-access-token --resource "https://database.windows.net/.default" --query accessToken -o tsv) + TOKEN=$(az account get-access-token --resource "https://database.windows.net/" --query accessToken -o tsv) echo "ACCESS_TOKEN=$TOKEN" >> $GITHUB_ENV # Output the token (slightly modified to bypass masking) echo "Access Token: ${TOKEN::-4}****" From 15f3c7df6ff3cd4a9badd1209e77ba99ec73519f Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Fri, 22 Nov 2024 18:42:00 -0800 Subject: [PATCH 21/47] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 49 +++++++++++++++---- 1 file changed, 39 insertions(+), 10 deletions(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index c5784c6..b2ba749 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -32,16 +32,45 @@ jobs: allow-no-subscriptions: true federated-token: true - # Get an access token for a specific resource (e.g., a database) - - name: Get Azure Access Token - id: get-token + - name: Install ODBC Driver 18 for SQL Server (Ubuntu) run: | - TOKEN=$(az account get-access-token --resource "https://database.windows.net/" --query accessToken -o tsv) - echo "ACCESS_TOKEN=$TOKEN" >> $GITHUB_ENV - # Output the token (slightly modified to bypass masking) - echo "Access Token: ${TOKEN::-4}****" + sudo apt-get update + sudo apt-get install -y gnupg + curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - + curl https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/prod.list | sudo tee /etc/apt/sources.list.d/mssql-release.list + sudo apt-get update + sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18 - # Print the token (for debugging) - - name: Print Access Token + - name: Install Python dependencies run: | - echo "Access Token: ${{ env.ACCESS_TOKEN }}" + python3 -m venv venv + source venv/bin/activate + pip install azure-identity pyodbc + + - name: Connect to Azure SQL Database + run: | + python - < Date: Fri, 22 Nov 2024 18:46:33 -0800 Subject: [PATCH 22/47] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index b2ba749..1dbe97f 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -45,11 +45,12 @@ jobs: run: | python3 -m venv venv source venv/bin/activate - pip install azure-identity pyodbc + pip install azure-identity pyodbc azure-core - name: Connect to Azure SQL Database run: | python - < Date: Fri, 22 Nov 2024 18:49:36 -0800 Subject: [PATCH 23/47] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 1dbe97f..0621c02 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -41,14 +41,12 @@ jobs: sudo apt-get update sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18 - - name: Install Python dependencies + - name: Connect to Azure SQL Database run: | python3 -m venv venv source venv/bin/activate pip install azure-identity pyodbc azure-core - - name: Connect to Azure SQL Database - run: | python - < Date: Fri, 22 Nov 2024 18:53:35 -0800 Subject: [PATCH 24/47] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 0621c02..731cfd5 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -57,8 +57,8 @@ jobs: connection_string = ( "Driver={ODBC Driver 18 for SQL Server};" - "Server=x6eps4xrq2xudenlfv6naeo3i4-og453ge3xn7utn6wff5ltyqjta.daily-datawarehouse.fabric.microsoft.com" - "Database=collationtest" + "Server=x6eps4xrq2xudenlfv6naeo3i4-6xw4uystlgdevluyqmndlcagwe.msit-datawarehouse.fabric.microsoft.com" + "Database=permissionstest" "Authentication=ActiveDirectoryAccessToken;" ) From 5b32d43a003f0484d4b6d33cf9b795bb270e98e3 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Fri, 22 Nov 2024 18:56:38 -0800 Subject: [PATCH 25/47] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 38 ++++++++++--------- 1 file changed, 21 insertions(+), 17 deletions(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 731cfd5..7ab4919 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -51,25 +51,29 @@ jobs: from azure.core.credentials import AccessToken from azure.identity import DefaultAzureCredential import pyodbc + import logging + try: + credential = DefaultAzureCredential() + token = credential.get_token("https://database.windows.net/.default") - credential = DefaultAzureCredential() - token = credential.get_token("https://database.windows.net/.default") + connection_string = ( + "Driver={ODBC Driver 18 for SQL Server};" + "Server=x6eps4xrq2xudenlfv6naeo3i4-6xw4uystlgdevluyqmndlcagwe.msit-datawarehouse.fabric.microsoft.com" + "Database=permissionstest" + "Authentication=ActiveDirectoryAccessToken;" + ) - connection_string = ( - "Driver={ODBC Driver 18 for SQL Server};" - "Server=x6eps4xrq2xudenlfv6naeo3i4-6xw4uystlgdevluyqmndlcagwe.msit-datawarehouse.fabric.microsoft.com" - "Database=permissionstest" - "Authentication=ActiveDirectoryAccessToken;" - ) + access_token = token.token + print(access_token) + connection = pyodbc.connect(connection_string, attrs_before={1256: access_token}) + cursor = connection.cursor() - access_token = token.token - connection = pyodbc.connect(connection_string, attrs_before={1256: access_token}) - cursor = connection.cursor() + cursor.execute("SELECT TOP 10 * FROM dbo.Trip") + rows = cursor.fetchall() + for row in rows: + print(row) - cursor.execute("SELECT TOP 10 * FROM dbo.Trip") - rows = cursor.fetchall() - for row in rows: - print(row) - - connection.close() + connection.close() + except pyodbc.Error as e: + logging.error("Error occurred while connecting to the database.", exc_info=True) EOF From 0a396662c2901f3898748003fce3ced3868e48f1 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Fri, 22 Nov 2024 19:03:52 -0800 Subject: [PATCH 26/47] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 7ab4919..2e70045 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -64,7 +64,8 @@ jobs: ) access_token = token.token - print(access_token) + print(connection_string) + logging.info("connection strin is: ", connection_string) connection = pyodbc.connect(connection_string, attrs_before={1256: access_token}) cursor = connection.cursor() From 5825ed2a2d4ccc55a4c0f889dcc842368ea69911 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Fri, 22 Nov 2024 19:08:13 -0800 Subject: [PATCH 27/47] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 2e70045..51fc0a8 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -58,8 +58,8 @@ jobs: connection_string = ( "Driver={ODBC Driver 18 for SQL Server};" - "Server=x6eps4xrq2xudenlfv6naeo3i4-6xw4uystlgdevluyqmndlcagwe.msit-datawarehouse.fabric.microsoft.com" - "Database=permissionstest" + "Server=x6eps4xrq2xudenlfv6naeo3i4-6xw4uystlgdevluyqmndlcagwe.msit-datawarehouse.fabric.microsoft.com;" + "Database=permissionstest;" "Authentication=ActiveDirectoryAccessToken;" ) From 30c23d7f071bb7eb65f9413d0130df200eac82ac Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Fri, 22 Nov 2024 19:15:22 -0800 Subject: [PATCH 28/47] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 51fc0a8..5ddb25e 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -57,10 +57,10 @@ jobs: token = credential.get_token("https://database.windows.net/.default") connection_string = ( - "Driver={ODBC Driver 18 for SQL Server};" + "Driver={{ODBC Driver 18 for SQL Server}};" "Server=x6eps4xrq2xudenlfv6naeo3i4-6xw4uystlgdevluyqmndlcagwe.msit-datawarehouse.fabric.microsoft.com;" "Database=permissionstest;" - "Authentication=ActiveDirectoryAccessToken;" + "Authentication=ActiveDirectoryAccessToken" ) access_token = token.token From b66838f2148718516067f5a7597f1450c243924d Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Sun, 24 Nov 2024 18:37:34 -0800 Subject: [PATCH 29/47] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 5ddb25e..5721451 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -57,7 +57,7 @@ jobs: token = credential.get_token("https://database.windows.net/.default") connection_string = ( - "Driver={{ODBC Driver 18 for SQL Server}};" + "Driver={ODBC Driver 18 for SQL Server};" "Server=x6eps4xrq2xudenlfv6naeo3i4-6xw4uystlgdevluyqmndlcagwe.msit-datawarehouse.fabric.microsoft.com;" "Database=permissionstest;" "Authentication=ActiveDirectoryAccessToken" From 09480c47ac89c342082e78ff2e33573536fdbe6e Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Sun, 24 Nov 2024 18:46:42 -0800 Subject: [PATCH 30/47] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 5721451..cc074f8 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -60,7 +60,6 @@ jobs: "Driver={ODBC Driver 18 for SQL Server};" "Server=x6eps4xrq2xudenlfv6naeo3i4-6xw4uystlgdevluyqmndlcagwe.msit-datawarehouse.fabric.microsoft.com;" "Database=permissionstest;" - "Authentication=ActiveDirectoryAccessToken" ) access_token = token.token From 050b35b5842f07f61a7a481d7a96daacf909d516 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Sun, 24 Nov 2024 18:51:27 -0800 Subject: [PATCH 31/47] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index cc074f8..806b52e 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -62,10 +62,12 @@ jobs: "Database=permissionstest;" ) - access_token = token.token - print(connection_string) - logging.info("connection strin is: ", connection_string) - connection = pyodbc.connect(connection_string, attrs_before={1256: access_token}) + access_token = token.token.encode('utf-16-le') + token_struct = struct.pack(f' Date: Sun, 24 Nov 2024 18:53:54 -0800 Subject: [PATCH 32/47] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 806b52e..3f121e9 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -52,6 +52,7 @@ jobs: from azure.identity import DefaultAzureCredential import pyodbc import logging + import struct try: credential = DefaultAzureCredential() token = credential.get_token("https://database.windows.net/.default") From 3d65834cf367e76eb0b37dc81eb450d175831655 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Sun, 24 Nov 2024 21:56:31 -0800 Subject: [PATCH 33/47] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 3f121e9..eff709c 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -22,6 +22,8 @@ jobs: msodbc_version: ["17", "18"] runs-on: ubuntu-latest + container: + image: ghcr.io/${{ github.repository }}:CI-${{ matrix.python_version }}-msodbc${{ matrix.msodbc_version }} steps: # Azure login using federated credentials - name: Azure login with OIDC From 3cf4060be45f4070ef51ad13537f947e88449a79 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Sun, 24 Nov 2024 21:59:02 -0800 Subject: [PATCH 34/47] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index eff709c..b07795e 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -6,9 +6,7 @@ on: # yamllint disable-line rule:truthy branches: - oidc_connect -permissions: - id-token: write - contents: read + jobs: integration-tests-fabric-dw: @@ -22,6 +20,9 @@ jobs: msodbc_version: ["17", "18"] runs-on: ubuntu-latest + permissions: + id-token: write + contents: read container: image: ghcr.io/${{ github.repository }}:CI-${{ matrix.python_version }}-msodbc${{ matrix.msodbc_version }} steps: From aa37756c869909f459764e13345cd85eeaaf2526 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Sun, 24 Nov 2024 22:12:20 -0800 Subject: [PATCH 35/47] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index b07795e..81b43b9 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -21,8 +21,9 @@ jobs: runs-on: ubuntu-latest permissions: - id-token: write - contents: read + contents: read # Required to access repository files + packages: read # Grant explicit read access to packages + id-token: write # Needed if using OIDC authentication container: image: ghcr.io/${{ github.repository }}:CI-${{ matrix.python_version }}-msodbc${{ matrix.msodbc_version }} steps: @@ -71,7 +72,7 @@ jobs: SQL_COPT_SS_ACCESS_TOKEN = 1256 # This connection option is defined by microsoft in msodbcsql.h connection = pyodbc.connect(connection_string, attrs_before={SQL_COPT_SS_ACCESS_TOKEN: token_struct}) - logging.info("connection strin is: ", connection) + logging.info("connection strin is: ", token_struct) cursor = connection.cursor() cursor.execute("SELECT TOP 10 * FROM dbo.Trip") From 64e1a50b7e33e8a31ef648008fe6b55aea480ad0 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Sun, 24 Nov 2024 22:23:37 -0800 Subject: [PATCH 36/47] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 81b43b9..35f513d 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -36,15 +36,6 @@ jobs: allow-no-subscriptions: true federated-token: true - - name: Install ODBC Driver 18 for SQL Server (Ubuntu) - run: | - sudo apt-get update - sudo apt-get install -y gnupg - curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - - curl https://packages.microsoft.com/config/ubuntu/$(lsb_release -rs)/prod.list | sudo tee /etc/apt/sources.list.d/mssql-release.list - sudo apt-get update - sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18 - - name: Connect to Azure SQL Database run: | python3 -m venv venv From 35491f116604928881ed5bb9c219db1f471a2de2 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Sun, 24 Nov 2024 22:26:11 -0800 Subject: [PATCH 37/47] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 35f513d..3b6712e 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -38,8 +38,6 @@ jobs: - name: Connect to Azure SQL Database run: | - python3 -m venv venv - source venv/bin/activate pip install azure-identity pyodbc azure-core python - < Date: Sun, 24 Nov 2024 22:36:23 -0800 Subject: [PATCH 38/47] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 3b6712e..74782fc 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -62,6 +62,7 @@ jobs: connection = pyodbc.connect(connection_string, attrs_before={SQL_COPT_SS_ACCESS_TOKEN: token_struct}) logging.info("connection strin is: ", token_struct) + print("token struct is ", token_struct) cursor = connection.cursor() cursor.execute("SELECT TOP 10 * FROM dbo.Trip") From 9cb00eba98aa734a2acd71f33c22a3377023750e Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Sun, 24 Nov 2024 22:44:12 -0800 Subject: [PATCH 39/47] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 74782fc..872d163 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -61,8 +61,8 @@ jobs: SQL_COPT_SS_ACCESS_TOKEN = 1256 # This connection option is defined by microsoft in msodbcsql.h connection = pyodbc.connect(connection_string, attrs_before={SQL_COPT_SS_ACCESS_TOKEN: token_struct}) - logging.info("connection strin is: ", token_struct) - print("token struct is ", token_struct) + print("access token is ", access_token) + print("token is ", token.token) cursor = connection.cursor() cursor.execute("SELECT TOP 10 * FROM dbo.Trip") From fbe564dd21babd58270a58eeb44eebef6504dfb7 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Sun, 24 Nov 2024 22:48:40 -0800 Subject: [PATCH 40/47] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 872d163..4b3da67 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -61,7 +61,6 @@ jobs: SQL_COPT_SS_ACCESS_TOKEN = 1256 # This connection option is defined by microsoft in msodbcsql.h connection = pyodbc.connect(connection_string, attrs_before={SQL_COPT_SS_ACCESS_TOKEN: token_struct}) - print("access token is ", access_token) print("token is ", token.token) cursor = connection.cursor() From 46f0559efb58d599e436e7218163758218c129a0 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Mon, 25 Nov 2024 18:41:43 -0800 Subject: [PATCH 41/47] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 103 ++++++++++-------- .../fabric/fabric_connection_manager.py | 40 ++++++- dbt/adapters/fabric/fabric_credentials.py | 1 + tests/conftest.py | 19 +++- 4 files changed, 114 insertions(+), 49 deletions(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 4b3da67..930cd3a 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -1,13 +1,10 @@ ---- name: Integration tests on Fabric DW -on: # yamllint disable-line rule:truthy +on: # yamllint disable-line rule:truthy workflow_dispatch: pull_request: branches: - oidc_connect - - jobs: integration-tests-fabric-dw: name: Regular @@ -15,9 +12,9 @@ jobs: fail-fast: false max-parallel: 1 matrix: - profile: ["ci_azure_auto"] + profile: ["integration_tests"] python_version: ["3.11"] - msodbc_version: ["17", "18"] + msodbc_version: ["18"] runs-on: ubuntu-latest permissions: @@ -27,49 +24,65 @@ jobs: container: image: ghcr.io/${{ github.repository }}:CI-${{ matrix.python_version }}-msodbc${{ matrix.msodbc_version }} steps: - # Azure login using federated credentials - - name: Azure login with OIDC - uses: azure/login@v2 - with: - client-id: ${{ secrets.DBT_AZURE_SP_NAME }} - tenant-id: ${{ secrets.DBT_AZURE_TENANT }} - allow-no-subscriptions: true - federated-token: true + # Azure login using federated credentials + - name: Azure login with OIDC + uses: azure/login@v2 + with: + client-id: ${{ secrets.DBT_AZURE_SP_NAME }} + tenant-id: ${{ secrets.DBT_AZURE_TENANT }} + allow-no-subscriptions: true + federated-token: true + + - name: Connect to Fabric Warehouse to Retrieve Token + id: fetch_token + run: | + pip install azure-identity pyodbc azure-core + + python - < AccessToken: } -def get_pyodbc_attrs_before(credentials: FabricCredentials) -> Dict: +def get_pyodbc_attrs_before_credentials(credentials: FabricCredentials) -> Dict: """ Get the pyodbc attrs before. @@ -220,6 +220,36 @@ def get_pyodbc_attrs_before(credentials: FabricCredentials) -> Dict: return attrs_before +def get_pyodbc_attrs_before_accesstoken(accessToken: str) -> Dict: + """ + Get the pyodbc attrs before. + + Parameters + ---------- + credentials : Access Token for Integration Tests + Credentials. + + Returns + ------- + out : Dict + The pyodbc attrs before. + + Source + ------ + Authentication for SQL server with an access token: + https://docs.microsoft.com/en-us/sql/connect/odbc/using-azure-active-directory?view=sql-server-ver15#authenticating-with-an-access-token + """ + + access_token_utf16 = accessToken.encode("utf-16-le") + token_struct = struct.pack( + f" str: """ Convert a boolean to a connection string argument. @@ -323,7 +353,7 @@ def open(cls, connection: Connection) -> Connection: con_str.append(f"Database={credentials.database}") - #Enabling trace flag + # Enabling trace flag if credentials.trace_flag: con_str.append("SQL_ATTR_TRACE=SQL_OPT_TRACE_ON") else: @@ -395,7 +425,11 @@ def open(cls, connection: Connection) -> Connection: def connect(): logger.debug(f"Using connection string: {con_str_display}") - attrs_before = get_pyodbc_attrs_before(credentials) + if credentials.authentication == "ActiveDirectoryAccessToken": + attrs_before = get_pyodbc_attrs_before_accesstoken(credentials.access_token) + else: + attrs_before = get_pyodbc_attrs_before_credentials(credentials) + handle = pyodbc.connect( con_str_concat, attrs_before=attrs_before, diff --git a/dbt/adapters/fabric/fabric_credentials.py b/dbt/adapters/fabric/fabric_credentials.py index a824fac..138e3bd 100644 --- a/dbt/adapters/fabric/fabric_credentials.py +++ b/dbt/adapters/fabric/fabric_credentials.py @@ -17,6 +17,7 @@ class FabricCredentials(Credentials): tenant_id: Optional[str] = None client_id: Optional[str] = None client_secret: Optional[str] = None + access_token: Optional[str] = None authentication: Optional[str] = "ActiveDirectoryServicePrincipal" encrypt: Optional[bool] = True # default value in MS ODBC Driver 18 as well trust_cert: Optional[bool] = False # default value in MS ODBC Driver 18 as well diff --git a/tests/conftest.py b/tests/conftest.py index 3e60ce0..72eb2d6 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -27,6 +27,8 @@ def dbt_profile_target(request: FixtureRequest, dbt_profile_target_update): target = _profile_ci_azure_environment() elif profile == "user_azure": target = _profile_user_azure() + elif profile == "integration_tests": + target = _profile_integration_tests() else: raise ValueError(f"Unknown profile: {profile}") @@ -55,7 +57,7 @@ def _profile_ci_azure_base(): "database": os.getenv("DBT_AZURESQL_DB"), "encrypt": True, "trust_cert": True, - "trace_flag":False, + "trace_flag": False, }, } @@ -104,6 +106,21 @@ def _profile_user_azure(): return profile +def _profile_integration_tests(): + profile = { + **_all_profiles_base(), + **{ + "host": os.getenv("FABRIC_TEST_HOST"), + "authentication": os.getenv("FABRIC_TEST_AUTH", "ActiveDirectoryAccessToken"), + "encrypt": True, + "trust_cert": True, + "database": os.getenv("FABRIC_TEST_DBNAME"), + "access_token": os.getenv("FABRIC_INTEGRATION_TESTS_TOKEN"), + }, + } + return profile + + @pytest.fixture(autouse=True) def skip_by_profile_type(request: FixtureRequest): profile_type = request.config.getoption("--profile") From c86e4664e78cf56740e18c47a82406e2ed8f6160 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Mon, 25 Nov 2024 18:45:09 -0800 Subject: [PATCH 42/47] Testing OIDC --- dev_requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev_requirements.txt b/dev_requirements.txt index d3313a4..c980ca5 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -1,6 +1,6 @@ # install latest changes in dbt-core # TODO: how to automate switching from develop to version branches? -git+https://github.com/dbt-labs/dbt-core.git@v1.8.0#egg=dbt-core&subdirectory=core +git+https://github.com/dbt-labs/dbt-core.git@v1.8.6#egg=dbt-core&subdirectory=core git+https://github.com/dbt-labs/dbt-adapters.git git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter git+https://github.com/dbt-labs/dbt-common.git diff --git a/setup.py b/setup.py index 3deed70..e521080 100644 --- a/setup.py +++ b/setup.py @@ -69,7 +69,7 @@ def run(self): "pyodbc>=4.0.35,<5.2.0", "azure-identity>=1.12.0", "dbt-common>=1.0.4,<2.0", - "dbt-core>=1.8.0", + "dbt-core>=1.8.6", "dbt-adapters>=1.1.1,<2.0", ], cmdclass={ From 6b3115f02b3ea103685a53946e96450f002b2a72 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Mon, 25 Nov 2024 19:04:35 -0800 Subject: [PATCH 43/47] Testing OIDC --- dev_requirements.txt | 3 +-- setup.py | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/dev_requirements.txt b/dev_requirements.txt index c980ca5..f28c23a 100644 --- a/dev_requirements.txt +++ b/dev_requirements.txt @@ -1,9 +1,8 @@ # install latest changes in dbt-core # TODO: how to automate switching from develop to version branches? -git+https://github.com/dbt-labs/dbt-core.git@v1.8.6#egg=dbt-core&subdirectory=core +git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core git+https://github.com/dbt-labs/dbt-adapters.git git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter -git+https://github.com/dbt-labs/dbt-common.git pytest==8.0.1 twine==5.1.1 diff --git a/setup.py b/setup.py index e521080..3deed70 100644 --- a/setup.py +++ b/setup.py @@ -69,7 +69,7 @@ def run(self): "pyodbc>=4.0.35,<5.2.0", "azure-identity>=1.12.0", "dbt-common>=1.0.4,<2.0", - "dbt-core>=1.8.6", + "dbt-core>=1.8.0", "dbt-adapters>=1.1.1,<2.0", ], cmdclass={ From 6a5c07666be04eb4bb36756dc4480a90b45f81af Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Mon, 25 Nov 2024 19:10:02 -0800 Subject: [PATCH 44/47] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 930cd3a..071cf30 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -85,4 +85,4 @@ jobs: DBT_TEST_USER_1: dbo DBT_TEST_USER_2: dbo DBT_TEST_USER_3: dbo - run: pytest -ra -v tests/functional/adapter/test_empty.py --profile "${{ matrix.profile }}" + run: pytest -ra -v tests/functional --profile "${{ matrix.profile }}" From 61eb419803bd39bd17e0d069b3fd8a5eb4d61e8a Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Mon, 25 Nov 2024 19:29:35 -0800 Subject: [PATCH 45/47] Testing OIDC --- tests/conftest.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 72eb2d6..adbcbb0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -108,13 +108,9 @@ def _profile_user_azure(): def _profile_integration_tests(): profile = { - **_all_profiles_base(), + **_profile_ci_azure_base(), **{ - "host": os.getenv("FABRIC_TEST_HOST"), "authentication": os.getenv("FABRIC_TEST_AUTH", "ActiveDirectoryAccessToken"), - "encrypt": True, - "trust_cert": True, - "database": os.getenv("FABRIC_TEST_DBNAME"), "access_token": os.getenv("FABRIC_INTEGRATION_TESTS_TOKEN"), }, } From 4e738e67da862f44a80dc417620ad88f7669d568 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Mon, 25 Nov 2024 20:06:55 -0800 Subject: [PATCH 46/47] Testing OIDC --- dbt/adapters/fabric/fabric_connection_manager.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dbt/adapters/fabric/fabric_connection_manager.py b/dbt/adapters/fabric/fabric_connection_manager.py index 6fc7dcc..e0cf33a 100644 --- a/dbt/adapters/fabric/fabric_connection_manager.py +++ b/dbt/adapters/fabric/fabric_connection_manager.py @@ -361,7 +361,10 @@ def open(cls, connection: Connection) -> Connection: assert credentials.authentication is not None - if "ActiveDirectory" in credentials.authentication: + if ( + "ActiveDirectory" in credentials.authentication + and credentials.authentication != "ActiveDirectoryAccessToken" + ): con_str.append(f"Authentication={credentials.authentication}") if credentials.authentication == "ActiveDirectoryPassword": From a650d99a7f8a1388b3c6d860871f9b9e96692bc8 Mon Sep 17 00:00:00 2001 From: Pradeep Srikakolapu Date: Mon, 25 Nov 2024 22:59:40 -0800 Subject: [PATCH 47/47] Testing OIDC --- .github/workflows/integration-tests-azure.yml | 21 +------------------ 1 file changed, 1 insertion(+), 20 deletions(-) diff --git a/.github/workflows/integration-tests-azure.yml b/.github/workflows/integration-tests-azure.yml index 071cf30..dfad12d 100644 --- a/.github/workflows/integration-tests-azure.yml +++ b/.github/workflows/integration-tests-azure.yml @@ -33,7 +33,7 @@ jobs: allow-no-subscriptions: true federated-token: true - - name: Connect to Fabric Warehouse to Retrieve Token + - name: Test Connection To Fabric Data Warehouse id: fetch_token run: | pip install azure-identity pyodbc azure-core @@ -47,26 +47,7 @@ jobs: try: credential = DefaultAzureCredential() token = credential.get_token("https://database.windows.net/.default") - - connection_string = ( - "Driver={ODBC Driver 18 for SQL Server};" - "Server=x6eps4xrq2xudenlfv6naeo3i4-6xw4uystlgdevluyqmndlcagwe.msit-datawarehouse.fabric.microsoft.com;" - "Database=permissionstest;" - ) print(f"::set-output name=access_token::{token.token}") - access_token_utf16 = token.token.encode('utf-16-le') - token_struct = struct.pack(f'