From cee8aaaacd939ee448c5ea835bb5d88cc22f3f6d Mon Sep 17 00:00:00 2001 From: Andy Smith Date: Wed, 21 Jul 2021 22:40:57 +0100 Subject: [PATCH 01/12] fix broken quotes --- .github/workflows/new_airflow_environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/new_airflow_environment.yml b/.github/workflows/new_airflow_environment.yml index 40fb053..9de2456 100644 --- a/.github/workflows/new_airflow_environment.yml +++ b/.github/workflows/new_airflow_environment.yml @@ -26,7 +26,7 @@ jobs: id: dag_path run: | dagpath=$(gcloud composer environments describe --location=europe-west2 --project=datapipeline-295515 ${{ github.event.inputs.newComposerEnvName }} --flatten=config.dagGcsPrefix --format=object) - echo ""::debug::$dagpath" + echo "::debug::$dagpath" echo "::set-output name=dag_url::$dagpath" - name: Get URL of new DAG bucket root id: bucket_root From 61cc5ad43932faba4b9e8c9d9660e7b10827fff1 Mon Sep 17 00:00:00 2001 From: Andy Smith Date: Wed, 21 Jul 2021 22:54:07 +0100 Subject: [PATCH 02/12] test local variable syntax --- .github/workflows/new_airflow_environment.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/new_airflow_environment.yml b/.github/workflows/new_airflow_environment.yml index 9de2456..a05851d 100644 --- a/.github/workflows/new_airflow_environment.yml +++ b/.github/workflows/new_airflow_environment.yml @@ -26,7 +26,9 @@ jobs: id: dag_path run: | dagpath=$(gcloud composer environments describe --location=europe-west2 --project=datapipeline-295515 ${{ github.event.inputs.newComposerEnvName }} --flatten=config.dagGcsPrefix --format=object) - echo "::debug::$dagpath" + echo "::debug::${{ dagpath }}" + echo "::debug::${{ $dagpath }}" + echo "::debug::${dagpath}" echo "::set-output name=dag_url::$dagpath" - name: Get URL of new DAG bucket root id: bucket_root From f90f6e02269f744b7c1abac56d9cae4a7e450938 Mon Sep 17 00:00:00 2001 From: Andy Smith Date: Wed, 21 Jul 2021 22:56:46 +0100 Subject: [PATCH 03/12] Update new_airflow_environment.yml --- .github/workflows/new_airflow_environment.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/new_airflow_environment.yml b/.github/workflows/new_airflow_environment.yml index a05851d..0de5386 100644 --- a/.github/workflows/new_airflow_environment.yml +++ b/.github/workflows/new_airflow_environment.yml @@ -27,9 +27,9 @@ jobs: run: | dagpath=$(gcloud composer environments describe --location=europe-west2 --project=datapipeline-295515 ${{ github.event.inputs.newComposerEnvName }} --flatten=config.dagGcsPrefix --format=object) echo "::debug::${{ dagpath }}" - echo "::debug::${{ $dagpath }}" - echo "::debug::${dagpath}" echo "::set-output name=dag_url::$dagpath" + # echo "::debug::${{ $dagpath }}" + # echo "::debug::${dagpath}" - name: Get URL of new DAG bucket root id: bucket_root run: | From e0d93a274a23402fb0a1fae85cf17db9b850f3a5 Mon Sep 17 00:00:00 2001 From: Andy Smith Date: Wed, 21 Jul 2021 22:59:00 +0100 Subject: [PATCH 04/12] Update new_airflow_environment.yml --- .github/workflows/new_airflow_environment.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/new_airflow_environment.yml b/.github/workflows/new_airflow_environment.yml index 0de5386..ef404a9 100644 --- a/.github/workflows/new_airflow_environment.yml +++ b/.github/workflows/new_airflow_environment.yml @@ -26,9 +26,9 @@ jobs: id: dag_path run: | dagpath=$(gcloud composer environments describe --location=europe-west2 --project=datapipeline-295515 ${{ github.event.inputs.newComposerEnvName }} --flatten=config.dagGcsPrefix --format=object) - echo "::debug::${{ dagpath }}" + echo "::debug::${{ $dagpath }}" echo "::set-output name=dag_url::$dagpath" - # echo "::debug::${{ $dagpath }}" + # echo "::debug::${{ dagpath }}" # echo "::debug::${dagpath}" - name: Get URL of new DAG bucket root id: bucket_root From 8a640c67986de55982f83fb9f27dca20c0856d4b Mon Sep 17 00:00:00 2001 From: Andy Smith Date: Wed, 21 Jul 2021 23:00:49 +0100 Subject: [PATCH 05/12] Update new_airflow_environment.yml --- .github/workflows/new_airflow_environment.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/new_airflow_environment.yml b/.github/workflows/new_airflow_environment.yml index ef404a9..ab1246e 100644 --- a/.github/workflows/new_airflow_environment.yml +++ b/.github/workflows/new_airflow_environment.yml @@ -26,10 +26,11 @@ jobs: id: dag_path run: | dagpath=$(gcloud composer environments describe --location=europe-west2 --project=datapipeline-295515 ${{ github.event.inputs.newComposerEnvName }} --flatten=config.dagGcsPrefix --format=object) - echo "::debug::${{ $dagpath }}" + echo "::debug::${dagpath}" echo "::set-output name=dag_url::$dagpath" + # Did not work # echo "::debug::${{ dagpath }}" - # echo "::debug::${dagpath}" + # echo "::debug::${{ $dagpath }}" - name: Get URL of new DAG bucket root id: bucket_root run: | From ba89116f58224964edce218063fc952835128d3f Mon Sep 17 00:00:00 2001 From: Andy Smith Date: Wed, 21 Jul 2021 23:06:20 +0100 Subject: [PATCH 06/12] fix syntax for local vars --- .github/workflows/new_airflow_environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/new_airflow_environment.yml b/.github/workflows/new_airflow_environment.yml index ab1246e..8d0bc9d 100644 --- a/.github/workflows/new_airflow_environment.yml +++ b/.github/workflows/new_airflow_environment.yml @@ -35,7 +35,7 @@ jobs: id: bucket_root run: | url=$(python -c "import re; print(re.search('(gs:.+\/)(dags$)', r'${{ steps.dag_path.outputs.dag_url }}').group(1))") - echo "::set-output name=bucket_url::$url" + echo "::set-output name=bucket_url::${url}" - name: Update DAG bucket secret run: echo ${{ steps.bucket_root.outputs.dag_url }} # uses: hmanzur/actions-set-secret@v2.0.0 From 837f760c8b933723ef41874d53738b5d54c5062e Mon Sep 17 00:00:00 2001 From: Andy Smith Date: Wed, 21 Jul 2021 23:10:58 +0100 Subject: [PATCH 07/12] consistent var names! --- .github/workflows/new_airflow_environment.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/new_airflow_environment.yml b/.github/workflows/new_airflow_environment.yml index 8d0bc9d..43a160c 100644 --- a/.github/workflows/new_airflow_environment.yml +++ b/.github/workflows/new_airflow_environment.yml @@ -37,10 +37,10 @@ jobs: url=$(python -c "import re; print(re.search('(gs:.+\/)(dags$)', r'${{ steps.dag_path.outputs.dag_url }}').group(1))") echo "::set-output name=bucket_url::${url}" - name: Update DAG bucket secret - run: echo ${{ steps.bucket_root.outputs.dag_url }} + run: echo ${{ steps.bucket_root.outputs.bucket_url }} # uses: hmanzur/actions-set-secret@v2.0.0 # with: # name: 'GCLOUD_BUCKET_PATH' - # value: ${{ steps.bucket_root.outputs.dag_url }} + # value: ${{ steps.bucket_root.outputs.bucket_url }} # repository: ${{ env.GITHUB_REPOSITORY }} # token: ${{ secrets.REPO_ACCESS_TOKEN }} From 4848ec5530eea753c2d61b014f5bc01ae7b941ec Mon Sep 17 00:00:00 2001 From: Andy Smith Date: Wed, 21 Jul 2021 23:18:00 +0100 Subject: [PATCH 08/12] Update new_airflow_environment.yml --- .github/workflows/new_airflow_environment.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/new_airflow_environment.yml b/.github/workflows/new_airflow_environment.yml index 43a160c..91b261c 100644 --- a/.github/workflows/new_airflow_environment.yml +++ b/.github/workflows/new_airflow_environment.yml @@ -1,4 +1,4 @@ -name: New_Airflow_Environment +name: Create New GCP Composer Environment (Airflow instance) on: workflow_dispatch: @@ -27,10 +27,7 @@ jobs: run: | dagpath=$(gcloud composer environments describe --location=europe-west2 --project=datapipeline-295515 ${{ github.event.inputs.newComposerEnvName }} --flatten=config.dagGcsPrefix --format=object) echo "::debug::${dagpath}" - echo "::set-output name=dag_url::$dagpath" - # Did not work - # echo "::debug::${{ dagpath }}" - # echo "::debug::${{ $dagpath }}" + echo "::set-output name=dag_url::${dagpath}" - name: Get URL of new DAG bucket root id: bucket_root run: | From 112eee62bcc36708631180d4f6e59053d600a5b7 Mon Sep 17 00:00:00 2001 From: Andy Smith Date: Wed, 21 Jul 2021 23:32:29 +0100 Subject: [PATCH 09/12] Switch to really creating new env (not just query existing ones) --- .github/workflows/new_airflow_environment.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/new_airflow_environment.yml b/.github/workflows/new_airflow_environment.yml index 91b261c..75ece48 100644 --- a/.github/workflows/new_airflow_environment.yml +++ b/.github/workflows/new_airflow_environment.yml @@ -20,8 +20,8 @@ jobs: project_id: ${{ secrets.GCS_PROJECT }} export_default_credentials: true - name: Create New Environment - # run: gcloud composer environments create ${{ github.event.inputs.newComposerEnvName }} --location=europe-west2 --project=datapipeline-295515 --machine-type=n1-standard-2 - run: gcloud composer environments describe ${{ github.event.inputs.newComposerEnvName }} --location=europe-west2 --project=datapipeline-295515 + run: gcloud composer environments create ${{ github.event.inputs.newComposerEnvName }} --location=europe-west2 --project=datapipeline-295515 --machine-type=n1-standard-2 + # run: gcloud composer environments describe ${{ github.event.inputs.newComposerEnvName }} --location=europe-west2 --project=datapipeline-295515 - name: Get URL of new DAG bucket full path id: dag_path run: | From f27dc33ef6397ad2718a6a0f272145c47cf1eeee Mon Sep 17 00:00:00 2001 From: Andy Smith Date: Wed, 21 Jul 2021 23:58:43 +0100 Subject: [PATCH 10/12] Add `zone` switch to create composer cmd --- .github/workflows/new_airflow_environment.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/new_airflow_environment.yml b/.github/workflows/new_airflow_environment.yml index 75ece48..277f824 100644 --- a/.github/workflows/new_airflow_environment.yml +++ b/.github/workflows/new_airflow_environment.yml @@ -20,7 +20,7 @@ jobs: project_id: ${{ secrets.GCS_PROJECT }} export_default_credentials: true - name: Create New Environment - run: gcloud composer environments create ${{ github.event.inputs.newComposerEnvName }} --location=europe-west2 --project=datapipeline-295515 --machine-type=n1-standard-2 + run: gcloud composer environments create ${{ github.event.inputs.newComposerEnvName }} --location=europe-west2 --project=datapipeline-295515 --machine-type=n1-standard-2 --zone=europe-west2-a # run: gcloud composer environments describe ${{ github.event.inputs.newComposerEnvName }} --location=europe-west2 --project=datapipeline-295515 - name: Get URL of new DAG bucket full path id: dag_path From 3122ee690555367f57685c6f494332ebcf03967a Mon Sep 17 00:00:00 2001 From: Andy Smith Date: Thu, 22 Jul 2021 22:25:57 +0100 Subject: [PATCH 11/12] Include env vars in composer create cmd --- .github/workflows/new_airflow_environment.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/new_airflow_environment.yml b/.github/workflows/new_airflow_environment.yml index 277f824..0f532bf 100644 --- a/.github/workflows/new_airflow_environment.yml +++ b/.github/workflows/new_airflow_environment.yml @@ -20,18 +20,18 @@ jobs: project_id: ${{ secrets.GCS_PROJECT }} export_default_credentials: true - name: Create New Environment - run: gcloud composer environments create ${{ github.event.inputs.newComposerEnvName }} --location=europe-west2 --project=datapipeline-295515 --machine-type=n1-standard-2 --zone=europe-west2-a + run: gcloud composer environments create ${{ github.event.inputs.newComposerEnvName }} --location=europe-west2 --project ${{ secrets.GCLOUD_PROJECT }} --machine-type=n1-standard-2 --zone=europe-west2-a --env-variables ENVIRONMENT=PRODUCTION --env-variables GCP=TRUE # run: gcloud composer environments describe ${{ github.event.inputs.newComposerEnvName }} --location=europe-west2 --project=datapipeline-295515 - name: Get URL of new DAG bucket full path id: dag_path run: | - dagpath=$(gcloud composer environments describe --location=europe-west2 --project=datapipeline-295515 ${{ github.event.inputs.newComposerEnvName }} --flatten=config.dagGcsPrefix --format=object) + dagpath=$(gcloud composer environments describe ${{ github.event.inputs.newComposerEnvName }} --location=europe-west2 --project= ${{ secrets.GCLOUD_PROJECT }} --flatten=config.dagGcsPrefix --format=object) echo "::debug::${dagpath}" echo "::set-output name=dag_url::${dagpath}" - name: Get URL of new DAG bucket root id: bucket_root run: | - url=$(python -c "import re; print(re.search('(gs:.+\/)(dags$)', r'${{ steps.dag_path.outputs.dag_url }}').group(1))") + url=$(python -c "import re; print(re.search('gs:\/\/(.+)(\/dags$)', r'${{ steps.dag_path.outputs.dag_url }}').group(1))") echo "::set-output name=bucket_url::${url}" - name: Update DAG bucket secret run: echo ${{ steps.bucket_root.outputs.bucket_url }} From 42595d152eb42d20167c39ac0d2bf444470245f6 Mon Sep 17 00:00:00 2001 From: Andy Smith Date: Thu, 22 Jul 2021 22:37:15 +0100 Subject: [PATCH 12/12] Update CHANGELOG.md --- CHANGELOG.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 88f31c9..bb0ec3b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,8 +9,9 @@ Please maintain both this file AND the README.md file. ## [Unreleased] ### Added - -* initial GitHub Actions for Continuous Integration (CI) +- Changed to GitHub Actions: + * initial GitHub Actions for Continuous Integration (CI) + * A new GCP Composer Environment can now be creatd using GitHub Actions ### Changed ### Deprecated