Clean up GKE workflow (hashicorp#2299)

trunet · Sep 25, 2023 · 7ddc60f · 7ddc60f
1 parent bc9b451
commit 7ddc60f
Show file tree

Hide file tree

Showing 11 changed files with 139 additions and 225 deletions.
diff --git a/.github/workflows/acceptance_tests_aks.yaml b/.github/workflows/acceptance_tests_aks.yaml
@@ -40,7 +40,7 @@ jobs:
   acceptanceTests:
     runs-on: [custom, linux, medium]
     steps:
-      - name: Checkout 
+      - name: Checkout repository
         uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
       - name: Set up go
         uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1
@@ -59,7 +59,7 @@ jobs:
         working-directory: ${{ github.workspace }}/kubernetes/test-infra/aks-new
         run: | 
           terraform init 
-          terraform apply --auto-approve
+          terraform apply -auto-approve
       - name: Run Tests  
         env: 
           TESTARGS: -run '${{ inputs.runTests }}'
@@ -70,4 +70,4 @@ jobs:
         if: always()
         working-directory: ${{ github.workspace }}/kubernetes/test-infra/aks-new 
         run: |
-          terraform destroy --auto-approve
+          terraform destroy -auto-approve
diff --git a/.github/workflows/acceptance_tests_eks.yaml b/.github/workflows/acceptance_tests_eks.yaml
@@ -44,7 +44,8 @@ jobs:
   acceptance_tests:
     runs-on: [custom, linux, medium]
     steps:
-      - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+      - name: Checkout repository
+        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
       - name: Set up Go
         uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1
         with:
@@ -67,12 +68,12 @@ jobs:
         working-directory: ${{ github.workspace }}/kubernetes/test-infra/eks
         run: |
           terraform init
-          terraform apply --auto-approve
+          terraform apply -auto-approve
       - name: Install AWS EBS CSI Driver
         working-directory: ${{ github.workspace }}/kubernetes/test-infra/aws-ebs-csi-driver
         run: |
           terraform init
-          terraform apply --auto-approve
+          terraform apply -auto-approve
       - name: Run Acceptance Test Suite
         env:
           TESTARGS: -run '${{ github.event.inputs.runTests }}'
@@ -86,4 +87,4 @@ jobs:
         if: always() # we should destroy the cluster even if the tests fail
         working-directory: ${{ github.workspace }}/kubernetes/test-infra/eks
         run: |
-          terraform destroy --auto-approve
+          terraform destroy -auto-approve
diff --git a/.github/workflows/acceptance_tests_gke.yaml b/.github/workflows/acceptance_tests_gke.yaml
@@ -5,50 +5,49 @@ on:
     inputs:
       region:
         description: The GKE region
-        default: us-east1
+        default: us-west1
       zone:
         description: The GKE zone
-        default: us-east1-b
-      kubernetesVersion:
-        description: The GKE kubernetes version
-        default: 1.24
-      workersCount:
+        default: us-west1-c
+      clusterVersion:
+        description: The GKE cluster version
+        default: 1.27
+      nodeCount:
         description: The number of cluster nodes to provision
-        default: 10
-      nodeMachineType:
+        default: 1
+      instanceType:
         description: The type of GKE instance to use for cluster nodes
-        default: e2-standard-8
+        default: e2-standard-2
+      parallelRuns:
+        description: The maximum number of tests to run simultaneously
+        default: 8
       runTests:
         description: The regex passed to the -run option of `go test`
-        default: ^TestAcc
+        default: ".*"
       terraformVersion:
         description: Terraform version
-        default: 1.4.2
+        default: 1.5.6
   schedule:
     - cron: '0 23 * * *'
+
 env:
-  GOOGLE_CREDENTIALS: ${{ secrets.GOOGLE_CREDENTIALS }}
   GOOGLE_PROJECT: ${{ secrets.GOOGLE_PROJECT }}
   GOOGLE_REGION: ${{ github.event.inputs.region || vars.GOOGLE_REGION }}
   GOOGLE_ZONE: ${{github.event.inputs.zone || vars.GOOGLE_ZONE }}
-  PARALLEL_RUNS: ${{ github.event.inputs.parallelRuns || vars.PARALLEL_RUNS }}
-  TERRAFORM_VERSION: ${{ github.event.inputs.terraformVersion || vars.TERRAFORM_VERSION }}
   USE_GKE_GCLOUD_AUTH_PLUGIN: True
-  KUBECONFIG: ${{ github.workspace }}/kubernetes/test-infra/gke/kubeconfig
   KUBE_CONFIG_PATH: ${{ github.workspace }}/kubernetes/test-infra/gke/kubeconfig
-  TF_ACC_KUBERNETES_RC_REPLICAS: 50
+  PARALLEL_RUNS: ${{ github.event.inputs.parallelRuns || vars.PARALLEL_RUNS }}
+  TERRAFORM_VERSION: ${{ github.event.inputs.terraformVersion || vars.TERRAFORM_VERSION }}
+  TF_VAR_cluster_version: ${{ github.event.inputs.clusterVersion || vars.CLUSTER_VERSION }}
+  TF_VAR_node_count: ${{ github.event.inputs.nodeCount || vars.GOOGLE_NODE_COUNT }}
+  TF_VAR_instance_type: ${{ github.event.inputs.instanceType || vars.GOOGLE_INSTANCE_TYPE }}
 
 jobs:
-  prepare-gke-environment:
-    name: "Prepare GKE cluster"
-    permissions:
-      contents: "read"
-      id-token: "write"
+  acceptance_tests:
     runs-on: [custom, linux, medium]
-    env:
-      KUBECONFIG: ${{ github.workspace }}/kubernetes/test-infra/gke/kubeconfig
     steps:
-      - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
+      - name: Checkout repository
+        uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
       - name: Set up Go
         uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1
         with:
@@ -57,135 +56,36 @@ jobs:
         uses: google-github-actions/auth@35b0e87d162680511bf346c299f71c9c5c379033 # v1.1.1
         with:
           credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }}
-          access_token_lifetime: "10800s"
+          access_token_lifetime: '14400s'
+      - name: Set up gcloud SDK
+        uses: google-github-actions/setup-gcloud@e30db14379863a8c79331b04a9969f4c1e225e0b # v1.1.1
+        with:
+          install_components: 'beta,gke-gcloud-auth-plugin'
+      - name: Initialize gcloud SDK
+        run: |
+          gcloud init
+          gcloud info
       - name: Install Terraform
         uses: hashicorp/setup-terraform@633666f66e0061ca3b725c73b2ec20cd13a8fdd1 # v2.0.3
         with:
-          terraform_version: ${{ github.event.inputs.terraformVersion }}
+          terraform_version: ${{ env.TERRAFORM_VERSION }}
           terraform_wrapper: false
       - name: Provision GKE Cluster
         working-directory: ${{ github.workspace }}/kubernetes/test-infra/gke
-        env:
-          TF_VAR_kubernetes_version: ${{ github.event.inputs.kubernetesVersion }}
-          TF_VAR_workers_count: ${{ github.event.inputs.workersCount }}
-          TF_VAR_node_machine_type: ${{ github.event.inputs.nodeMachineType }}
         run: |
           terraform init
           terraform apply -auto-approve
-      - name: "Persist state"
-        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
-        with:
-          name: gke-cluster
-          retention-days: 1
-          path: |
-            ${{ github.workspace }}/kubernetes/test-infra/gke/kubeconfig
-            ${{ github.workspace }}/kubernetes/test-infra/gke/terraform.tfstate
-
-  generate-case-matrix:
-    name: "Generate matrix of test cases"
-    runs-on: [custom, linux, medium]
-    outputs:
-      test-case-matrix: ${{ steps.generate.outputs.test-case-matrix }}
-    steps:
-      - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
-      - name: Set up Go
-        uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1
-        with:
-          go-version-file: 'go.mod'
-      - name: "Generate testcase matrix"
-        id: generate
-        run: |
-          make test-compile
-          ./kubernetes.test -test.list '${{ github.event.inputs.runTests }}' | go run tools/batchacc.go -sort -depth 3 | tee groups.json
-          echo "::set-output name=test-case-matrix::$(cat groups.json)"
-      - name: "Persist test binary"
-        uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce # v3.1.2
-        with:
-          name: test-binary
-          retention-days: 1
-          path: |
-            ${{ github.workspace }}/kubernetes.test
-
-  acceptance-tests:
-    name: "Test"
-    needs: [prepare-gke-environment, generate-case-matrix]
-    runs-on: [custom, linux, medium]
-    strategy:
-      fail-fast: false
-      matrix:
-        test-case: ${{ fromJson(needs.generate-case-matrix.outputs.test-case-matrix) }}
-    steps:
-      - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
-      - name: Set up Go
-        uses: actions/setup-go@fac708d6674e30b6ba41289acaab6d4b75aa0753 # v4.0.1
-        with:
-          go-version-file: 'go.mod'
-      - name: Authenticate to Google Cloud
-        uses: google-github-actions/auth@35b0e87d162680511bf346c299f71c9c5c379033 # v1.1.1
-        with:
-          credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }}
-          access_token_lifetime: "10800s"
-      - name: "Set up Cloud SDK"
-        uses: google-github-actions/setup-gcloud@e30db14379863a8c79331b04a9969f4c1e225e0b # v1.1.1
-        with:
-          install_components: "beta,gke-gcloud-auth-plugin"
-      - name: "Initialize gcloud SDK"
-        run: |
-          gcloud init
-          gcloud info
-      - name: "Fetch kubeconfig"
-        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
-        with:
-          name: gke-cluster
-          path: |
-            ${{ github.workspace }}/kubernetes/test-infra/gke
-      - name: "Fetch test binary"
-        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
-        with:
-          name: test-binary
-          path: |
-            ${{ github.workspace }}
-      - name: Run Acceptance Test
+      - name: Run Acceptance Test Suite
         env:
-          TF_ACC_TERRAFORM_VERSION: ${{ github.event.inputs.terraformVersion }}
-          TF_ACC: 1
-        run: |
-          chmod a+x ${{ github.workspace }}/kubernetes.test
-          ${{ github.workspace }}/kubernetes.test -test.v -test.timeout=3h -test.run='^${{ matrix.test-case }}'
-
-  clean-up:
-    name: "Destroy GKE cluster"
-    if: ${{ always() }}
-    needs: [prepare-gke-environment, acceptance-tests]
-    runs-on: [custom, linux, medium]
-    steps:
-      - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 # v3.5.3
-      - name: "Retrieve state"
-        uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2
-        with:
-          name: gke-cluster
-          path: |
-            ${{ github.workspace }}/kubernetes/test-infra/gke
-      - name: Authenticate to Google Cloud
-        uses: google-github-actions/auth@35b0e87d162680511bf346c299f71c9c5c379033 # v1.1.1
-        with:
-          credentials_json: ${{ secrets.GOOGLE_CREDENTIALS }}
-          access_token_lifetime: "10800s"
-      - name: Install Terraform
-        uses: hashicorp/setup-terraform@633666f66e0061ca3b725c73b2ec20cd13a8fdd1 # v2.0.3
-        with:
-          terraform_version: ${{ github.event.inputs.terraformVersion }}
-          terraform_wrapper: false
-      - name: "Set up Cloud SDK"
-        uses: google-github-actions/setup-gcloud@e30db14379863a8c79331b04a9969f4c1e225e0b # v1.1.1
-        with:
-          install_components: "beta,gke-gcloud-auth-plugin"
-      - name: "Initialize gcloud SDK"
+          TESTARGS: -run '${{ github.event.inputs.runTests }}'
+          # Do not set TF_ACC_TERRAFORM_PATH or TF_ACC_TERRAFORM_VERSION.
+          # In this case, the framework will search for the Terraform CLI binary based on the operating system PATH.
+          # Eventually, it will use the one we set up.
+          # More information: https://developer.hashicorp.com/terraform/plugin/sdkv2/testing/acceptance-tests#terraform-cli-installation-behaviors
         run: |
-          gcloud init
-          gcloud info
+          make testacc
       - name: Destroy GKE cluster
+        if: always() # we should destroy the cluster even if the tests fail
         working-directory: ${{ github.workspace }}/kubernetes/test-infra/gke
         run: |
-          terraform init
-          terraform destroy --auto-approve
+          terraform destroy -auto-approve
diff --git a/kubernetes/data_source_kubernetes_persistent_volume_v1_test.go b/kubernetes/data_source_kubernetes_persistent_volume_v1_test.go
@@ -17,7 +17,10 @@ func TestAccKubernetesDataSourcePersistentVolumeV1_basic(t *testing.T) {
 	name := fmt.Sprintf("tf-acc-test-%s", acctest.RandStringFromCharSet(10, acctest.CharSetAlphaNum))
 
 	resource.ParallelTest(t, resource.TestCase{
-		PreCheck:          func() { testAccPreCheck(t) },
+		PreCheck: func() {
+			testAccPreCheck(t)
+			skipIfRunningInGke(t)
+		},
 		ProviderFactories: testAccProviderFactories,
 		Steps: []resource.TestStep{
 			{

diff --git a/kubernetes/data_source_kubernetes_storage_class_v1_test.go b/kubernetes/data_source_kubernetes_storage_class_v1_test.go
@@ -159,7 +159,7 @@ func testAccKubernetesDataSourceStorageClassV1_basic(name, provisioner string) s
   }
   allowed_topologies {
     match_label_expressions {
-      key = "failure-domain.beta.kubernetes.io/zone"
+      key = "topology.kubernetes.io/zone"
       values = [
         "us-east-1a",
         "us-east-1b"

diff --git a/kubernetes/provider_test.go b/kubernetes/provider_test.go
@@ -305,6 +305,16 @@ func skipIfRunningInEks(t *testing.T) {
 	}
 }
 
+func skipIfRunningInGke(t *testing.T) {
+	isInGke, err := isRunningInGke()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if isInGke {
+		t.Skip("This test cannot be run in GKE cluster")
+	}
+}
+
 func skipIfNotRunningInMinikube(t *testing.T) {
 	isInMinikube, err := isRunningInMinikube()
 	if err != nil {