Replace tf backends with TF workspaces

infra/aws/terraform/prow-build-cluster/Makefile

@@ -16,30 +16,33 @@ TF ?= terraform
 ASSUME_ROLE ?= true
 
 # Valid values are: canary, prod
-PROW_CLUSTER ?= canary
+WORKSPACE_NAME ?= canary
+
+.PHONY: workspace-select
+workspace-select:
+	$(TF) workspace select $(WORKSPACE_NAME)
 
 .PHONY: init
 init:
-	$(TF) $@ \
-		-backend-config=./tfbackends/$(PROW_CLUSTER).tfbackend
+	$(TF) $@
 
 .PHONY: plan
-plan:
+plan: workspace-select
 	$(TF) $@ \
-		-var-file=./terraform.$(PROW_CLUSTER).tfvars \
-		-var="assume_role=$(ASSUME_ROLE)"
+		-var="assume_role=$(ASSUME_ROLE)" \
+		-var-file=./terraform.$(WORKSPACE_NAME).tfvars
 
 .PHONY: apply
-apply:
+apply: workspace-select
 	$(TF) $@ \
-		-var-file=./terraform.$(PROW_CLUSTER).tfvars \
-		-var="assume_role=$(ASSUME_ROLE)"
+		-var="assume_role=$(ASSUME_ROLE)" \
+		-var-file=./terraform.$(WORKSPACE_NAME).tfvars
 
 .PHONY: destroy
-destory:
+destory: workspace-select
 	$(TF) $@ \
-		-var-file=./terraform.$(PROW_CLUSTER).tfvars \
-		-var="assume_role=$(ASSUME_ROLE)"
+		-var="assume_role=$(ASSUME_ROLE)" \
+		-var-file=./terraform.$(WORKSPACE_NAME).tfvars
 
 .PHONY: fmt
 fmt:

infra/aws/terraform/prow-build-cluster/README.md

@@ -22,13 +22,11 @@ variable has to be set:
 
 ```bash
 # For provisioning Prod:
-export PROW_CLUSTER=prod
+export WORKSPACE_NAME=prod
 # For provisioning Canary:
-export PROW_CLUSTER=canary
+export WORKSPACE_NAME=canary
 
 # Just making sure we don't have state cached locally.
-make clean
-
 ASSUME_ROLE=false make init
 ASSUME_ROLE=false make apply
 ```
@@ -94,17 +92,18 @@ args:
   - arn:aws:iam::468814281478:role/canary-Prow-Cluster-Admin
 ```
 
-
-
 ## Removing cluster
 
 Same as for installation, cluster removal requires running Terraform twice.
 **IMPORTANT**: It's possible only for users with assigned `AdministratorAccess` policy.
 
 ```bash
+export WORKSPACE_NAME= # choose between canary/prod
+
 # First remove resources running on the cluster and IAM role. This fails once assumed role gets deleted.
 make destroy
 
 # Clean up the rest. 
 ASSUME_ROLE=false make destroy
 ```
+

infra/aws/terraform/prow-build-cluster/eks.tf

@@ -19,27 +19,28 @@ limitations under the License.
 ###############################################
 
 locals {
-  aws_auth_roles_base = [
-    # Allow access to the Prow-Cluster-Admin IAM role (used with assume role with other IAM accounts).
-    {
-      "rolearn"  = aws_iam_role.iam_cluster_admin.arn
-      "username" = "eks-cluster-admin"
-      "groups" = [
-        "eks-cluster-admin"
-      ]
-    },
-  ]
-
-  aws_auth_roles = var.is_canary_installation ? local.aws_auth_roles_base : concat([
-    # Allow access to the Prow-EKS-Admin IAM role (used by Prow directly).
-    {
-      "rolearn"  = aws_iam_role.eks_admin[0].arn
-      "username" = "eks-admin"
-      "groups" = [
-        "eks-prow-cluster-admin"
-      ]
-    }
-  ], local.aws_auth_roles_base)
+  aws_auth_roles = concat(
+    terraform.workspace == "prod" ? [
+      # Allow access to the Prow-EKS-Admin IAM role (used by Prow directly).
+      {
+        "rolearn"  = aws_iam_role.eks_admin[0].arn
+        "username" = "eks-admin"
+        "groups" = [
+          "eks-prow-cluster-admin"
+        ]
+      }
+    ] : [],
+    [
+      # Allow access to the Prow-Cluster-Admin IAM role (used with assume role with other IAM accounts).
+      {
+        "rolearn"  = aws_iam_role.iam_cluster_admin.arn
+        "username" = "eks-cluster-admin"
+        "groups" = [
+          "eks-cluster-admin"
+        ]
+      }
+    ]
+  )
 }
 
 module "eks" {

infra/aws/terraform/prow-build-cluster/main.tf

@@ -22,7 +22,7 @@ data "aws_caller_identity" "current" {}
 data "aws_availability_zones" "available" {}
 
 locals {
-  canary_prefix = var.is_canary_installation ? "canary-" : ""
+  canary_prefix = terraform.workspace != "prod" ? "canary-" : ""
 
   root_account_arn  = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:root"
   aws_cli_base_args = ["eks", "get-token", "--cluster-name", module.eks.cluster_name]

infra/aws/terraform/prow-build-cluster/providers.tf

@@ -15,8 +15,11 @@ limitations under the License.
 */
 
 terraform {
-  # Backend conifguration lives inside *.tfbackend files.
-  backend "s3" {}
+  backend "s3" {
+    bucket = "prow-build-cluster-tfstate"
+    key    = "terraform.tfstate"
+    region = "us-east-2"
+  }
 
   required_version = "~> 1.3.0"
 

infra/aws/terraform/prow-build-cluster/prow.tf

@@ -19,7 +19,7 @@ limitations under the License.
 
 # Recognize federated identities from the prow trusted cluster
 resource "aws_iam_openid_connect_provider" "k8s_prow" {
-  count = var.is_canary_installation ? 0 : 1
+  count = terraform.workspace == "prod" ? 1 : 0
 
   url             = "https://container.googleapis.com/v1/projects/k8s-prow/locations/us-central1-f/clusters/prow"
   client_id_list  = ["sts.amazonaws.com"]
@@ -28,7 +28,7 @@ resource "aws_iam_openid_connect_provider" "k8s_prow" {
 
 # We allow Prow Pods with specific service acccounts on the a particular cluster to assume this role
 resource "aws_iam_role" "eks_admin" {
-  count = var.is_canary_installation ? 0 : 1
+  count = terraform.workspace == "prod" ? 1 : 0
 
   name = "Prow-EKS-Admin"
 

infra/aws/terraform/prow-build-cluster/terraform.canary.tfvars

@@ -15,7 +15,6 @@ limitations under the License.
 */
 
 assume_role            = true
-is_canary_installation = true
 
 cluster_name    = "prow-build-canary-cluster"
 cluster_region  = "us-east-2"

infra/aws/terraform/prow-build-cluster/terraform.prod.tfvars

@@ -15,7 +15,6 @@ limitations under the License.
 */
 
 assume_role            = true
-is_canary_installation = false
 
 cluster_name    = "prow-build-cluster"
 cluster_region  = "us-east-2"

infra/aws/terraform/prow-build-cluster/variables.tf

@@ -22,12 +22,6 @@ variable "assume_role" {
   default     = true
 }
 
-variable "is_canary_installation" {
-  type        = bool
-  description = "If set, scripts provision canary cluster instead of production."
-  default     = false
-}
-
 variable "vpc_cidr" {
   type        = string
   description = "CIDR of the VPC"