From a9adb740727034027d812af1aeadc57f09e5261b Mon Sep 17 00:00:00 2001
From: Jim Enright <jenright@cloudera.com>
Date: Mon, 10 Jun 2024 16:55:44 +0100
Subject: [PATCH 1/3] Update CDP provider to version 0.6.1

Signed-off-by: Jim Enright <jenright@cloudera.com>
---
 .../examples/ex01-minimal_inputs/main.tf      |  2 +-
 .../examples/ex02-existing-vpc/main.tf        |  2 +-
 .../examples/ex03-create-keypair/main.tf      |  2 +-
 modules/terraform-cdp-deploy/README.md        | 10 +++-
 .../examples/ex01-aws-basic/main.tf           |  2 +-
 .../examples/ex02-azure-basic/main.tf         |  5 +-
 modules/terraform-cdp-deploy/main.tf          | 56 +++++++++++--------
 .../terraform-cdp-deploy/modules/aws/main.tf  | 10 ++--
 .../modules/aws/provider.tf                   |  2 +-
 .../modules/aws/variables.tf                  | 14 +++++
 .../modules/azure/main.tf                     | 16 ++++--
 .../modules/azure/provider.tf                 |  2 +-
 .../modules/azure/variables.tf                | 39 ++++++++++++-
 .../terraform-cdp-deploy/modules/gcp/main.tf  | 13 +++--
 .../modules/gcp/provider.tf                   |  2 +-
 .../modules/gcp/variables.tf                  | 22 +++++++-
 modules/terraform-cdp-deploy/provider.tf      |  2 +-
 modules/terraform-cdp-deploy/variables.tf     | 51 ++++++++++++++++-
 18 files changed, 198 insertions(+), 54 deletions(-)

diff --git a/modules/terraform-cdp-aws-pre-reqs/examples/ex01-minimal_inputs/main.tf b/modules/terraform-cdp-aws-pre-reqs/examples/ex01-minimal_inputs/main.tf
index 8572dcb..39bed77 100644
--- a/modules/terraform-cdp-aws-pre-reqs/examples/ex01-minimal_inputs/main.tf
+++ b/modules/terraform-cdp-aws-pre-reqs/examples/ex01-minimal_inputs/main.tf
@@ -38,7 +38,7 @@ terraform {
   required_providers {
     cdp = {
       source  = "cloudera/cdp"
-      version = "0.5.8"
+      version = "0.6.1"
     }
   }
 }
diff --git a/modules/terraform-cdp-aws-pre-reqs/examples/ex02-existing-vpc/main.tf b/modules/terraform-cdp-aws-pre-reqs/examples/ex02-existing-vpc/main.tf
index 1b48013..25be5d1 100644
--- a/modules/terraform-cdp-aws-pre-reqs/examples/ex02-existing-vpc/main.tf
+++ b/modules/terraform-cdp-aws-pre-reqs/examples/ex02-existing-vpc/main.tf
@@ -51,7 +51,7 @@ terraform {
   required_providers {
     cdp = {
       source  = "cloudera/cdp"
-      version = "0.5.8"
+      version = "0.6.1"
     }
   }
 }
diff --git a/modules/terraform-cdp-aws-pre-reqs/examples/ex03-create-keypair/main.tf b/modules/terraform-cdp-aws-pre-reqs/examples/ex03-create-keypair/main.tf
index 35c0222..1a84bb6 100644
--- a/modules/terraform-cdp-aws-pre-reqs/examples/ex03-create-keypair/main.tf
+++ b/modules/terraform-cdp-aws-pre-reqs/examples/ex03-create-keypair/main.tf
@@ -58,7 +58,7 @@ terraform {
   required_providers {
     cdp = {
       source  = "cloudera/cdp"
-      version = "0.5.8"
+      version = "0.6.1"
     }
   }
 }
diff --git a/modules/terraform-cdp-deploy/README.md b/modules/terraform-cdp-deploy/README.md
index c068928..4298b2e 100644
--- a/modules/terraform-cdp-deploy/README.md
+++ b/modules/terraform-cdp-deploy/README.md
@@ -20,7 +20,7 @@ In each directory an example `terraform.tfvars.sample` values file is included t
 | Name | Version |
 |------|---------|
 | <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.3.0 |
-| <a name="requirement_cdp"></a> [cdp](#requirement\_cdp) | 0.5.8 |
+| <a name="requirement_cdp"></a> [cdp](#requirement\_cdp) | 0.6.1 |
 
 ## Providers
 
@@ -63,13 +63,15 @@ No resources.
 | <a name="input_aws_xaccount_role_arn"></a> [aws\_xaccount\_role\_arn](#input\_aws\_xaccount\_role\_arn) | Cross Account Role ARN. Required for CDP deployment on AWS. | `string` | `null` | no |
 | <a name="input_azure_accept_image_terms"></a> [azure\_accept\_image\_terms](#input\_azure\_accept\_image\_terms) | Flag to automatically accept Azure Marketplace image terms during CDP cluster deployment. | `bool` | `true` | no |
 | <a name="input_azure_aks_private_dns_zone_id"></a> [azure\_aks\_private\_dns\_zone\_id](#input\_azure\_aks\_private\_dns\_zone\_id) | The ID of an existing private DNS zone used for the AKS. | `string` | `null` | no |
-| <a name="input_azure_cdp_flexible_server_delegated_subnet_names"></a> [azure\_cdp\_flexible\_server\_delegated\_subnet\_names](#input\_azure\_cdp\_flexible\_server\_delegated\_subnet\_names) | List of Azure Subnet Names delegated for Private Flexible servers. Required for CDP deployment on Azure. | `list(any)` | `null` | no |
 | <a name="input_azure_cdp_gateway_subnet_names"></a> [azure\_cdp\_gateway\_subnet\_names](#input\_azure\_cdp\_gateway\_subnet\_names) | List of Azure Subnet Names CDP Endpoint Access Gateway. Required for CDP deployment on Azure. | `list(any)` | `null` | no |
 | <a name="input_azure_cdp_subnet_names"></a> [azure\_cdp\_subnet\_names](#input\_azure\_cdp\_subnet\_names) | List of Azure Subnet Names for CDP Resources. Required for CDP deployment on Azure. | `list(any)` | `null` | no |
 | <a name="input_azure_create_private_endpoints"></a> [azure\_create\_private\_endpoints](#input\_azure\_create\_private\_endpoints) | Flag to specify that Azure Postgres will be configured with Private Endpoint and a Private DNS Zone. | `bool` | `null` | no |
 | <a name="input_azure_database_private_dns_zone_id"></a> [azure\_database\_private\_dns\_zone\_id](#input\_azure\_database\_private\_dns\_zone\_id) | The ID of an existing private DNS zone used for the database. | `string` | `null` | no |
+| <a name="input_azure_datalake_flexible_server_delegated_subnet_name"></a> [azure\_datalake\_flexible\_server\_delegated\_subnet\_name](#input\_azure\_datalake\_flexible\_server\_delegated\_subnet\_name) | The subnet ID for the subnet within which you want to configure your Azure Flexible Server for the CDP datalake | `string` | `null` | no |
 | <a name="input_azure_datalakeadmin_identity_id"></a> [azure\_datalakeadmin\_identity\_id](#input\_azure\_datalakeadmin\_identity\_id) | Datalake Admin Managed Identity ID. Required for CDP deployment on Azure. | `string` | `null` | no |
+| <a name="input_azure_environment_flexible_server_delegated_subnet_names"></a> [azure\_environment\_flexible\_server\_delegated\_subnet\_names](#input\_azure\_environment\_flexible\_server\_delegated\_subnet\_names) | List of Azure Subnet Names delegated for Private Flexible servers. Required for CDP deployment on Azure. | `list(any)` | `null` | no |
 | <a name="input_azure_idbroker_identity_id"></a> [azure\_idbroker\_identity\_id](#input\_azure\_idbroker\_identity\_id) | IDBroker Managed Identity ID. Required for CDP deployment on Azure. | `string` | `null` | no |
+| <a name="input_azure_load_balancer_sku"></a> [azure\_load\_balancer\_sku](#input\_azure\_load\_balancer\_sku) | The Azure load balancer SKU type. Possible values are BASIC, STANDARD or None. The current default is BASIC. To disable the load balancer, use type NONE. | `string` | `null` | no |
 | <a name="input_azure_log_identity_id"></a> [azure\_log\_identity\_id](#input\_azure\_log\_identity\_id) | Log Data Access Managed Identity ID. Required for CDP deployment on Azure. | `string` | `null` | no |
 | <a name="input_azure_ranger_audit_identity_id"></a> [azure\_ranger\_audit\_identity\_id](#input\_azure\_ranger\_audit\_identity\_id) | Ranger Audit Managed Identity ID. Required for CDP deployment on Azure. | `string` | `null` | no |
 | <a name="input_azure_raz_identity_id"></a> [azure\_raz\_identity\_id](#input\_azure\_raz\_identity\_id) | RAZ Managed Identity ID. Required for CDP deployment on Azure. | `string` | `null` | no |
@@ -87,6 +89,7 @@ No resources.
 | <a name="input_cdp_xacccount_credential_name"></a> [cdp\_xacccount\_credential\_name](#input\_cdp\_xacccount\_credential\_name) | Name of the CDP Cross Account Credential. Defaults to '<env\_prefix>-xaccount-cred' if not specified. If create\_cdp\_credential is set to false then this should should be a valid pre-existing credential. | `string` | `null` | no |
 | <a name="input_create_cdp_credential"></a> [create\_cdp\_credential](#input\_create\_cdp\_credential) | Flag to specify if the CDP Cross Account Credential should be created. If set to false then cdp\_xacccount\_credential\_name should be a valid pre-existing credential. | `bool` | `true` | no |
 | <a name="input_datalake_async_creation"></a> [datalake\_async\_creation](#input\_datalake\_async\_creation) | Flag to specify if Terraform should wait for CDP datalake resource creation/deletion | `bool` | `false` | no |
+| <a name="input_datalake_call_failure_threshold"></a> [datalake\_call\_failure\_threshold](#input\_datalake\_call\_failure\_threshold) | Threshold value that specifies how many times should a single CDP Datalake API call failure happen before giving up the polling | `number` | `3` | no |
 | <a name="input_datalake_custom_instance_groups"></a> [datalake\_custom\_instance\_groups](#input\_datalake\_custom\_instance\_groups) | A set of custom instance groups for the datalake. Only applicable for CDP deployment on AWS and GCP. | <pre>list(<br>    object({<br>      name          = string,<br>      instance_type = optional(string)<br>    })<br>  )</pre> | `null` | no |
 | <a name="input_datalake_image"></a> [datalake\_image](#input\_datalake\_image) | The image to use for the datalake. Can only be used when the 'datalake\_version' parameter is set to null. You can use 'catalog' name and/or 'id' for selecting an image. | <pre>object({<br>    id      = optional(string)<br>    catalog = optional(string)<br>  })</pre> | `null` | no |
 | <a name="input_datalake_java_version"></a> [datalake\_java\_version](#input\_datalake\_java\_version) | The Java major version to use on the datalake cluster. | `number` | `null` | no |
@@ -102,9 +105,11 @@ No resources.
 | <a name="input_encryption_key_arn"></a> [encryption\_key\_arn](#input\_encryption\_key\_arn) | ARN of the AWS KMS CMK to use for the server-side encryption of AWS storage resources. Only applicable for CDP deployment on AWS. | `string` | `null` | no |
 | <a name="input_encryption_key_resource_group_name"></a> [encryption\_key\_resource\_group\_name](#input\_encryption\_key\_resource\_group\_name) | Name of the existing Azure resource group hosting the Azure Key Vault containing customer managed key which will be used to encrypt the Azure Managed Disk. Only applicable for CDP deployment on Azure. | `string` | `null` | no |
 | <a name="input_encryption_key_url"></a> [encryption\_key\_url](#input\_encryption\_key\_url) | URL of the key which will be used to encrypt the Azure Managed Disks. Only applicable for CDP deployment on Azure. | `string` | `null` | no |
+| <a name="input_encryption_user_managed_identity"></a> [encryption\_user\_managed\_identity](#input\_encryption\_user\_managed\_identity) | Managed Identity ID for encryption | `string` | `null` | no |
 | <a name="input_endpoint_access_scheme"></a> [endpoint\_access\_scheme](#input\_endpoint\_access\_scheme) | The scheme for the workload endpoint gateway. PUBLIC creates an external endpoint that can be accessed over the Internet. PRIVATE which restricts the traffic to be internal to the VPC / Vnet. Relevant in Private Networks. | `string` | `null` | no |
 | <a name="input_env_tags"></a> [env\_tags](#input\_env\_tags) | Tags applied to provisioned resources | `map(any)` | `null` | no |
 | <a name="input_environment_async_creation"></a> [environment\_async\_creation](#input\_environment\_async\_creation) | Flag to specify if Terraform should wait for CDP environment resource creation/deletion | `bool` | `false` | no |
+| <a name="input_environment_call_failure_threshold"></a> [environment\_call\_failure\_threshold](#input\_environment\_call\_failure\_threshold) | Threshold value that specifies how many times should a single CDP Environment API call failure happen before giving up the polling | `number` | `3` | no |
 | <a name="input_environment_name"></a> [environment\_name](#input\_environment\_name) | Name of the CDP environment. Defaults to '<env\_prefix>-cdp-env' if not specified. | `string` | `null` | no |
 | <a name="input_environment_polling_timeout"></a> [environment\_polling\_timeout](#input\_environment\_polling\_timeout) | Timeout value in minutes for how long to poll for CDP Environment resource creation/deletion | `number` | `60` | no |
 | <a name="input_freeipa_catalog"></a> [freeipa\_catalog](#input\_freeipa\_catalog) | Image catalog to use for FreeIPA image selection | `string` | `null` | no |
@@ -112,6 +117,7 @@ No resources.
 | <a name="input_freeipa_instance_type"></a> [freeipa\_instance\_type](#input\_freeipa\_instance\_type) | Instance Type to use for creating FreeIPA instances | `string` | `null` | no |
 | <a name="input_freeipa_instances"></a> [freeipa\_instances](#input\_freeipa\_instances) | The number of FreeIPA instances to create in the environment | `number` | `3` | no |
 | <a name="input_freeipa_recipes"></a> [freeipa\_recipes](#input\_freeipa\_recipes) | The recipes for the FreeIPA cluster | `set(string)` | `null` | no |
+| <a name="input_gcp_availability_zones"></a> [gcp\_availability\_zones](#input\_gcp\_availability\_zones) | The zones of the environment in the given region. Multi-zone selection is not supported in GCP yet. It accepts only one zone until support is added. | `list(string)` | `null` | no |
 | <a name="input_gcp_cdp_subnet_names"></a> [gcp\_cdp\_subnet\_names](#input\_gcp\_cdp\_subnet\_names) | List of GCP Subnet Names for CDP Resources. Required for CDP deployment on GCP. | `list(any)` | `null` | no |
 | <a name="input_gcp_datalake_admin_service_account_email"></a> [gcp\_datalake\_admin\_service\_account\_email](#input\_gcp\_datalake\_admin\_service\_account\_email) | Email id of the service account for Datalake Admin. Required for CDP deployment on GCP. | `string` | `null` | no |
 | <a name="input_gcp_encryption_key"></a> [gcp\_encryption\_key](#input\_gcp\_encryption\_key) | Key Resource ID of the customer managed encryption key to encrypt GCP resources. Only applicable for CDP deployment on GCP. | `string` | `null` | no |
diff --git a/modules/terraform-cdp-deploy/examples/ex01-aws-basic/main.tf b/modules/terraform-cdp-deploy/examples/ex01-aws-basic/main.tf
index bdce45b..c04a3c9 100644
--- a/modules/terraform-cdp-deploy/examples/ex01-aws-basic/main.tf
+++ b/modules/terraform-cdp-deploy/examples/ex01-aws-basic/main.tf
@@ -87,7 +87,7 @@ terraform {
   required_providers {
     cdp = {
       source  = "cloudera/cdp"
-      version = "0.5.8"
+      version = "0.6.1"
     }
   }
 }
diff --git a/modules/terraform-cdp-deploy/examples/ex02-azure-basic/main.tf b/modules/terraform-cdp-deploy/examples/ex02-azure-basic/main.tf
index 4942b0a..780664b 100755
--- a/modules/terraform-cdp-deploy/examples/ex02-azure-basic/main.tf
+++ b/modules/terraform-cdp-deploy/examples/ex02-azure-basic/main.tf
@@ -66,8 +66,9 @@ module "cdp_deploy" {
   azure_cdp_subnet_names         = module.cdp_azure_prereqs.azure_cdp_subnet_names
   azure_cdp_gateway_subnet_names = module.cdp_azure_prereqs.azure_cdp_gateway_subnet_names
 
-  azure_cdp_flexible_server_delegated_subnet_names = module.cdp_azure_prereqs.azure_cdp_flexible_server_delegated_subnet_names
-  azure_database_private_dns_zone_id               = module.cdp_azure_prereqs.azure_database_private_dns_zone_id
+  azure_environment_flexible_server_delegated_subnet_names = module.cdp_azure_prereqs.azure_cdp_flexible_server_delegated_subnet_names
+  azure_datalake_flexible_server_delegated_subnet_name     = try(module.cdp_azure_prereqs.azure_cdp_flexible_server_delegated_subnet_names[0], null)
+  azure_database_private_dns_zone_id                       = module.cdp_azure_prereqs.azure_database_private_dns_zone_id
 
   azure_security_group_default_uri = module.cdp_azure_prereqs.azure_security_group_default_uri
   azure_security_group_knox_uri    = module.cdp_azure_prereqs.azure_security_group_knox_uri
diff --git a/modules/terraform-cdp-deploy/main.tf b/modules/terraform-cdp-deploy/main.tf
index b3a5cfa..1818c83 100644
--- a/modules/terraform-cdp-deploy/main.tf
+++ b/modules/terraform-cdp-deploy/main.tf
@@ -41,10 +41,12 @@ module "cdp_on_aws" {
   report_deployment_logs = var.report_deployment_logs
   endpoint_access_scheme = local.endpoint_access_scheme
 
-  environment_async_creation  = var.environment_async_creation
-  environment_polling_timeout = var.environment_polling_timeout
-  datalake_async_creation     = var.datalake_async_creation
-  datalake_polling_timeout    = var.datalake_polling_timeout
+  environment_async_creation         = var.environment_async_creation
+  environment_call_failure_threshold = var.environment_call_failure_threshold
+  environment_polling_timeout        = var.environment_polling_timeout
+  datalake_async_creation            = var.datalake_async_creation
+  datalake_call_failure_threshold    = var.datalake_call_failure_threshold
+  datalake_polling_timeout           = var.datalake_polling_timeout
 
   region            = var.region
   vpc_id            = var.aws_vpc_id
@@ -112,10 +114,12 @@ module "cdp_on_azure" {
   report_deployment_logs = var.report_deployment_logs
   endpoint_access_scheme = local.endpoint_access_scheme
 
-  environment_async_creation  = var.environment_async_creation
-  environment_polling_timeout = var.environment_polling_timeout
-  datalake_async_creation     = var.datalake_async_creation
-  datalake_polling_timeout    = var.datalake_polling_timeout
+  environment_async_creation         = var.environment_async_creation
+  environment_call_failure_threshold = var.environment_call_failure_threshold
+  environment_polling_timeout        = var.environment_polling_timeout
+  datalake_async_creation            = var.datalake_async_creation
+  datalake_call_failure_threshold    = var.datalake_call_failure_threshold
+  datalake_polling_timeout           = var.datalake_polling_timeout
 
   azure_accept_image_terms  = var.azure_accept_image_terms
   use_single_resource_group = var.use_single_resource_group
@@ -124,13 +128,13 @@ module "cdp_on_azure" {
   subscription_id = var.azure_subscription_id
   tenant_id       = var.azure_tenant_id
 
-  region                                     = var.region
-  resource_group_name                        = var.azure_resource_group_name
-  vnet_name                                  = var.azure_vnet_name
-  cdp_subnet_names                           = var.azure_cdp_subnet_names
-  cdp_gateway_subnet_names                   = var.azure_cdp_gateway_subnet_names
-  cdp_flexible_server_delegated_subnet_names = var.azure_cdp_flexible_server_delegated_subnet_names
-  public_key_text                            = var.public_key_text
+  region                                             = var.region
+  resource_group_name                                = var.azure_resource_group_name
+  vnet_name                                          = var.azure_vnet_name
+  cdp_subnet_names                                   = var.azure_cdp_subnet_names
+  cdp_gateway_subnet_names                           = var.azure_cdp_gateway_subnet_names
+  environment_flexible_server_delegated_subnet_names = var.azure_environment_flexible_server_delegated_subnet_names
+  public_key_text                                    = var.public_key_text
 
   data_storage_location   = var.data_storage_location
   log_storage_location    = var.log_storage_location
@@ -152,10 +156,12 @@ module "cdp_on_azure" {
   freeipa_recipes       = var.freeipa_recipes
 
   enable_outbound_load_balancer = var.enable_outbound_load_balancer
+  load_balancer_sku             = var.azure_load_balancer_sku
 
   encryption_key_resource_group_name = var.encryption_key_resource_group_name
   encryption_key_url                 = var.encryption_key_url
   encryption_at_host                 = var.encryption_at_host
+  encryption_user_managed_identity   = var.encryption_user_managed_identity
 
   azure_aks_private_dns_zone_id      = var.azure_aks_private_dns_zone_id
   azure_database_private_dns_zone_id = var.azure_database_private_dns_zone_id
@@ -163,9 +169,10 @@ module "cdp_on_azure" {
 
   proxy_config_name = var.proxy_config_name
 
-  datalake_image        = var.datalake_image
-  datalake_java_version = var.datalake_java_version
-  datalake_recipes      = var.datalake_recipes
+  datalake_image                                 = var.datalake_image
+  datalake_java_version                          = var.datalake_java_version
+  datalake_recipes                               = var.datalake_recipes
+  datalake_flexible_server_delegated_subnet_name = var.azure_datalake_flexible_server_delegated_subnet_name
 }
 
 # ------- Call sub-module for GCP Deployment -------
@@ -198,11 +205,14 @@ module "cdp_on_gcp" {
   workload_analytics     = var.workload_analytics
   report_deployment_logs = var.report_deployment_logs
   endpoint_access_scheme = local.endpoint_access_scheme
-
-  environment_async_creation  = var.environment_async_creation
-  environment_polling_timeout = var.environment_polling_timeout
-  datalake_async_creation     = var.datalake_async_creation
-  datalake_polling_timeout    = var.datalake_polling_timeout
+  availability_zones     = var.gcp_availability_zones
+
+  environment_async_creation         = var.environment_async_creation
+  environment_call_failure_threshold = var.environment_call_failure_threshold
+  environment_polling_timeout        = var.environment_polling_timeout
+  datalake_async_creation            = var.datalake_async_creation
+  datalake_call_failure_threshold    = var.datalake_call_failure_threshold
+  datalake_polling_timeout           = var.datalake_polling_timeout
 
   use_public_ips = local.use_public_ips
 
diff --git a/modules/terraform-cdp-deploy/modules/aws/main.tf b/modules/terraform-cdp-deploy/modules/aws/main.tf
index d35d063..6308874 100644
--- a/modules/terraform-cdp-deploy/modules/aws/main.tf
+++ b/modules/terraform-cdp-deploy/modules/aws/main.tf
@@ -68,8 +68,9 @@ resource "cdp_environments_aws_environment" "cdp_env" {
   encryption_key_arn = var.encryption_key_arn
 
   polling_options = {
-    async           = var.environment_async_creation
-    polling_timeout = var.environment_polling_timeout
+    async                  = var.environment_async_creation
+    call_failure_threshold = var.environment_call_failure_threshold
+    polling_timeout        = var.environment_polling_timeout
   }
 
   tags = var.tags
@@ -144,8 +145,9 @@ resource "cdp_datalake_aws_datalake" "cdp_datalake" {
   recipes                = var.datalake_recipes
 
   polling_options = {
-    async           = var.datalake_async_creation
-    polling_timeout = var.datalake_polling_timeout
+    async                  = var.datalake_async_creation
+    call_failure_threshold = var.datalake_call_failure_threshold
+    polling_timeout        = var.datalake_polling_timeout
   }
 
   tags = var.tags
diff --git a/modules/terraform-cdp-deploy/modules/aws/provider.tf b/modules/terraform-cdp-deploy/modules/aws/provider.tf
index e4cdd91..97ead95 100644
--- a/modules/terraform-cdp-deploy/modules/aws/provider.tf
+++ b/modules/terraform-cdp-deploy/modules/aws/provider.tf
@@ -16,7 +16,7 @@ terraform {
   required_providers {
     cdp = {
       source  = "cloudera/cdp"
-      version = "0.5.8"
+      version = "0.6.1"
     }
   }
 
diff --git a/modules/terraform-cdp-deploy/modules/aws/variables.tf b/modules/terraform-cdp-deploy/modules/aws/variables.tf
index d37a8a1..2cebdb7 100644
--- a/modules/terraform-cdp-deploy/modules/aws/variables.tf
+++ b/modules/terraform-cdp-deploy/modules/aws/variables.tf
@@ -92,6 +92,13 @@ variable "environment_async_creation" {
 
 }
 
+variable "environment_call_failure_threshold" {
+  type = number
+
+  description = "Threshold value that specifies how many times should a single CDP Environment API call failure happen before giving up the polling"
+
+}
+
 variable "environment_polling_timeout" {
   type = number
 
@@ -229,6 +236,13 @@ variable "datalake_async_creation" {
 
 }
 
+variable "datalake_call_failure_threshold" {
+  type = number
+
+  description = "Threshold value that specifies how many times should a single CDP Datalake API call failure happen before giving up the polling"
+
+}
+
 variable "datalake_polling_timeout" {
   type = number
 
diff --git a/modules/terraform-cdp-deploy/modules/azure/main.tf b/modules/terraform-cdp-deploy/modules/azure/main.tf
index e9f7f36..b6289ca 100644
--- a/modules/terraform-cdp-deploy/modules/azure/main.tf
+++ b/modules/terraform-cdp-deploy/modules/azure/main.tf
@@ -61,7 +61,7 @@ resource "cdp_environments_azure_environment" "cdp_env" {
     subnet_ids                   = var.cdp_subnet_names
     aks_private_dns_zone_id      = var.azure_aks_private_dns_zone_id
     database_private_dns_zone_id = var.azure_database_private_dns_zone_id
-    flexible_server_subnet_ids   = var.cdp_flexible_server_delegated_subnet_names
+    flexible_server_subnet_ids   = var.environment_flexible_server_delegated_subnet_names
   }
   create_private_endpoints = var.create_private_endpoints
 
@@ -89,10 +89,12 @@ resource "cdp_environments_azure_environment" "cdp_env" {
   encryption_key_resource_group_name = var.encryption_key_resource_group_name
   encryption_key_url                 = var.encryption_key_url
   encryption_at_host                 = var.encryption_at_host
+  encryption_user_managed_identity   = var.encryption_user_managed_identity
 
   polling_options = {
-    async           = var.environment_async_creation
-    polling_timeout = var.environment_polling_timeout
+    async                  = var.environment_async_creation
+    call_failure_threshold = var.environment_call_failure_threshold
+    polling_timeout        = var.environment_polling_timeout
   }
 
   tags = var.tags
@@ -163,13 +165,17 @@ resource "cdp_datalake_azure_datalake" "cdp_datalake" {
   enable_ranger_raz = var.enable_raz
   multi_az          = var.datalake_scale == "LIGHT_DUTY" ? null : var.multiaz
 
+  flexible_server_delegated_subnet_id = var.datalake_flexible_server_delegated_subnet_name
+  load_balancer_sku                   = var.load_balancer_sku
+
   image        = var.datalake_image
   java_version = var.datalake_java_version
   recipes      = var.datalake_recipes
 
   polling_options = {
-    async           = var.datalake_async_creation
-    polling_timeout = var.datalake_polling_timeout
+    async                  = var.datalake_async_creation
+    call_failure_threshold = var.datalake_call_failure_threshold
+    polling_timeout        = var.datalake_polling_timeout
   }
 
   tags = var.tags
diff --git a/modules/terraform-cdp-deploy/modules/azure/provider.tf b/modules/terraform-cdp-deploy/modules/azure/provider.tf
index e4cdd91..97ead95 100644
--- a/modules/terraform-cdp-deploy/modules/azure/provider.tf
+++ b/modules/terraform-cdp-deploy/modules/azure/provider.tf
@@ -16,7 +16,7 @@ terraform {
   required_providers {
     cdp = {
       source  = "cloudera/cdp"
-      version = "0.5.8"
+      version = "0.6.1"
     }
   }
 
diff --git a/modules/terraform-cdp-deploy/modules/azure/variables.tf b/modules/terraform-cdp-deploy/modules/azure/variables.tf
index a62d24a..c0c9487 100644
--- a/modules/terraform-cdp-deploy/modules/azure/variables.tf
+++ b/modules/terraform-cdp-deploy/modules/azure/variables.tf
@@ -98,6 +98,13 @@ variable "environment_async_creation" {
 
 }
 
+variable "environment_call_failure_threshold" {
+  type = number
+
+  description = "Threshold value that specifies how many times should a single CDP Environment API call failure happen before giving up the polling"
+
+}
+
 variable "environment_polling_timeout" {
   type = number
 
@@ -176,6 +183,12 @@ variable "encryption_at_host" {
 
 }
 
+variable "encryption_user_managed_identity" {
+  type = string
+
+  description = "Managed Identity ID for encryption"
+}
+
 variable "proxy_config_name" {
   type = string
 
@@ -245,12 +258,34 @@ variable "datalake_async_creation" {
 
 }
 
+variable "datalake_call_failure_threshold" {
+  type = number
+
+  description = "Threshold value that specifies how many times should a single CDP Datalake API call failure happen before giving up the polling"
+
+}
+
 variable "datalake_polling_timeout" {
   type = number
 
   description = "Timeout value in minutes for how long to poll for CDP datalake resource creation/deletion"
 
 }
+
+variable "datalake_flexible_server_delegated_subnet_name" {
+  type = string
+
+  description = "The subnet ID for the subnet within which you want to configure your Azure Flexible Server for the CDP datalake"
+
+}
+
+variable "load_balancer_sku" {
+  type = string
+
+  description = "The Azure load balancer SKU type. Possible values are BASIC, STANDARD or None. The current default is BASIC. To disable the load balancer, use type NONE."
+
+}
+
 # ------- Cloud Service Provider Settings -------
 variable "subscription_id" {
   type = string
@@ -340,9 +375,9 @@ variable "cdp_gateway_subnet_names" {
 
 }
 
-variable "cdp_flexible_server_delegated_subnet_names" {
+variable "environment_flexible_server_delegated_subnet_names" {
   type        = list(any)
-  description = "Azure Subnet Names delegated for Private Flexible servers."
+  description = "Azure Subnet Names delegated for Private Flexible servers for the CDP Environment."
 
 }
 
diff --git a/modules/terraform-cdp-deploy/modules/gcp/main.tf b/modules/terraform-cdp-deploy/modules/gcp/main.tf
index 3fb86c7..66ba45c 100644
--- a/modules/terraform-cdp-deploy/modules/gcp/main.tf
+++ b/modules/terraform-cdp-deploy/modules/gcp/main.tf
@@ -47,7 +47,7 @@ resource "cdp_environments_gcp_environment" "cdp_env" {
     shared_project_id = var.project_id
     subnet_names      = var.cdp_subnet_names
   }
-
+  availability_zones             = var.availability_zones
   endpoint_access_gateway_scheme = var.endpoint_access_scheme
 
   encryption_key    = var.encryption_key
@@ -63,10 +63,10 @@ resource "cdp_environments_gcp_environment" "cdp_env" {
   report_deployment_logs = var.report_deployment_logs
   enable_tunnel          = var.enable_ccm_tunnel
 
-
   polling_options = {
-    async           = var.environment_async_creation
-    polling_timeout = var.environment_polling_timeout
+    async                  = var.environment_async_creation
+    call_failure_threshold = var.environment_call_failure_threshold
+    polling_timeout        = var.environment_polling_timeout
   }
 
   tags = var.tags
@@ -142,8 +142,9 @@ resource "cdp_datalake_gcp_datalake" "cdp_datalake" {
   recipes                = var.datalake_recipes
 
   polling_options = {
-    async           = var.datalake_async_creation
-    polling_timeout = var.datalake_polling_timeout
+    async                  = var.datalake_async_creation
+    call_failure_threshold = var.datalake_call_failure_threshold
+    polling_timeout        = var.datalake_polling_timeout
   }
 
   tags = var.tags
diff --git a/modules/terraform-cdp-deploy/modules/gcp/provider.tf b/modules/terraform-cdp-deploy/modules/gcp/provider.tf
index e4cdd91..97ead95 100644
--- a/modules/terraform-cdp-deploy/modules/gcp/provider.tf
+++ b/modules/terraform-cdp-deploy/modules/gcp/provider.tf
@@ -16,7 +16,7 @@ terraform {
   required_providers {
     cdp = {
       source  = "cloudera/cdp"
-      version = "0.5.8"
+      version = "0.6.1"
     }
   }
 
diff --git a/modules/terraform-cdp-deploy/modules/gcp/variables.tf b/modules/terraform-cdp-deploy/modules/gcp/variables.tf
index dbfaa6f..b0be883 100644
--- a/modules/terraform-cdp-deploy/modules/gcp/variables.tf
+++ b/modules/terraform-cdp-deploy/modules/gcp/variables.tf
@@ -85,6 +85,13 @@ variable "environment_async_creation" {
 
 }
 
+variable "environment_call_failure_threshold" {
+  type = number
+
+  description = "Threshold value that specifies how many times should a single CDP Environment API call failure happen before giving up the polling"
+
+}
+
 variable "environment_polling_timeout" {
   type = number
 
@@ -139,7 +146,6 @@ variable "datalake_scale" {
 
 }
 
-
 variable "datalake_version" {
   type = string
 
@@ -200,6 +206,13 @@ variable "datalake_async_creation" {
 
 }
 
+variable "datalake_call_failure_threshold" {
+  type = number
+
+  description = "Threshold value that specifies how many times should a single CDP Datalake API call failure happen before giving up the polling"
+
+}
+
 variable "datalake_polling_timeout" {
   type = number
 
@@ -288,6 +301,13 @@ variable "endpoint_access_scheme" {
   }
 }
 
+variable "availability_zones" {
+  type = list(string)
+
+  description = "The zones of the environment in the given region. Multi-zone selection is not supported in GCP yet. It accepts only one zone until support is added."
+
+}
+
 variable "encryption_key" {
   type = string
 
diff --git a/modules/terraform-cdp-deploy/provider.tf b/modules/terraform-cdp-deploy/provider.tf
index e4cdd91..97ead95 100644
--- a/modules/terraform-cdp-deploy/provider.tf
+++ b/modules/terraform-cdp-deploy/provider.tf
@@ -16,7 +16,7 @@ terraform {
   required_providers {
     cdp = {
       source  = "cloudera/cdp"
-      version = "0.5.8"
+      version = "0.6.1"
     }
   }
 
diff --git a/modules/terraform-cdp-deploy/variables.tf b/modules/terraform-cdp-deploy/variables.tf
index eb0b6c6..bf6a7c2 100644
--- a/modules/terraform-cdp-deploy/variables.tf
+++ b/modules/terraform-cdp-deploy/variables.tf
@@ -122,6 +122,14 @@ variable "environment_async_creation" {
   default = false
 }
 
+variable "environment_call_failure_threshold" {
+  type = number
+
+  description = "Threshold value that specifies how many times should a single CDP Environment API call failure happen before giving up the polling"
+
+  default = 3
+}
+
 variable "environment_polling_timeout" {
   type = number
 
@@ -284,6 +292,14 @@ variable "datalake_async_creation" {
   default = false
 }
 
+variable "datalake_call_failure_threshold" {
+  type = number
+
+  description = "Threshold value that specifies how many times should a single CDP Datalake API call failure happen before giving up the polling"
+
+  default = 3
+}
+
 variable "datalake_polling_timeout" {
   type = number
 
@@ -354,6 +370,15 @@ variable "encryption_at_host" {
 
   default = null
 }
+
+variable "encryption_user_managed_identity" {
+  type = string
+
+  description = "Managed Identity ID for encryption"
+
+  default = null
+}
+
 # ------- Cloud Service Provider Settings - General -------
 variable "region" {
   type        = string
@@ -569,7 +594,7 @@ variable "azure_cdp_gateway_subnet_names" {
 
 }
 
-variable "azure_cdp_flexible_server_delegated_subnet_names" {
+variable "azure_environment_flexible_server_delegated_subnet_names" {
   type        = list(any)
   description = "List of Azure Subnet Names delegated for Private Flexible servers. Required for CDP deployment on Azure."
 
@@ -671,6 +696,22 @@ variable "azure_raz_identity_id" {
 
 }
 
+variable "azure_datalake_flexible_server_delegated_subnet_name" {
+  type = string
+
+  description = "The subnet ID for the subnet within which you want to configure your Azure Flexible Server for the CDP datalake"
+
+  default = null
+}
+
+variable "azure_load_balancer_sku" {
+  type = string
+
+  description = "The Azure load balancer SKU type. Possible values are BASIC, STANDARD or None. The current default is BASIC. To disable the load balancer, use type NONE."
+
+  default = null
+}
+
 # ------- Cloud Service Provider Settings - GCP specific -------
 
 variable "gcp_project_id" {
@@ -706,6 +747,14 @@ variable "gcp_cdp_subnet_names" {
 
 }
 
+variable "gcp_availability_zones" {
+  type = list(string)
+
+  description = "The zones of the environment in the given region. Multi-zone selection is not supported in GCP yet. It accepts only one zone until support is added."
+
+  default = null
+}
+
 variable "gcp_firewall_default_id" {
   type        = string
   description = "Default Firewall for CDP environment.  Required for CDP deployment on GCP."

From 6f4b67ef044b38ced1d2c9b92621e5f03af9bf3b Mon Sep 17 00:00:00 2001
From: Jim Enright <jenright@cloudera.com>
Date: Fri, 14 Jun 2024 15:58:41 +0100
Subject: [PATCH 2/3] Update CDP provider to version 0.6.1

Signed-off-by: Jim Enright <jenright@cloudera.com>
---
 modules/terraform-cdp-deploy/README.md             | 2 +-
 modules/terraform-cdp-deploy/modules/azure/main.tf | 1 -
 modules/terraform-cdp-deploy/variables.tf          | 2 +-
 3 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/modules/terraform-cdp-deploy/README.md b/modules/terraform-cdp-deploy/README.md
index 4298b2e..b319421 100644
--- a/modules/terraform-cdp-deploy/README.md
+++ b/modules/terraform-cdp-deploy/README.md
@@ -105,7 +105,7 @@ No resources.
 | <a name="input_encryption_key_arn"></a> [encryption\_key\_arn](#input\_encryption\_key\_arn) | ARN of the AWS KMS CMK to use for the server-side encryption of AWS storage resources. Only applicable for CDP deployment on AWS. | `string` | `null` | no |
 | <a name="input_encryption_key_resource_group_name"></a> [encryption\_key\_resource\_group\_name](#input\_encryption\_key\_resource\_group\_name) | Name of the existing Azure resource group hosting the Azure Key Vault containing customer managed key which will be used to encrypt the Azure Managed Disk. Only applicable for CDP deployment on Azure. | `string` | `null` | no |
 | <a name="input_encryption_key_url"></a> [encryption\_key\_url](#input\_encryption\_key\_url) | URL of the key which will be used to encrypt the Azure Managed Disks. Only applicable for CDP deployment on Azure. | `string` | `null` | no |
-| <a name="input_encryption_user_managed_identity"></a> [encryption\_user\_managed\_identity](#input\_encryption\_user\_managed\_identity) | Managed Identity ID for encryption | `string` | `null` | no |
+| <a name="input_encryption_user_managed_identity"></a> [encryption\_user\_managed\_identity](#input\_encryption\_user\_managed\_identity) | Managed Identity ID for encryption | `string` | `""` | no |
 | <a name="input_endpoint_access_scheme"></a> [endpoint\_access\_scheme](#input\_endpoint\_access\_scheme) | The scheme for the workload endpoint gateway. PUBLIC creates an external endpoint that can be accessed over the Internet. PRIVATE which restricts the traffic to be internal to the VPC / Vnet. Relevant in Private Networks. | `string` | `null` | no |
 | <a name="input_env_tags"></a> [env\_tags](#input\_env\_tags) | Tags applied to provisioned resources | `map(any)` | `null` | no |
 | <a name="input_environment_async_creation"></a> [environment\_async\_creation](#input\_environment\_async\_creation) | Flag to specify if Terraform should wait for CDP environment resource creation/deletion | `bool` | `false` | no |
diff --git a/modules/terraform-cdp-deploy/modules/azure/main.tf b/modules/terraform-cdp-deploy/modules/azure/main.tf
index b6289ca..91f5bf8 100644
--- a/modules/terraform-cdp-deploy/modules/azure/main.tf
+++ b/modules/terraform-cdp-deploy/modules/azure/main.tf
@@ -90,7 +90,6 @@ resource "cdp_environments_azure_environment" "cdp_env" {
   encryption_key_url                 = var.encryption_key_url
   encryption_at_host                 = var.encryption_at_host
   encryption_user_managed_identity   = var.encryption_user_managed_identity
-
   polling_options = {
     async                  = var.environment_async_creation
     call_failure_threshold = var.environment_call_failure_threshold
diff --git a/modules/terraform-cdp-deploy/variables.tf b/modules/terraform-cdp-deploy/variables.tf
index bf6a7c2..86220c2 100644
--- a/modules/terraform-cdp-deploy/variables.tf
+++ b/modules/terraform-cdp-deploy/variables.tf
@@ -376,7 +376,7 @@ variable "encryption_user_managed_identity" {
 
   description = "Managed Identity ID for encryption"
 
-  default = null
+  default = ""
 }
 
 # ------- Cloud Service Provider Settings - General -------

From b99dbe463893bce43d9c498bd2f80e367e278fca Mon Sep 17 00:00:00 2001
From: Jim Enright <jenright@cloudera.com>
Date: Tue, 18 Jun 2024 09:32:21 +0100
Subject: [PATCH 3/3] Update datalake_version validation to allow null

Signed-off-by: Jim Enright <jenright@cloudera.com>
---
 modules/terraform-cdp-deploy/modules/aws/variables.tf   | 4 +++-
 modules/terraform-cdp-deploy/modules/azure/variables.tf | 4 +++-
 modules/terraform-cdp-deploy/modules/gcp/variables.tf   | 4 +++-
 modules/terraform-cdp-deploy/variables.tf               | 4 +++-
 4 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/modules/terraform-cdp-deploy/modules/aws/variables.tf b/modules/terraform-cdp-deploy/modules/aws/variables.tf
index 2cebdb7..1b8e3c0 100644
--- a/modules/terraform-cdp-deploy/modules/aws/variables.tf
+++ b/modules/terraform-cdp-deploy/modules/aws/variables.tf
@@ -181,7 +181,9 @@ variable "datalake_version" {
   description = "The Datalake Runtime version. Valid values are latest or a semantic version, e.g. 7.2.17"
 
   validation {
-    condition     = (var.datalake_version == "latest" ? true : length(regexall("\\d+\\.\\d+.\\d+", var.datalake_version)) > 0)
+    condition = (var.datalake_version == null ? true :
+      (var.datalake_version == "latest" ? true :
+    length(regexall("\\d+\\.\\d+.\\d+", var.datalake_version)) > 0))
     error_message = "Valid values for var: datalake_version are 'latest' or a semantic versioning conventions."
   }
 
diff --git a/modules/terraform-cdp-deploy/modules/azure/variables.tf b/modules/terraform-cdp-deploy/modules/azure/variables.tf
index c0c9487..f12ff3c 100644
--- a/modules/terraform-cdp-deploy/modules/azure/variables.tf
+++ b/modules/terraform-cdp-deploy/modules/azure/variables.tf
@@ -215,7 +215,9 @@ variable "datalake_version" {
   description = "The Datalake Runtime version. Valid values are latest or a semantic version, e.g. 7.2.17"
 
   validation {
-    condition     = (var.datalake_version == "latest" ? true : length(regexall("\\d+\\.\\d+.\\d+", var.datalake_version)) > 0)
+    condition = (var.datalake_version == null ? true :
+      (var.datalake_version == "latest" ? true :
+    length(regexall("\\d+\\.\\d+.\\d+", var.datalake_version)) > 0))
     error_message = "Valid values for var: datalake_version are 'latest' or a semantic versioning conventions."
   }
 
diff --git a/modules/terraform-cdp-deploy/modules/gcp/variables.tf b/modules/terraform-cdp-deploy/modules/gcp/variables.tf
index b0be883..114d0f7 100644
--- a/modules/terraform-cdp-deploy/modules/gcp/variables.tf
+++ b/modules/terraform-cdp-deploy/modules/gcp/variables.tf
@@ -152,7 +152,9 @@ variable "datalake_version" {
   description = "The Datalake Runtime version. Valid values are latest or a semantic version, e.g. 7.2.17"
 
   validation {
-    condition     = (var.datalake_version == "latest" ? true : length(regexall("\\d+\\.\\d+.\\d+", var.datalake_version)) > 0)
+    condition = (var.datalake_version == null ? true :
+      (var.datalake_version == "latest" ? true :
+    length(regexall("\\d+\\.\\d+.\\d+", var.datalake_version)) > 0))
     error_message = "Valid values for var: datalake_version are 'latest' or a semantic versioning conventions."
   }
 
diff --git a/modules/terraform-cdp-deploy/variables.tf b/modules/terraform-cdp-deploy/variables.tf
index 86220c2..bd3b015 100644
--- a/modules/terraform-cdp-deploy/variables.tf
+++ b/modules/terraform-cdp-deploy/variables.tf
@@ -231,7 +231,9 @@ variable "datalake_version" {
   description = "The Datalake Runtime version. Valid values are latest or a semantic version, e.g. 7.2.17"
 
   validation {
-    condition     = (var.datalake_version == "latest" ? true : length(regexall("\\d+\\.\\d+.\\d+", var.datalake_version)) > 0)
+    condition = (var.datalake_version == null ? true :
+      (var.datalake_version == "latest" ? true :
+    length(regexall("\\d+\\.\\d+.\\d+", var.datalake_version)) > 0))
     error_message = "Valid values for var: datalake_version are 'latest' or a semantic versioning conventions."
   }