From a0fe82328b01d0d1c2d132c4ab93ac5768d9d040 Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Fri, 30 Aug 2024 23:00:08 +0000 Subject: [PATCH 01/12] Make `internal_ip_only` a computed, not defaulted, field. The 'default' value for this field is determined on Dataproc's servers, based on the image version used. Users can still provide a value to override the default, but if the user does not provide the value, then the value generated by Dataproc's servers should be used. --- .../terraform/services/dataproc/resource_dataproc_cluster.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmv1/third_party/terraform/services/dataproc/resource_dataproc_cluster.go b/mmv1/third_party/terraform/services/dataproc/resource_dataproc_cluster.go index e6b7c6e27d1a..d86b92a47e20 100644 --- a/mmv1/third_party/terraform/services/dataproc/resource_dataproc_cluster.go +++ b/mmv1/third_party/terraform/services/dataproc/resource_dataproc_cluster.go @@ -650,9 +650,9 @@ func ResourceDataprocCluster() *schema.Resource { "internal_ip_only": { Type: schema.TypeBool, Optional: true, + Computed: true, AtLeastOneOf: gceClusterConfigKeys, ForceNew: true, - Default: false, Description: `By default, clusters are not restricted to internal IP addresses, and will have ephemeral external IP addresses assigned to each instance. If set to true, all instances in the cluster will only have internal IP addresses. Note: Private Google Access (also known as privateIpGoogleAccess) must be enabled on the subnetwork that the cluster will be launched in.`, }, From 00d39dc47f87e9c5cc49fab56692cb43e1927c1a Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Wed, 2 Oct 2024 18:30:55 +0000 Subject: [PATCH 02/12] Always send `internal_ip_only`, if it is set by the user. This is required because the autogenerated Dataproc API client has `omitempty` on the field. So if the user specifies `internal_ip_only: false`, then it would not get sent to Dataproc. Thus Dataproc uses server-side defaulting. We don't want that. If the user specifies the value, we want to ensure that value is used. --- .../dataproc/resource_dataproc_cluster.go | 46 ++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/mmv1/third_party/terraform/services/dataproc/resource_dataproc_cluster.go b/mmv1/third_party/terraform/services/dataproc/resource_dataproc_cluster.go index d86b92a47e20..dd7bf6ff96e8 100644 --- a/mmv1/third_party/terraform/services/dataproc/resource_dataproc_cluster.go +++ b/mmv1/third_party/terraform/services/dataproc/resource_dataproc_cluster.go @@ -2177,8 +2177,15 @@ func expandGceClusterConfig(d *schema.ResourceData, config *transport_tpg.Config } if v, ok := cfg["internal_ip_only"]; ok { conf.InternalIpOnly = v.(bool) - conf.ForceSendFields = append(conf.ForceSendFields, "InternalIpOnly") + if isInternalIpOnlySpecified(d) { + // Normally default values are omitted. Concretely this means that if v is + // false, then it would not get sent on the request to Dataproc. However, + // the user has explicitly set internal_ip_only, therefore we need to send + // it on the wire to ensure Dataproc does not use server-side defaulting. + conf.ForceSendFields = append(conf.ForceSendFields, "InternalIpOnly") + } } + if v, ok := cfg["metadata"]; ok { conf.Metadata = tpgresource.ConvertStringMap(v.(map[string]interface{})) } @@ -2218,6 +2225,43 @@ func expandGceClusterConfig(d *schema.ResourceData, config *transport_tpg.Config return conf, nil } +// isInternalIpOnlySpecified returns true if the original configuration specified by +// the user sets 'cluster_config.0.gce_cluster_config.0.internal_ip_only' to +// any non-nil value (true or false). +func isInternalIpOnlySpecified(d *schema.ResourceData) bool { + rawConfig := d.GetRawConfig() + if !rawConfig.Type().IsObjectType() { + return false + } + if _, ok := rawConfig.Type().AttributeTypes()["cluster_config"]; !ok { + return false + } + clusterConfig := rawConfig.GetAttr("cluster_config") + if !clusterConfig.Type().IsListType() || len(clusterConfig.AsValueSlice()) == 0 { + return false + } + clusterConfig0 := clusterConfig.AsValueSlice()[0] + if !clusterConfig0.Type().IsObjectType() { + return false + } + if _, ok := clusterConfig0.Type().AttributeTypes()["gce_cluster_config"]; !ok { + return false + } + gceClusterConfig := clusterConfig0.GetAttr("gce_cluster_config") + if !gceClusterConfig.Type().IsListType() || len(gceClusterConfig.AsValueSlice()) == 0 { + return false + } + gceClusterConfig0 := gceClusterConfig.AsValueSlice()[0] + if !gceClusterConfig0.Type().IsObjectType() { + return false + } + if _, ok := gceClusterConfig0.Type().AttributeTypes()["internal_ip_only"]; !ok { + return false + } + internalIpOnly := gceClusterConfig0.GetAttr("internal_ip_only") + return !internalIpOnly.IsNull() +} + func expandSecurityConfig(cfg map[string]interface{}) *dataproc.SecurityConfig { conf := &dataproc.SecurityConfig{} if kfg, ok := cfg["kerberos_config"]; ok { From 6b8c01b85b4b570b30176f0762d95862260210d4 Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Fri, 4 Oct 2024 19:38:41 +0000 Subject: [PATCH 03/12] Explicitly set the subnetworks. The switch to Dataproc 2.2 means that internal_ip_only defaults to true, which means the VMs must be on a subnetwork with private IP access. Have all the tests create such a subnet. --- .../resource_dataproc_cluster_test.go.tmpl | 111 +++++++++++++----- .../resource_dataproc_job_test.go.tmpl | 16 +-- 2 files changed, 90 insertions(+), 37 deletions(-) diff --git a/mmv1/third_party/terraform/services/dataproc/resource_dataproc_cluster_test.go.tmpl b/mmv1/third_party/terraform/services/dataproc/resource_dataproc_cluster_test.go.tmpl index 7cb8c98bd466..12f274deda16 100644 --- a/mmv1/third_party/terraform/services/dataproc/resource_dataproc_cluster_test.go.tmpl +++ b/mmv1/third_party/terraform/services/dataproc/resource_dataproc_cluster_test.go.tmpl @@ -58,15 +58,19 @@ func TestAccDataprocCluster_missingZoneGlobalRegion2(t *testing.T) { func TestAccDataprocCluster_basic(t *testing.T) { t.Parallel() - var cluster dataproc.Cluster rnd := acctest.RandString(t, 10) + networkName := acctest.BootstrapSharedTestNetwork(t, "dataproc-cluster") + subnetworkName := acctest.BootstrapSubnetForDataprocBatches(t, "dataproc-cluster", networkName) + acctest.BootstrapFirewallForDataprocSharedNetwork(t, "dataproc-cluster", networkName) + + var cluster dataproc.Cluster acctest.VcrTest(t, resource.TestCase{ PreCheck: func() { acctest.AccTestPreCheck(t) }, ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t), CheckDestroy: testAccCheckDataprocClusterDestroy(t), Steps: []resource.TestStep{ { - Config: testAccDataprocCluster_basic(rnd), + Config: testAccDataprocCluster_basic(rnd, subnetworkName), Check: resource.ComposeTestCheckFunc( testAccCheckDataprocClusterExists(t, "google_dataproc_cluster.basic", &cluster), @@ -112,7 +116,7 @@ func TestAccDataprocVirtualCluster_basic(t *testing.T) { var cluster dataproc.Cluster rnd := acctest.RandString(t, 10) pid := envvar.GetTestProjectFromEnv() - version := "3.1-dataproc-7" + version := "3.5-dataproc-19" networkName := acctest.BootstrapSharedTestNetwork(t, "gke-cluster") subnetworkName := acctest.BootstrapSubnet(t, "gke-cluster", networkName) @@ -122,7 +126,7 @@ func TestAccDataprocVirtualCluster_basic(t *testing.T) { CheckDestroy: testAccCheckDataprocClusterDestroy(t), Steps: []resource.TestStep{ { - Config: testAccDataprocVirtualCluster_basic(pid, rnd, networkName, subnetworkName), + Config: testAccDataprocVirtualCluster_basic(pid, rnd, networkName, subnetworkName, version), Check: resource.ComposeTestCheckFunc( testAccCheckDataprocClusterExists(t, "google_dataproc_cluster.virtual_cluster", &cluster), @@ -426,15 +430,18 @@ func TestAccDataprocCluster_updatable(t *testing.T) { t.Parallel() rnd := acctest.RandString(t, 10) - var cluster dataproc.Cluster + networkName := acctest.BootstrapSharedTestNetwork(t, "dataproc-cluster") + subnetworkName := acctest.BootstrapSubnetForDataprocBatches(t, "dataproc-cluster", networkName) + acctest.BootstrapFirewallForDataprocSharedNetwork(t, "dataproc-cluster", networkName) + var cluster dataproc.Cluster acctest.VcrTest(t, resource.TestCase{ PreCheck: func() { acctest.AccTestPreCheck(t) }, ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t), CheckDestroy: testAccCheckDataprocClusterDestroy(t), Steps: []resource.TestStep{ { - Config: testAccDataprocCluster_updatable(rnd, 2, 1), + Config: testAccDataprocCluster_updatable(rnd, subnetworkName, 2, 1), Check: resource.ComposeTestCheckFunc( testAccCheckDataprocClusterExists(t, "google_dataproc_cluster.updatable", &cluster), resource.TestCheckResourceAttr("google_dataproc_cluster.updatable", "cluster_config.0.master_config.0.num_instances", "1"), @@ -442,7 +449,7 @@ func TestAccDataprocCluster_updatable(t *testing.T) { resource.TestCheckResourceAttr("google_dataproc_cluster.updatable", "cluster_config.0.preemptible_worker_config.0.num_instances", "1")), }, { - Config: testAccDataprocCluster_updatable(rnd, 2, 0), + Config: testAccDataprocCluster_updatable(rnd, subnetworkName, 2, 0), Check: resource.ComposeTestCheckFunc( testAccCheckDataprocClusterExists(t, "google_dataproc_cluster.updatable", &cluster), resource.TestCheckResourceAttr("google_dataproc_cluster.updatable", "cluster_config.0.master_config.0.num_instances", "1"), @@ -450,7 +457,7 @@ func TestAccDataprocCluster_updatable(t *testing.T) { resource.TestCheckResourceAttr("google_dataproc_cluster.updatable", "cluster_config.0.preemptible_worker_config.0.num_instances", "0")), }, { - Config: testAccDataprocCluster_updatable(rnd, 3, 2), + Config: testAccDataprocCluster_updatable(rnd, subnetworkName, 3, 2), Check: resource.ComposeTestCheckFunc( resource.TestCheckResourceAttr("google_dataproc_cluster.updatable", "cluster_config.0.master_config.0.num_instances", "1"), resource.TestCheckResourceAttr("google_dataproc_cluster.updatable", "cluster_config.0.worker_config.0.num_instances", "3"), @@ -514,6 +521,10 @@ func TestAccDataprocCluster_spotWithInstanceFlexibilityPolicy(t *testing.T) { t.Parallel() rnd := acctest.RandString(t, 10) + networkName := acctest.BootstrapSharedTestNetwork(t, "dataproc-cluster") + subnetworkName := acctest.BootstrapSubnetForDataprocBatches(t, "dataproc-cluster", networkName) + acctest.BootstrapFirewallForDataprocSharedNetwork(t, "dataproc-cluster", networkName) + var cluster dataproc.Cluster acctest.VcrTest(t, resource.TestCase{ PreCheck: func() { acctest.AccTestPreCheck(t) }, @@ -521,7 +532,7 @@ func TestAccDataprocCluster_spotWithInstanceFlexibilityPolicy(t *testing.T) { CheckDestroy: testAccCheckDataprocClusterDestroy(t), Steps: []resource.TestStep{ { - Config: testAccDataprocCluster_spotWithInstanceFlexibilityPolicy(rnd), + Config: testAccDataprocCluster_spotWithInstanceFlexibilityPolicy(rnd, subnetworkName), Check: resource.ComposeTestCheckFunc( testAccCheckDataprocClusterExists(t, "google_dataproc_cluster.spot_with_instance_flexibility_policy", &cluster), resource.TestCheckResourceAttr("google_dataproc_cluster.spot_with_instance_flexibility_policy", "cluster_config.0.preemptible_worker_config.0.preemptibility", "SPOT"), @@ -538,6 +549,10 @@ func TestAccDataprocCluster_spotWithAuxiliaryNodeGroups(t *testing.T) { project := envvar.GetTestProjectFromEnv() rnd := acctest.RandString(t, 10) + networkName := acctest.BootstrapSharedTestNetwork(t, "dataproc-cluster") + subnetworkName := acctest.BootstrapSubnetForDataprocBatches(t, "dataproc-cluster", networkName) + acctest.BootstrapFirewallForDataprocSharedNetwork(t, "dataproc-cluster", networkName) + var cluster dataproc.Cluster acctest.VcrTest(t, resource.TestCase{ PreCheck: func() { acctest.AccTestPreCheck(t) }, @@ -545,7 +560,7 @@ func TestAccDataprocCluster_spotWithAuxiliaryNodeGroups(t *testing.T) { CheckDestroy: testAccCheckDataprocClusterDestroy(t), Steps: []resource.TestStep{ { - Config: testAccDataprocCluster_withAuxiliaryNodeGroups(rnd), + Config: testAccDataprocCluster_withAuxiliaryNodeGroups(rnd, subnetworkName), Check: resource.ComposeTestCheckFunc( testAccCheckDataprocClusterExists(t, "google_dataproc_cluster.with_auxiliary_node_groups", &cluster), resource.TestCheckResourceAttr("google_dataproc_cluster.with_auxiliary_node_groups", "cluster_config.0.auxiliary_node_groups.0.node_group.0.roles.0", "DRIVER"), @@ -1046,22 +1061,26 @@ func TestAccDataprocCluster_withMetastoreConfig(t *testing.T) { msName_basic := fmt.Sprintf("projects/%s/locations/us-central1/services/%s", pid, basicServiceId) msName_update := fmt.Sprintf("projects/%s/locations/us-central1/services/%s", pid, updateServiceId) - var cluster dataproc.Cluster clusterName := "tf-test-" + acctest.RandString(t, 10) + networkName := acctest.BootstrapSharedTestNetwork(t, "dataproc-cluster") + subnetworkName := acctest.BootstrapSubnetForDataprocBatches(t, "dataproc-cluster", networkName) + acctest.BootstrapFirewallForDataprocSharedNetwork(t, "dataproc-cluster", networkName) + + var cluster dataproc.Cluster acctest.VcrTest(t, resource.TestCase{ PreCheck: func() { acctest.AccTestPreCheck(t) }, ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t), CheckDestroy: testAccCheckDataprocClusterDestroy(t), Steps: []resource.TestStep{ { - Config: testAccDataprocCluster_withMetastoreConfig(clusterName, basicServiceId), + Config: testAccDataprocCluster_withMetastoreConfig(clusterName, subnetworkName, basicServiceId), Check: resource.ComposeTestCheckFunc( testAccCheckDataprocClusterExists(t, "google_dataproc_cluster.with_metastore_config", &cluster), resource.TestCheckResourceAttr("google_dataproc_cluster.with_metastore_config", "cluster_config.0.metastore_config.0.dataproc_metastore_service", msName_basic), ), }, { - Config: testAccDataprocCluster_withMetastoreConfig_update(clusterName, updateServiceId), + Config: testAccDataprocCluster_withMetastoreConfig_update(clusterName, subnetworkName, updateServiceId), Check: resource.ComposeTestCheckFunc( testAccCheckDataprocClusterExists(t, "google_dataproc_cluster.with_metastore_config", &cluster), resource.TestCheckResourceAttr("google_dataproc_cluster.with_metastore_config", "cluster_config.0.metastore_config.0.dataproc_metastore_service", msName_update), @@ -1323,16 +1342,22 @@ resource "google_dataproc_cluster" "basic" { `, rnd) } -func testAccDataprocCluster_basic(rnd string) string { +func testAccDataprocCluster_basic(rnd, subnetworkName string) string { return fmt.Sprintf(` resource "google_dataproc_cluster" "basic" { name = "tf-test-dproc-%s" region = "us-central1" + + cluster_config { + gce_cluster_config { + subnetwork = "%s" + } + } } -`, rnd) +`, rnd, subnetworkName) } -func testAccDataprocVirtualCluster_basic(projectID, rnd, networkName, subnetworkName string) string { +func testAccDataprocVirtualCluster_basic(projectID, rnd, networkName, subnetworkName, version string) string { return fmt.Sprintf(` data "google_project" "project" { project_id = "%s" @@ -1376,7 +1401,7 @@ resource "google_dataproc_cluster" "virtual_cluster" { kubernetes_namespace = "tf-test-dproc-%s" kubernetes_software_config { component_version = { - "SPARK": "3.1-dataproc-7", + "SPARK": "%s", } } gke_cluster_config { @@ -1391,7 +1416,7 @@ resource "google_dataproc_cluster" "virtual_cluster" { } } } -`, projectID, rnd, networkName, subnetworkName, projectID, rnd, rnd, rnd, rnd, rnd, rnd) +`, projectID, rnd, networkName, subnetworkName, projectID, rnd, rnd, rnd, rnd, rnd, version, rnd) } func testAccCheckDataprocGkeClusterNodePoolsHaveRoles(cluster *dataproc.Cluster, roles ...string) func(s *terraform.State) error { @@ -1424,6 +1449,8 @@ resource "google_dataproc_cluster" "accelerated_cluster" { } master_config { + # The default machine family is n2, which does not support (most) accelerators. + machine_type = "n1-standard-2" accelerators { accelerator_type = "%s" accelerator_count = "1" @@ -1431,6 +1458,8 @@ resource "google_dataproc_cluster" "accelerated_cluster" { } worker_config { + # The default machine family is n2, which does not support (most) accelerators. + machine_type = "n1-standard-2" accelerators { accelerator_type = "%s" accelerator_count = "1" @@ -1635,7 +1664,7 @@ resource "google_compute_node_template" "nodetmpl" { tfacc = "test" } - node_type = "n1-node-96-624" + node_type = "n2-node-80-640" cpu_overcommit_type = "ENABLED" } @@ -1793,7 +1822,7 @@ resource "google_dataproc_cluster" "with_init_action" { `, bucket, rnd, objName, objName, rnd, subnetworkName) } -func testAccDataprocCluster_updatable(rnd string, w, p int) string { +func testAccDataprocCluster_updatable(rnd, subnetworkName string, w, p int) string { return fmt.Sprintf(` resource "google_dataproc_cluster" "updatable" { name = "tf-test-dproc-%s" @@ -1801,6 +1830,10 @@ resource "google_dataproc_cluster" "updatable" { graceful_decommission_timeout = "0.2s" cluster_config { + gce_cluster_config { + subnetwork = "%s" + } + master_config { num_instances = "1" machine_type = "e2-medium" @@ -1825,7 +1858,7 @@ resource "google_dataproc_cluster" "updatable" { } } } -`, rnd, w, p) +`, rnd, subnetworkName, w, p) } func testAccDataprocCluster_nonPreemptibleSecondary(rnd, subnetworkName string) string { @@ -1906,13 +1939,17 @@ resource "google_dataproc_cluster" "spot_secondary" { `, rnd, subnetworkName) } -func testAccDataprocCluster_spotWithInstanceFlexibilityPolicy(rnd string) string { +func testAccDataprocCluster_spotWithInstanceFlexibilityPolicy(rnd, subnetworkName string) string { return fmt.Sprintf(` resource "google_dataproc_cluster" "spot_with_instance_flexibility_policy" { name = "tf-test-dproc-%s" region = "us-central1" cluster_config { + gce_cluster_config { + subnetwork = "%s" + } + master_config { num_instances = "1" machine_type = "e2-medium" @@ -1944,16 +1981,20 @@ resource "google_dataproc_cluster" "spot_with_instance_flexibility_policy" { } } } - `, rnd) + `, rnd, subnetworkName) } -func testAccDataprocCluster_withAuxiliaryNodeGroups(rnd string) string { +func testAccDataprocCluster_withAuxiliaryNodeGroups(rnd, subnetworkName string) string { return fmt.Sprintf(` resource "google_dataproc_cluster" "with_auxiliary_node_groups" { name = "tf-test-dproc-%s" region = "us-central1" cluster_config { + gce_cluster_config { + subnetwork = "%s" + } + master_config { num_instances = "1" machine_type = "e2-medium" @@ -1993,7 +2034,7 @@ resource "google_dataproc_cluster" "with_auxiliary_node_groups" { } } } - `, rnd) + `, rnd, subnetworkName) } func testAccDataprocCluster_withStagingBucketOnly(bucketName string) string { @@ -2351,6 +2392,9 @@ resource "google_dataproc_cluster" "with_net_ref_by_name" { gce_cluster_config { network = google_compute_network.dataproc_network.name + # The network we are creating does not have private Google access, so it + # cannot support internal IPs only. + internal_ip_only = false } } } @@ -2378,6 +2422,9 @@ resource "google_dataproc_cluster" "with_net_ref_by_url" { gce_cluster_config { network = google_compute_network.dataproc_network.self_link + # The network we are creating does not have private Google access, so it + # cannot support internal IPs only. + internal_ip_only = false } } } @@ -2507,13 +2554,16 @@ resource "google_dataproc_autoscaling_policy" "asp" { `, rnd, subnetworkName, rnd) } -func testAccDataprocCluster_withMetastoreConfig(clusterName, serviceId string) string { +func testAccDataprocCluster_withMetastoreConfig(clusterName, subnetworkName, serviceId string) string { return fmt.Sprintf(` resource "google_dataproc_cluster" "with_metastore_config" { name = "%s" region = "us-central1" cluster_config { + gce_cluster_config { + subnetwork = "%s" + } metastore_config { dataproc_metastore_service = google_dataproc_metastore_service.ms.name } @@ -2535,16 +2585,19 @@ resource "google_dataproc_metastore_service" "ms" { version = "3.1.2" } } -`, clusterName, serviceId) +`, clusterName, subnetworkName, serviceId) } -func testAccDataprocCluster_withMetastoreConfig_update(clusterName, serviceId string) string { +func testAccDataprocCluster_withMetastoreConfig_update(clusterName, subnetworkName, serviceId string) string { return fmt.Sprintf(` resource "google_dataproc_cluster" "with_metastore_config" { name = "%s" region = "us-central1" cluster_config { + gce_cluster_config { + subnetwork = "%s" + } metastore_config { dataproc_metastore_service = google_dataproc_metastore_service.ms.name } @@ -2566,6 +2619,6 @@ resource "google_dataproc_metastore_service" "ms" { version = "3.1.2" } } -`, clusterName, serviceId) +`, clusterName, subnetworkName, serviceId) } diff --git a/mmv1/third_party/terraform/services/dataproc/resource_dataproc_job_test.go.tmpl b/mmv1/third_party/terraform/services/dataproc/resource_dataproc_job_test.go.tmpl index 247e6b5c834b..aeff5bce147e 100644 --- a/mmv1/third_party/terraform/services/dataproc/resource_dataproc_job_test.go.tmpl +++ b/mmv1/third_party/terraform/services/dataproc/resource_dataproc_job_test.go.tmpl @@ -50,7 +50,7 @@ func TestAccDataprocJob_updatable(t *testing.T) { rnd := acctest.RandString(t, 10) jobId := fmt.Sprintf("dproc-update-job-id-%s", rnd) networkName := acctest.BootstrapSharedTestNetwork(t, "dataproc-cluster") - subnetworkName := acctest.BootstrapSubnet(t, "dataproc-cluster", networkName) + subnetworkName := acctest.BootstrapSubnetForDataprocBatches(t, "dataproc-cluster", networkName) acctest.BootstrapFirewallForDataprocSharedNetwork(t, "dataproc-cluster", networkName) acctest.VcrTest(t, resource.TestCase{ @@ -83,7 +83,7 @@ func TestAccDataprocJob_PySpark(t *testing.T) { rnd := acctest.RandString(t, 10) jobId := fmt.Sprintf("dproc-custom-job-id-%s", rnd) networkName := acctest.BootstrapSharedTestNetwork(t, "dataproc-cluster") - subnetworkName := acctest.BootstrapSubnet(t, "dataproc-cluster", networkName) + subnetworkName := acctest.BootstrapSubnetForDataprocBatches(t, "dataproc-cluster", networkName) acctest.BootstrapFirewallForDataprocSharedNetwork(t, "dataproc-cluster", networkName) acctest.VcrTest(t, resource.TestCase{ @@ -125,7 +125,7 @@ func TestAccDataprocJob_Spark(t *testing.T) { var job dataproc.Job rnd := acctest.RandString(t, 10) networkName := acctest.BootstrapSharedTestNetwork(t, "dataproc-cluster") - subnetworkName := acctest.BootstrapSubnet(t, "dataproc-cluster", networkName) + subnetworkName := acctest.BootstrapSubnetForDataprocBatches(t, "dataproc-cluster", networkName) acctest.BootstrapFirewallForDataprocSharedNetwork(t, "dataproc-cluster", networkName) acctest.VcrTest(t, resource.TestCase{ @@ -161,7 +161,7 @@ func TestAccDataprocJob_Hadoop(t *testing.T) { var job dataproc.Job rnd := acctest.RandString(t, 10) networkName := acctest.BootstrapSharedTestNetwork(t, "dataproc-cluster") - subnetworkName := acctest.BootstrapSubnet(t, "dataproc-cluster", networkName) + subnetworkName := acctest.BootstrapSubnetForDataprocBatches(t, "dataproc-cluster", networkName) acctest.BootstrapFirewallForDataprocSharedNetwork(t, "dataproc-cluster", networkName) acctest.VcrTest(t, resource.TestCase{ @@ -197,7 +197,7 @@ func TestAccDataprocJob_Hive(t *testing.T) { var job dataproc.Job rnd := acctest.RandString(t, 10) networkName := acctest.BootstrapSharedTestNetwork(t, "dataproc-cluster") - subnetworkName := acctest.BootstrapSubnet(t, "dataproc-cluster", networkName) + subnetworkName := acctest.BootstrapSubnetForDataprocBatches(t, "dataproc-cluster", networkName) acctest.BootstrapFirewallForDataprocSharedNetwork(t, "dataproc-cluster", networkName) acctest.VcrTest(t, resource.TestCase{ @@ -233,7 +233,7 @@ func TestAccDataprocJob_Pig(t *testing.T) { var job dataproc.Job rnd := acctest.RandString(t, 10) networkName := acctest.BootstrapSharedTestNetwork(t, "dataproc-cluster") - subnetworkName := acctest.BootstrapSubnet(t, "dataproc-cluster", networkName) + subnetworkName := acctest.BootstrapSubnetForDataprocBatches(t, "dataproc-cluster", networkName) acctest.BootstrapFirewallForDataprocSharedNetwork(t, "dataproc-cluster", networkName) acctest.VcrTest(t, resource.TestCase{ @@ -269,7 +269,7 @@ func TestAccDataprocJob_SparkSql(t *testing.T) { var job dataproc.Job rnd := acctest.RandString(t, 10) networkName := acctest.BootstrapSharedTestNetwork(t, "dataproc-cluster") - subnetworkName := acctest.BootstrapSubnet(t, "dataproc-cluster", networkName) + subnetworkName := acctest.BootstrapSubnetForDataprocBatches(t, "dataproc-cluster", networkName) acctest.BootstrapFirewallForDataprocSharedNetwork(t, "dataproc-cluster", networkName) acctest.VcrTest(t, resource.TestCase{ @@ -305,7 +305,7 @@ func TestAccDataprocJob_Presto(t *testing.T) { var job dataproc.Job rnd := acctest.RandString(t, 10) networkName := acctest.BootstrapSharedTestNetwork(t, "dataproc-cluster") - subnetworkName := acctest.BootstrapSubnet(t, "dataproc-cluster", networkName) + subnetworkName := acctest.BootstrapSubnetForDataprocBatches(t, "dataproc-cluster", networkName) acctest.BootstrapFirewallForDataprocSharedNetwork(t, "dataproc-cluster", networkName) acctest.VcrTest(t, resource.TestCase{ From 73cacbd9de1a16d356869da6faff73c4eb187240 Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Mon, 7 Oct 2024 19:49:53 +0000 Subject: [PATCH 04/12] Use a different subnet name, so that we can create new subnets with internal Google access enabled. --- .../terraform/acctest/bootstrap_test_utils.go | 2 +- .../resource_dataproc_cluster_test.go.tmpl | 12 ++++++------ .../dataproc/resource_dataproc_job_test.go.tmpl | 16 ++++++++-------- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/mmv1/third_party/terraform/acctest/bootstrap_test_utils.go b/mmv1/third_party/terraform/acctest/bootstrap_test_utils.go index 1c5d4cc04519..44dc8d233114 100644 --- a/mmv1/third_party/terraform/acctest/bootstrap_test_utils.go +++ b/mmv1/third_party/terraform/acctest/bootstrap_test_utils.go @@ -955,7 +955,7 @@ func BootstrapSubnetWithOverrides(t *testing.T, subnetName string, networkName s "name": subnetName, "region ": region, "network": networkUrl, - "ipCidrRange": "10.77.0.0/20", + "ipCidrRange": "10.78.0.0/20", } if len(subnetOptions) != 0 { diff --git a/mmv1/third_party/terraform/services/dataproc/resource_dataproc_cluster_test.go.tmpl b/mmv1/third_party/terraform/services/dataproc/resource_dataproc_cluster_test.go.tmpl index 12f274deda16..c7d2f0ab3ad2 100644 --- a/mmv1/third_party/terraform/services/dataproc/resource_dataproc_cluster_test.go.tmpl +++ b/mmv1/third_party/terraform/services/dataproc/resource_dataproc_cluster_test.go.tmpl @@ -60,7 +60,7 @@ func TestAccDataprocCluster_basic(t *testing.T) { rnd := acctest.RandString(t, 10) networkName := acctest.BootstrapSharedTestNetwork(t, "dataproc-cluster") - subnetworkName := acctest.BootstrapSubnetForDataprocBatches(t, "dataproc-cluster", networkName) + subnetworkName := acctest.BootstrapSubnetForDataprocBatches(t, "dataproc-cluster-internal", networkName) acctest.BootstrapFirewallForDataprocSharedNetwork(t, "dataproc-cluster", networkName) var cluster dataproc.Cluster @@ -78,7 +78,7 @@ func TestAccDataprocCluster_basic(t *testing.T) { resource.TestCheckResourceAttrSet("google_dataproc_cluster.basic", "cluster_config.0.bucket"), // Default behavior is for Dataproc to not use only internal IP addresses - resource.TestCheckResourceAttr("google_dataproc_cluster.basic", "cluster_config.0.gce_cluster_config.0.internal_ip_only", "false"), + resource.TestCheckResourceAttr("google_dataproc_cluster.basic", "cluster_config.0.gce_cluster_config.0.internal_ip_only", "true"), // Expect 1 master instances with computed values resource.TestCheckResourceAttr("google_dataproc_cluster.basic", "cluster_config.0.master_config.#", "1"), @@ -431,7 +431,7 @@ func TestAccDataprocCluster_updatable(t *testing.T) { rnd := acctest.RandString(t, 10) networkName := acctest.BootstrapSharedTestNetwork(t, "dataproc-cluster") - subnetworkName := acctest.BootstrapSubnetForDataprocBatches(t, "dataproc-cluster", networkName) + subnetworkName := acctest.BootstrapSubnetForDataprocBatches(t, "dataproc-cluster-internal", networkName) acctest.BootstrapFirewallForDataprocSharedNetwork(t, "dataproc-cluster", networkName) var cluster dataproc.Cluster @@ -522,7 +522,7 @@ func TestAccDataprocCluster_spotWithInstanceFlexibilityPolicy(t *testing.T) { rnd := acctest.RandString(t, 10) networkName := acctest.BootstrapSharedTestNetwork(t, "dataproc-cluster") - subnetworkName := acctest.BootstrapSubnetForDataprocBatches(t, "dataproc-cluster", networkName) + subnetworkName := acctest.BootstrapSubnetForDataprocBatches(t, "dataproc-cluster-internal", networkName) acctest.BootstrapFirewallForDataprocSharedNetwork(t, "dataproc-cluster", networkName) var cluster dataproc.Cluster @@ -550,7 +550,7 @@ func TestAccDataprocCluster_spotWithAuxiliaryNodeGroups(t *testing.T) { project := envvar.GetTestProjectFromEnv() rnd := acctest.RandString(t, 10) networkName := acctest.BootstrapSharedTestNetwork(t, "dataproc-cluster") - subnetworkName := acctest.BootstrapSubnetForDataprocBatches(t, "dataproc-cluster", networkName) + subnetworkName := acctest.BootstrapSubnetForDataprocBatches(t, "dataproc-cluster-internal", networkName) acctest.BootstrapFirewallForDataprocSharedNetwork(t, "dataproc-cluster", networkName) var cluster dataproc.Cluster @@ -1063,7 +1063,7 @@ func TestAccDataprocCluster_withMetastoreConfig(t *testing.T) { clusterName := "tf-test-" + acctest.RandString(t, 10) networkName := acctest.BootstrapSharedTestNetwork(t, "dataproc-cluster") - subnetworkName := acctest.BootstrapSubnetForDataprocBatches(t, "dataproc-cluster", networkName) + subnetworkName := acctest.BootstrapSubnetForDataprocBatches(t, "dataproc-cluster-internal", networkName) acctest.BootstrapFirewallForDataprocSharedNetwork(t, "dataproc-cluster", networkName) var cluster dataproc.Cluster diff --git a/mmv1/third_party/terraform/services/dataproc/resource_dataproc_job_test.go.tmpl b/mmv1/third_party/terraform/services/dataproc/resource_dataproc_job_test.go.tmpl index aeff5bce147e..6005a8d9c131 100644 --- a/mmv1/third_party/terraform/services/dataproc/resource_dataproc_job_test.go.tmpl +++ b/mmv1/third_party/terraform/services/dataproc/resource_dataproc_job_test.go.tmpl @@ -50,7 +50,7 @@ func TestAccDataprocJob_updatable(t *testing.T) { rnd := acctest.RandString(t, 10) jobId := fmt.Sprintf("dproc-update-job-id-%s", rnd) networkName := acctest.BootstrapSharedTestNetwork(t, "dataproc-cluster") - subnetworkName := acctest.BootstrapSubnetForDataprocBatches(t, "dataproc-cluster", networkName) + subnetworkName := acctest.BootstrapSubnetForDataprocBatches(t, "dataproc-cluster-internal", networkName) acctest.BootstrapFirewallForDataprocSharedNetwork(t, "dataproc-cluster", networkName) acctest.VcrTest(t, resource.TestCase{ @@ -83,7 +83,7 @@ func TestAccDataprocJob_PySpark(t *testing.T) { rnd := acctest.RandString(t, 10) jobId := fmt.Sprintf("dproc-custom-job-id-%s", rnd) networkName := acctest.BootstrapSharedTestNetwork(t, "dataproc-cluster") - subnetworkName := acctest.BootstrapSubnetForDataprocBatches(t, "dataproc-cluster", networkName) + subnetworkName := acctest.BootstrapSubnetForDataprocBatches(t, "dataproc-cluster-internal", networkName) acctest.BootstrapFirewallForDataprocSharedNetwork(t, "dataproc-cluster", networkName) acctest.VcrTest(t, resource.TestCase{ @@ -125,7 +125,7 @@ func TestAccDataprocJob_Spark(t *testing.T) { var job dataproc.Job rnd := acctest.RandString(t, 10) networkName := acctest.BootstrapSharedTestNetwork(t, "dataproc-cluster") - subnetworkName := acctest.BootstrapSubnetForDataprocBatches(t, "dataproc-cluster", networkName) + subnetworkName := acctest.BootstrapSubnetForDataprocBatches(t, "dataproc-cluster-internal", networkName) acctest.BootstrapFirewallForDataprocSharedNetwork(t, "dataproc-cluster", networkName) acctest.VcrTest(t, resource.TestCase{ @@ -161,7 +161,7 @@ func TestAccDataprocJob_Hadoop(t *testing.T) { var job dataproc.Job rnd := acctest.RandString(t, 10) networkName := acctest.BootstrapSharedTestNetwork(t, "dataproc-cluster") - subnetworkName := acctest.BootstrapSubnetForDataprocBatches(t, "dataproc-cluster", networkName) + subnetworkName := acctest.BootstrapSubnetForDataprocBatches(t, "dataproc-cluster-internal", networkName) acctest.BootstrapFirewallForDataprocSharedNetwork(t, "dataproc-cluster", networkName) acctest.VcrTest(t, resource.TestCase{ @@ -197,7 +197,7 @@ func TestAccDataprocJob_Hive(t *testing.T) { var job dataproc.Job rnd := acctest.RandString(t, 10) networkName := acctest.BootstrapSharedTestNetwork(t, "dataproc-cluster") - subnetworkName := acctest.BootstrapSubnetForDataprocBatches(t, "dataproc-cluster", networkName) + subnetworkName := acctest.BootstrapSubnetForDataprocBatches(t, "dataproc-cluster-internal", networkName) acctest.BootstrapFirewallForDataprocSharedNetwork(t, "dataproc-cluster", networkName) acctest.VcrTest(t, resource.TestCase{ @@ -233,7 +233,7 @@ func TestAccDataprocJob_Pig(t *testing.T) { var job dataproc.Job rnd := acctest.RandString(t, 10) networkName := acctest.BootstrapSharedTestNetwork(t, "dataproc-cluster") - subnetworkName := acctest.BootstrapSubnetForDataprocBatches(t, "dataproc-cluster", networkName) + subnetworkName := acctest.BootstrapSubnetForDataprocBatches(t, "dataproc-cluster-internal", networkName) acctest.BootstrapFirewallForDataprocSharedNetwork(t, "dataproc-cluster", networkName) acctest.VcrTest(t, resource.TestCase{ @@ -269,7 +269,7 @@ func TestAccDataprocJob_SparkSql(t *testing.T) { var job dataproc.Job rnd := acctest.RandString(t, 10) networkName := acctest.BootstrapSharedTestNetwork(t, "dataproc-cluster") - subnetworkName := acctest.BootstrapSubnetForDataprocBatches(t, "dataproc-cluster", networkName) + subnetworkName := acctest.BootstrapSubnetForDataprocBatches(t, "dataproc-cluster-internal", networkName) acctest.BootstrapFirewallForDataprocSharedNetwork(t, "dataproc-cluster", networkName) acctest.VcrTest(t, resource.TestCase{ @@ -305,7 +305,7 @@ func TestAccDataprocJob_Presto(t *testing.T) { var job dataproc.Job rnd := acctest.RandString(t, 10) networkName := acctest.BootstrapSharedTestNetwork(t, "dataproc-cluster") - subnetworkName := acctest.BootstrapSubnetForDataprocBatches(t, "dataproc-cluster", networkName) + subnetworkName := acctest.BootstrapSubnetForDataprocBatches(t, "dataproc-cluster-internal", networkName) acctest.BootstrapFirewallForDataprocSharedNetwork(t, "dataproc-cluster", networkName) acctest.VcrTest(t, resource.TestCase{ From c15dc4c12bda5430f29b48ac9856581c78bf2a9f Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Mon, 7 Oct 2024 21:20:02 +0000 Subject: [PATCH 05/12] Set Presto test to Dataproc image 2.0. --- .../services/dataproc/resource_dataproc_job_test.go.tmpl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mmv1/third_party/terraform/services/dataproc/resource_dataproc_job_test.go.tmpl b/mmv1/third_party/terraform/services/dataproc/resource_dataproc_job_test.go.tmpl index 6005a8d9c131..4578a16dd722 100644 --- a/mmv1/third_party/terraform/services/dataproc/resource_dataproc_job_test.go.tmpl +++ b/mmv1/third_party/terraform/services/dataproc/resource_dataproc_job_test.go.tmpl @@ -307,7 +307,7 @@ func TestAccDataprocJob_Presto(t *testing.T) { networkName := acctest.BootstrapSharedTestNetwork(t, "dataproc-cluster") subnetworkName := acctest.BootstrapSubnetForDataprocBatches(t, "dataproc-cluster-internal", networkName) acctest.BootstrapFirewallForDataprocSharedNetwork(t, "dataproc-cluster", networkName) - + acctest.VcrTest(t, resource.TestCase{ PreCheck: func() { acctest.AccTestPreCheck(t) }, ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t), @@ -866,6 +866,8 @@ resource "google_dataproc_cluster" "basic" { override_properties = { "dataproc:dataproc.allow.zero.workers" = "true" } + # 2.1 removes Presto in favor of Trino. So use 2.0 for the Presto test. + image_version = "2.0" optional_components = ["PRESTO"] } From 6db10e5c9d3e5a9b9be2080385aa031c045ec987 Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Tue, 8 Oct 2024 04:31:30 +0000 Subject: [PATCH 06/12] DPMS on the same subnet. --- .../resource_dataproc_cluster_test.go.tmpl | 24 ++++++++++++++----- 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/mmv1/third_party/terraform/services/dataproc/resource_dataproc_cluster_test.go.tmpl b/mmv1/third_party/terraform/services/dataproc/resource_dataproc_cluster_test.go.tmpl index c7d2f0ab3ad2..33c3c6a17c64 100644 --- a/mmv1/third_party/terraform/services/dataproc/resource_dataproc_cluster_test.go.tmpl +++ b/mmv1/third_party/terraform/services/dataproc/resource_dataproc_cluster_test.go.tmpl @@ -1073,14 +1073,14 @@ func TestAccDataprocCluster_withMetastoreConfig(t *testing.T) { CheckDestroy: testAccCheckDataprocClusterDestroy(t), Steps: []resource.TestStep{ { - Config: testAccDataprocCluster_withMetastoreConfig(clusterName, subnetworkName, basicServiceId), + Config: testAccDataprocCluster_withMetastoreConfig(pid, clusterName, subnetworkName, basicServiceId), Check: resource.ComposeTestCheckFunc( testAccCheckDataprocClusterExists(t, "google_dataproc_cluster.with_metastore_config", &cluster), resource.TestCheckResourceAttr("google_dataproc_cluster.with_metastore_config", "cluster_config.0.metastore_config.0.dataproc_metastore_service", msName_basic), ), }, { - Config: testAccDataprocCluster_withMetastoreConfig_update(clusterName, subnetworkName, updateServiceId), + Config: testAccDataprocCluster_withMetastoreConfig_update(pid, clusterName, subnetworkName, updateServiceId), Check: resource.ComposeTestCheckFunc( testAccCheckDataprocClusterExists(t, "google_dataproc_cluster.with_metastore_config", &cluster), resource.TestCheckResourceAttr("google_dataproc_cluster.with_metastore_config", "cluster_config.0.metastore_config.0.dataproc_metastore_service", msName_update), @@ -2554,7 +2554,7 @@ resource "google_dataproc_autoscaling_policy" "asp" { `, rnd, subnetworkName, rnd) } -func testAccDataprocCluster_withMetastoreConfig(clusterName, subnetworkName, serviceId string) string { +func testAccDataprocCluster_withMetastoreConfig(projectName, clusterName, subnetworkName, serviceId string) string { return fmt.Sprintf(` resource "google_dataproc_cluster" "with_metastore_config" { name = "%s" @@ -2584,11 +2584,17 @@ resource "google_dataproc_metastore_service" "ms" { hive_metastore_config { version = "3.1.2" } + + network_config { + consumers { + subnetwork = "projects/%s/regions/us-central1/subnetworks/%s" + } + } } -`, clusterName, subnetworkName, serviceId) +`, clusterName, subnetworkName, serviceId, projectName, subnetworkName) } -func testAccDataprocCluster_withMetastoreConfig_update(clusterName, subnetworkName, serviceId string) string { +func testAccDataprocCluster_withMetastoreConfig_update(projectName, clusterName, subnetworkName, serviceId string) string { return fmt.Sprintf(` resource "google_dataproc_cluster" "with_metastore_config" { name = "%s" @@ -2618,7 +2624,13 @@ resource "google_dataproc_metastore_service" "ms" { hive_metastore_config { version = "3.1.2" } + + network_config { + consumers { + subnetwork = "projects/%s/regions/us-central1/subnetworks/%s" + } + } } -`, clusterName, subnetworkName, serviceId) +`, clusterName, subnetworkName, serviceId, projectName, subnetworkName) } From 5ce3e1ee490761a15ffe09328a85886953d3d3c1 Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Tue, 8 Oct 2024 06:51:32 +0000 Subject: [PATCH 07/12] Switch from unsupported e2-medium to n1-standard-2. --- .../resource_dataproc_cluster_test.go.tmpl | 32 +++++++++---------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/mmv1/third_party/terraform/services/dataproc/resource_dataproc_cluster_test.go.tmpl b/mmv1/third_party/terraform/services/dataproc/resource_dataproc_cluster_test.go.tmpl index 33c3c6a17c64..0f5151e83c4a 100644 --- a/mmv1/third_party/terraform/services/dataproc/resource_dataproc_cluster_test.go.tmpl +++ b/mmv1/third_party/terraform/services/dataproc/resource_dataproc_cluster_test.go.tmpl @@ -1804,7 +1804,7 @@ resource "google_dataproc_cluster" "with_init_action" { } master_config { - machine_type = "e2-medium" + machine_type = "n1-standard-2" disk_config { boot_disk_size_gb = 35 } @@ -1836,7 +1836,7 @@ resource "google_dataproc_cluster" "updatable" { master_config { num_instances = "1" - machine_type = "e2-medium" + machine_type = "n1-standard-2" disk_config { boot_disk_size_gb = 35 } @@ -1844,7 +1844,7 @@ resource "google_dataproc_cluster" "updatable" { worker_config { num_instances = "%d" - machine_type = "e2-medium" + machine_type = "n1-standard-2" disk_config { boot_disk_size_gb = 35 } @@ -1874,7 +1874,7 @@ resource "google_dataproc_cluster" "non_preemptible_secondary" { master_config { num_instances = "1" - machine_type = "e2-medium" + machine_type = "n1-standard-2" disk_config { boot_disk_size_gb = 35 } @@ -1882,7 +1882,7 @@ resource "google_dataproc_cluster" "non_preemptible_secondary" { worker_config { num_instances = "2" - machine_type = "e2-medium" + machine_type = "n1-standard-2" disk_config { boot_disk_size_gb = 35 } @@ -1913,7 +1913,7 @@ resource "google_dataproc_cluster" "spot_secondary" { master_config { num_instances = "1" - machine_type = "e2-medium" + machine_type = "n1-standard-2" disk_config { boot_disk_size_gb = 35 } @@ -1921,7 +1921,7 @@ resource "google_dataproc_cluster" "spot_secondary" { worker_config { num_instances = "2" - machine_type = "e2-medium" + machine_type = "n1-standard-2" disk_config { boot_disk_size_gb = 35 } @@ -1952,7 +1952,7 @@ resource "google_dataproc_cluster" "spot_with_instance_flexibility_policy" { master_config { num_instances = "1" - machine_type = "e2-medium" + machine_type = "n1-standard-2" disk_config { boot_disk_size_gb = 35 } @@ -1960,7 +1960,7 @@ resource "google_dataproc_cluster" "spot_with_instance_flexibility_policy" { worker_config { num_instances = "2" - machine_type = "e2-medium" + machine_type = "n1-standard-2" disk_config { boot_disk_size_gb = 35 } @@ -1997,7 +1997,7 @@ resource "google_dataproc_cluster" "with_auxiliary_node_groups" { master_config { num_instances = "1" - machine_type = "e2-medium" + machine_type = "n1-standard-2" disk_config { boot_disk_size_gb = 35 } @@ -2005,7 +2005,7 @@ resource "google_dataproc_cluster" "with_auxiliary_node_groups" { worker_config { num_instances = "2" - machine_type = "e2-medium" + machine_type = "n1-standard-2" disk_config { boot_disk_size_gb = 35 } @@ -2081,7 +2081,7 @@ resource "google_dataproc_cluster" "with_bucket" { } master_config { - machine_type = "e2-medium" + machine_type = "n1-standard-2" disk_config { boot_disk_size_gb = 35 } @@ -2115,7 +2115,7 @@ resource "google_dataproc_cluster" "with_bucket" { } master_config { - machine_type = "e2-medium" + machine_type = "n1-standard-2" disk_config { boot_disk_size_gb = 35 } @@ -2305,7 +2305,7 @@ resource "google_dataproc_cluster" "with_service_account" { } master_config { - machine_type = "e2-medium" + machine_type = "n1-standard-2" disk_config { boot_disk_size_gb = 35 } @@ -2384,7 +2384,7 @@ resource "google_dataproc_cluster" "with_net_ref_by_name" { } master_config { - machine_type = "e2-medium" + machine_type = "n1-standard-2" disk_config { boot_disk_size_gb = 35 } @@ -2414,7 +2414,7 @@ resource "google_dataproc_cluster" "with_net_ref_by_url" { } master_config { - machine_type = "e2-medium" + machine_type = "n1-standard-2" disk_config { boot_disk_size_gb = 35 } From f96b9d6f1d899b00a05911924449d30e1f39c651 Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Tue, 8 Oct 2024 07:32:09 +0000 Subject: [PATCH 08/12] Dataproc subnet has a distinct ipCidrRange. --- mmv1/third_party/terraform/acctest/bootstrap_test_utils.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mmv1/third_party/terraform/acctest/bootstrap_test_utils.go b/mmv1/third_party/terraform/acctest/bootstrap_test_utils.go index 44dc8d233114..79ea419cc8b3 100644 --- a/mmv1/third_party/terraform/acctest/bootstrap_test_utils.go +++ b/mmv1/third_party/terraform/acctest/bootstrap_test_utils.go @@ -915,7 +915,7 @@ func BootstrapSubnetForDataprocBatches(t *testing.T, subnetName string, networkN subnetOptions := map[string]interface{}{ "privateIpGoogleAccess": true, } - return BootstrapSubnetWithOverrides(t, subnetName, networkName, subnetOptions) + return bootstrapSubnetWithOverrides(t, subnetName, networkName, subnetOptions, "10.78.0.0/20") } func BootstrapSubnet(t *testing.T, subnetName string, networkName string) string { @@ -930,6 +930,10 @@ func BootstrapSubnetWithFirewallForDataprocBatches(t *testing.T, testId string, } func BootstrapSubnetWithOverrides(t *testing.T, subnetName string, networkName string, subnetOptions map[string]interface{}) string { + return bootstrapSubnetWithOverrides(t, subnetName, networkName, subnetOptions, "10.77.0.0/20") +} + +func bootstrapSubnetWithOverrides(t *testing.T, subnetName string, networkName string, subnetOptions map[string]interface{}, ipCidrRange string) string { projectID := envvar.GetTestProjectFromEnv() region := envvar.GetTestRegionFromEnv() From 9d07c3442c7a17cc99358534350f88a50b5d0fa0 Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Tue, 8 Oct 2024 21:46:16 +0000 Subject: [PATCH 09/12] Use n1, rather than the default n2, machine type because the Terraform testing projects have far more n1 quota. --- .../resource_dataproc_cluster_test.go.tmpl | 176 +++++++++++++++++- 1 file changed, 174 insertions(+), 2 deletions(-) diff --git a/mmv1/third_party/terraform/services/dataproc/resource_dataproc_cluster_test.go.tmpl b/mmv1/third_party/terraform/services/dataproc/resource_dataproc_cluster_test.go.tmpl index 0f5151e83c4a..a648f678af70 100644 --- a/mmv1/third_party/terraform/services/dataproc/resource_dataproc_cluster_test.go.tmpl +++ b/mmv1/third_party/terraform/services/dataproc/resource_dataproc_cluster_test.go.tmpl @@ -1352,6 +1352,16 @@ resource "google_dataproc_cluster" "basic" { gce_cluster_config { subnetwork = "%s" } + + master_config { + # The Terraform testing projects have a lot more n1 quota than anything else. + machine_type = "n1-standard-2" + } + + worker_config { + # The Terraform testing projects have a lot more n1 quota than anything else. + machine_type = "n1-standard-2" + } } } `, rnd, subnetworkName) @@ -1537,6 +1547,16 @@ resource "google_dataproc_cluster" "basic" { enable_vtpm = true } } + + master_config { + # The Terraform testing projects have a lot more n1 quota than anything else. + machine_type = "n1-standard-2" + } + + worker_config { + # The Terraform testing projects have a lot more n1 quota than anything else. + machine_type = "n1-standard-2" + } } } `, rnd, rnd, rnd, rnd) @@ -1557,6 +1577,16 @@ resource "google_dataproc_cluster" "basic" { } tags = ["my-tag", "your-tag", "our-tag", "their-tag"] } + + master_config { + # The Terraform testing projects have a lot more n1 quota than anything else. + machine_type = "n1-standard-2" + } + + worker_config { + # The Terraform testing projects have a lot more n1 quota than anything else. + machine_type = "n1-standard-2" + } } } `, rnd, subnetworkName) @@ -1572,12 +1602,16 @@ resource "google_dataproc_cluster" "with_min_num_instances" { gce_cluster_config { subnetwork = "%s" } - master_config{ + master_config { num_instances=1 + # The Terraform testing projects have a lot more n1 quota than anything else. + machine_type = "n1-standard-2" } - worker_config{ + worker_config { num_instances = 3 min_num_instances = 2 + # The Terraform testing projects have a lot more n1 quota than anything else. + machine_type = "n1-standard-2" } } } @@ -1637,6 +1671,15 @@ resource "google_dataproc_cluster" "basic" { gce_cluster_config { subnetwork = "%s" } + + master_config { + machine_type = "n1-standard-2" + } + + worker_config { + machine_type = "n1-standard-2" + } + dataproc_metric_config { metrics { metric_source = "HDFS" @@ -1714,6 +1757,9 @@ resource "google_dataproc_cluster" "single_node_cluster" { "dataproc:dataproc.allow.zero.workers" = "true" } } + master_config { + machine_type = "n1-standard-2" + } } } `, rnd, subnetworkName) @@ -2134,6 +2180,16 @@ resource "google_dataproc_cluster" "with_labels" { gce_cluster_config { subnetwork = "%s" } + + master_config { + # The Terraform testing projects have a lot more n1 quota than anything else. + machine_type = "n1-standard-2" + } + + worker_config { + # The Terraform testing projects have a lot more n1 quota than anything else. + machine_type = "n1-standard-2" + } } labels = { @@ -2152,6 +2208,16 @@ resource "google_dataproc_cluster" "with_labels" { gce_cluster_config { subnetwork = "%s" } + + master_config { + # The Terraform testing projects have a lot more n1 quota than anything else. + machine_type = "n1-standard-2" + } + + worker_config { + # The Terraform testing projects have a lot more n1 quota than anything else. + machine_type = "n1-standard-2" + } } labels = { @@ -2170,6 +2236,16 @@ resource "google_dataproc_cluster" "with_labels" { gce_cluster_config { subnetwork = "%s" } + + master_config { + # The Terraform testing projects have a lot more n1 quota than anything else. + machine_type = "n1-standard-2" + } + + worker_config { + # The Terraform testing projects have a lot more n1 quota than anything else. + machine_type = "n1-standard-2" + } } } `, rnd, subnetworkName) @@ -2186,6 +2262,16 @@ resource "google_dataproc_cluster" "with_endpoint_config" { subnetwork = "%s" } + master_config { + # The Terraform testing projects have a lot more n1 quota than anything else. + machine_type = "n1-standard-2" + } + + worker_config { + # The Terraform testing projects have a lot more n1 quota than anything else. + machine_type = "n1-standard-2" + } + endpoint_config { enable_http_port_access = true } @@ -2205,6 +2291,16 @@ resource "google_dataproc_cluster" "with_image_version" { subnetwork = "%s" } + master_config { + # The Terraform testing projects have a lot more n1 quota than anything else. + machine_type = "n1-standard-2" + } + + worker_config { + # The Terraform testing projects have a lot more n1 quota than anything else. + machine_type = "n1-standard-2" + } + software_config { image_version = "%s" } @@ -2224,6 +2320,16 @@ resource "google_dataproc_cluster" "with_opt_components" { subnetwork = "%s" } + master_config { + # The Terraform testing projects have a lot more n1 quota than anything else. + machine_type = "n1-standard-2" + } + + worker_config { + # The Terraform testing projects have a lot more n1 quota than anything else. + machine_type = "n1-standard-2" + } + software_config { optional_components = ["DOCKER", "ZOOKEEPER"] } @@ -2243,6 +2349,16 @@ resource "google_dataproc_cluster" "with_lifecycle_config" { subnetwork = "%s" } + master_config { + # The Terraform testing projects have a lot more n1 quota than anything else. + machine_type = "n1-standard-2" + } + + worker_config { + # The Terraform testing projects have a lot more n1 quota than anything else. + machine_type = "n1-standard-2" + } + lifecycle_config { idle_delete_ttl = "%s" } @@ -2262,6 +2378,16 @@ resource "google_dataproc_cluster" "with_lifecycle_config" { subnetwork = "%s" } + master_config { + # The Terraform testing projects have a lot more n1 quota than anything else. + machine_type = "n1-standard-2" + } + + worker_config { + # The Terraform testing projects have a lot more n1 quota than anything else. + machine_type = "n1-standard-2" + } + lifecycle_config { auto_delete_time = "%s" } @@ -2471,6 +2597,16 @@ resource "google_dataproc_cluster" "kerb" { subnetwork = "%s" } + master_config { + # The Terraform testing projects have a lot more n1 quota than anything else. + machine_type = "n1-standard-2" + } + + worker_config { + # The Terraform testing projects have a lot more n1 quota than anything else. + machine_type = "n1-standard-2" + } + security_config { kerberos_config { root_principal_password_uri = google_storage_bucket_object.password.self_link @@ -2493,6 +2629,16 @@ resource "google_dataproc_cluster" "basic" { subnetwork = "%s" } + master_config { + # The Terraform testing projects have a lot more n1 quota than anything else. + machine_type = "n1-standard-2" + } + + worker_config { + # The Terraform testing projects have a lot more n1 quota than anything else. + machine_type = "n1-standard-2" + } + autoscaling_config { policy_uri = google_dataproc_autoscaling_policy.asp.id } @@ -2529,6 +2675,16 @@ resource "google_dataproc_cluster" "basic" { subnetwork = "%s" } + master_config { + # The Terraform testing projects have a lot more n1 quota than anything else. + machine_type = "n1-standard-2" + } + + worker_config { + # The Terraform testing projects have a lot more n1 quota than anything else. + machine_type = "n1-standard-2" + } + autoscaling_config { policy_uri = "" } @@ -2564,6 +2720,14 @@ resource "google_dataproc_cluster" "with_metastore_config" { gce_cluster_config { subnetwork = "%s" } + master_config { + # The Terraform testing projects have a lot more n1 quota than anything else. + machine_type = "n1-standard-2" + } + worker_config { + # The Terraform testing projects have a lot more n1 quota than anything else. + machine_type = "n1-standard-2" + } metastore_config { dataproc_metastore_service = google_dataproc_metastore_service.ms.name } @@ -2604,6 +2768,14 @@ resource "google_dataproc_cluster" "with_metastore_config" { gce_cluster_config { subnetwork = "%s" } + master_config { + # The Terraform testing projects have a lot more n1 quota than anything else. + machine_type = "n1-standard-2" + } + worker_config { + # The Terraform testing projects have a lot more n1 quota than anything else. + machine_type = "n1-standard-2" + } metastore_config { dataproc_metastore_service = google_dataproc_metastore_service.ms.name } From 29ea81549d56744fffb40cf7a5f3bc678d40d424 Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Wed, 9 Oct 2024 00:20:49 +0000 Subject: [PATCH 10/12] Remove version pinning. --- .../dataproc/resource_dataproc_cluster_test.go.tmpl | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/mmv1/third_party/terraform/services/dataproc/resource_dataproc_cluster_test.go.tmpl b/mmv1/third_party/terraform/services/dataproc/resource_dataproc_cluster_test.go.tmpl index a648f678af70..e6a4330629a2 100644 --- a/mmv1/third_party/terraform/services/dataproc/resource_dataproc_cluster_test.go.tmpl +++ b/mmv1/third_party/terraform/services/dataproc/resource_dataproc_cluster_test.go.tmpl @@ -1449,10 +1449,6 @@ resource "google_dataproc_cluster" "accelerated_cluster" { region = "us-central1" cluster_config { - software_config { - image_version = "2.0.35-debian10" - } - gce_cluster_config { subnetwork = "%s" zone = "%s" @@ -1725,9 +1721,6 @@ resource "google_dataproc_cluster" "basic" { region = "us-central1" cluster_config { - software_config { - image_version = "2.0.35-debian10" - } gce_cluster_config { subnetwork = "%s" zone = "us-central1-f" @@ -1843,7 +1836,6 @@ resource "google_dataproc_cluster" "with_init_action" { # Keep the costs down with smallest config we can get away with software_config { - image_version = "2.0.35-debian10" override_properties = { "dataproc:dataproc.allow.zero.workers" = "true" } @@ -2120,7 +2112,6 @@ resource "google_dataproc_cluster" "with_bucket" { # Keep the costs down with smallest config we can get away with software_config { - image_version = "2.0.35-debian10" override_properties = { "dataproc:dataproc.allow.zero.workers" = "true" } @@ -2154,7 +2145,6 @@ resource "google_dataproc_cluster" "with_bucket" { # Keep the costs down with smallest config we can get away with software_config { - image_version = "2.0.35-debian10" override_properties = { "dataproc:dataproc.allow.zero.workers" = "true" } @@ -2424,7 +2414,6 @@ resource "google_dataproc_cluster" "with_service_account" { cluster_config { # Keep the costs down with smallest config we can get away with software_config { - image_version = "2.0.35-debian10" override_properties = { "dataproc:dataproc.allow.zero.workers" = "true" } From e12ca738cfe29219e4df40a24c533e797837fde8 Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Thu, 10 Oct 2024 22:05:22 +0000 Subject: [PATCH 11/12] Fix typo. --- mmv1/third_party/terraform/acctest/bootstrap_test_utils.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmv1/third_party/terraform/acctest/bootstrap_test_utils.go b/mmv1/third_party/terraform/acctest/bootstrap_test_utils.go index 79ea419cc8b3..b53d3293a5c4 100644 --- a/mmv1/third_party/terraform/acctest/bootstrap_test_utils.go +++ b/mmv1/third_party/terraform/acctest/bootstrap_test_utils.go @@ -959,7 +959,7 @@ func bootstrapSubnetWithOverrides(t *testing.T, subnetName string, networkName s "name": subnetName, "region ": region, "network": networkUrl, - "ipCidrRange": "10.78.0.0/20", + "ipCidrRange": ipCidrRange, } if len(subnetOptions) != 0 { From 14290a10972100efee69377c93041d484b443f21 Mon Sep 17 00:00:00 2001 From: Adam Harwayne Date: Thu, 10 Oct 2024 22:35:23 +0000 Subject: [PATCH 12/12] Presto test uses newest 2.0 image. --- .../services/dataproc/resource_dataproc_job_test.go.tmpl | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/mmv1/third_party/terraform/services/dataproc/resource_dataproc_job_test.go.tmpl b/mmv1/third_party/terraform/services/dataproc/resource_dataproc_job_test.go.tmpl index 4578a16dd722..9b4237010303 100644 --- a/mmv1/third_party/terraform/services/dataproc/resource_dataproc_job_test.go.tmpl +++ b/mmv1/third_party/terraform/services/dataproc/resource_dataproc_job_test.go.tmpl @@ -862,12 +862,11 @@ resource "google_dataproc_cluster" "basic" { cluster_config { # Keep the costs down with smallest config we can get away with software_config { - image_version = "2.0.35-debian10" + # 2.1 removes Presto in favor of Trino. So use 2.0 for the Presto test. + image_version = "2.0" override_properties = { "dataproc:dataproc.allow.zero.workers" = "true" } - # 2.1 removes Presto in favor of Trino. So use 2.0 for the Presto test. - image_version = "2.0" optional_components = ["PRESTO"] }