diff --git a/mmv1/products/dlp/DiscoveryConfig.yaml b/mmv1/products/dlp/DiscoveryConfig.yaml index 6e0493f05907..dee58085ea16 100644 --- a/mmv1/products/dlp/DiscoveryConfig.yaml +++ b/mmv1/products/dlp/DiscoveryConfig.yaml @@ -125,6 +125,21 @@ properties: - name: 'folderId' type: String description: The ID for the folder within an organization to scan + - name: 'otherCloudStartingLocation' + type: NestedObject + properties: + - name: 'awsLocation' + type: NestedObject + properties: + - name: 'accountId' + type: String + description: 'The AWS account ID that this discovery config applies to. + Within an organization, you can find the AWS account ID inside an AWS account ARN. + Example: + arn::organizations:::account//' + - name: 'allAssetInventoryAssets' + type: Boolean + description: All AWS assets stored in Asset Inventory that didn't match other AWS discovery configs. - name: 'inspectTemplates' type: Array description: Detection logic for profile generation @@ -290,11 +305,11 @@ properties: - name: 'otherTables' type: NestedObject description: Catch-all. This should always be the last filter in the list because anything above it will apply first. - # The fields below are necessary to include the "otherTables" filter in the payload + # The fields below are necessary to include the "otherTables" filter in the payload send_empty_value: true allow_empty_object: true properties: - # Meant to be an empty object with no properties - see here : https://cloud.google.com/sensitive-data-protection/docs/reference/rest/v2/organizations.locations.discoveryConfigs#allotherbigquerytables + # Meant to be an empty object with no properties - see here : https://cloud.google.com/sensitive-data-protection/docs/reference/rest/v2/organizations.locations.discoveryConfigs#allotherbigquerytables [] - name: 'tableReference' type: NestedObject @@ -411,7 +426,7 @@ properties: send_empty_value: true allow_empty_object: true properties: - # Meant to be an empty object with no properties - see here : https://cloud.google.com/sensitive-data-protection/docs/reference/rest/v2/organizations.locations.discoveryConfigs#disabled + # Meant to be an empty object with no properties - see here : https://cloud.google.com/sensitive-data-protection/docs/reference/rest/v2/organizations.locations.discoveryConfigs#disabled [] - name: 'cloudSqlTarget' type: NestedObject @@ -454,7 +469,7 @@ properties: send_empty_value: true allow_empty_object: true properties: - # Meant to be an empty object with no properties. The fields below are necessary to include the "others" filter in the payload + # Meant to be an empty object with no properties. The fields below are necessary to include the "others" filter in the payload [] - name: 'databaseResourceReference' type: NestedObject @@ -562,7 +577,7 @@ properties: send_empty_value: true allow_empty_object: true properties: - # Meant to be an empty object with no properties - see here : https://cloud.google.com/sensitive-data-protection/docs/reference/rest/v2/organizations.locations.discoveryConfigs#DiscoveryConfig.SecretsDiscoveryTarget + # Meant to be an empty object with no properties - see here : https://cloud.google.com/sensitive-data-protection/docs/reference/rest/v2/organizations.locations.discoveryConfigs#DiscoveryConfig.SecretsDiscoveryTarget [] - name: 'cloudStorageTarget' type: NestedObject @@ -613,7 +628,7 @@ properties: send_empty_value: true allow_empty_object: true properties: - # Meant to be an empty object with no properties. The fields below are necessary to include the "others" filter in the payload + # Meant to be an empty object with no properties. The fields below are necessary to include the "others" filter in the payload [] - name: 'conditions' type: NestedObject @@ -687,6 +702,144 @@ properties: allow_empty_object: true properties: [] + - name: 'otherCloudTarget' + type: NestedObject + description: Other clouds target for discovery. The first target to match a resource will be the one applied. + properties: + - name: 'dataSourceType' + type: NestedObject + description: 'Required. The type of data profiles generated by this discovery target. Supported values are: aws/s3/bucket' + properties: + - name: 'dataSource' + type: String + - name: 'filter' + type: NestedObject + description: 'Required. The resources that the discovery cadence applies to. The + first target with a matching filter will be the one to apply to a resource.' + required: true + properties: + - name: 'collection' + type: NestedObject + description: A collection of resources for this filter to apply to. + properties: + - name: 'includeRegexes' + type: NestedObject + description: A collection of regular expressions to match a resource against. + properties: + - name: 'patterns' + type: Array + description: The group of regular expression patterns to match against one or more resources. Maximum of 100 entries. The sum of all lengths of regular expressions can't exceed 10 KiB. + item_type: + type: NestedObject + properties: + - name: 'amazonS3BucketRegex' + type: NestedObject + description: Regex for Cloud Storage. + properties: + - name: 'awsAccountRegex' + type: NestedObject + description: 'The AWS account regex' + properties: + - name: 'accountIdRegex' + type: String + description: 'Regex to test the AWS account ID against. + If empty, all accounts match. + Example: arn:aws:organizations::123:account/o-b2c3d4/345' + - name: 'bucketNameRegex' + type: String + description: 'Regex to test the bucket name against. If empty, all buckets match.' + - name: 'singleResource' + type: NestedObject + description: The resource to scan. Configs using this filter can only have one target (the target with this single resource reference). + properties: + - name: 'amazonS3Bucket' + type: NestedObject + description: Amazon S3 bucket. + properties: + - name: 'awsAccount' + type: NestedObject + description: The AWS account. + properties: + - name: 'accountId' + type: String + description: AWS account ID. + - name: 'bucketName' + type: String + description: The bucket name. + - name: 'others' + type: NestedObject + description: Match discovery resources not covered by any other filter. + send_empty_value: true + allow_empty_object: true + properties: + # Meant to be an empty object with no properties. The fields below are necessary to include the "others" filter in the payload + [] + - name: 'conditions' + type: NestedObject + description: In addition to matching the filter, these conditions must be true before a profile is generated. + properties: + - name: 'minAge' + type: String + description: Duration format. Minimum age a resource must be before a profile can be generated. Value must be 1 hour or greater. Minimum age is not supported for Azure Blob Storage containers. + - name: 'amazonS3BucketConditions' + type: NestedObject + description: Amazon S3 bucket conditions. + properties: + - name: 'bucketTypes' + type: Array + description: Bucket types that should be profiled. Optional. Defaults to TYPE_ALL_SUPPORTED if unspecified. + item_type: + type: Enum + description: | + This field only has a name and description because of MM + limitations. It should not appear in downstreams. + enum_values: + - 'TYPE_ALL_SUPPORTED' + - 'TYPE_GENERAL_PURPOSE' + - name: 'objectStorageClasses' + type: Array + description: Object classes that should be profiled. Optional. Defaults to ALL_SUPPORTED_CLASSES if unspecified. + item_type: + type: Enum + description: | + This field only has a name and description because of MM + limitations. It should not appear in downstreams. + enum_values: + - 'ALL_SUPPORTED_CLASSES' + - 'STANDARD' + - 'STANDARD_INFREQUENT_ACCESS' + - 'GLACIER_INSTANT_RETRIEVAL' + - 'INTELLIGENT_TIERING' + - name: 'generationCadence' + type: NestedObject + description: How often and when to update profiles. New resources that match both the filter and conditions are scanned as quickly as possible depending on system capacity. + properties: + - name: 'refreshFrequency' + type: Enum + description: Frequency to update profiles regardless of whether the underlying resource has changes. Defaults to never. + enum_values: + - 'UPDATE_FREQUENCY_NEVER' + - 'UPDATE_FREQUENCY_DAILY' + - 'UPDATE_FREQUENCY_MONTHLY' + - name: 'inspectTemplateModifiedCadence' + type: NestedObject + description: Governs when to update data profiles when the inspection rules defined by the `InspectTemplate` change. If not set, changing the template will not cause a data profile to update. + properties: + - name: 'frequency' + type: Enum + description: How frequently data profiles can be updated when the template is modified. Defaults to never. + enum_values: + - 'UPDATE_FREQUENCY_NEVER' + - 'UPDATE_FREQUENCY_DAILY' + - 'UPDATE_FREQUENCY_MONTHLY' + - name: 'disabled' + type: NestedObject + description: Disable profiling for resources that match this filter. + send_empty_value: true + allow_empty_object: true + properties: + [] + - name: 'errors' type: Array description: Output only. A stream of errors encountered when the config was activated. Repeated errors may result in the config automatically being paused. Output only field. Will return the last 100 errors. Whenever the config is modified this list will be cleared. diff --git a/mmv1/third_party/terraform/services/datalossprevention/resource_data_loss_prevention_discovery_config_test.go b/mmv1/third_party/terraform/services/datalossprevention/resource_data_loss_prevention_discovery_config_test.go index 78cd55847b1c..392d5f64800f 100644 --- a/mmv1/third_party/terraform/services/datalossprevention/resource_data_loss_prevention_discovery_config_test.go +++ b/mmv1/third_party/terraform/services/datalossprevention/resource_data_loss_prevention_discovery_config_test.go @@ -21,6 +21,7 @@ func TestAccDataLossPreventionDiscoveryConfig_Update(t *testing.T) { "secrets": testAccDataLossPreventionDiscoveryConfig_SecretsUpdate, "gcs": testAccDataLossPreventionDiscoveryConfig_CloudStorageUpdate, "gcs_single": testAccDataLossPreventionDiscoveryConfig_CloudStorageSingleBucket, + "aws_s3": testAccDataLossPreventionDiscoveryConfig_AwsS3Update, } for name, tc := range testCases { // shadow the tc variable into scope so that when @@ -394,6 +395,78 @@ func testAccDataLossPreventionDiscoveryConfig_CloudStorageSingleBucket(t *testin }) } +func testAccDataLossPreventionDiscoveryConfig_AwsS3Update(t *testing.T) { + + context := map[string]interface{}{ + "project": envvar.GetTestProjectFromEnv(), + "organization": envvar.GetTestOrgFromEnv(t), + "location": envvar.GetTestRegionFromEnv(), + "random_suffix": acctest.RandString(t, 10), + } + + acctest.VcrTest(t, resource.TestCase{ + PreCheck: func() { acctest.AccTestPreCheck(t) }, + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t), + CheckDestroy: testAccCheckDataLossPreventionDiscoveryConfigDestroyProducer(t), + Steps: []resource.TestStep{ + { + Config: testAccDataLossPreventionDiscoveryConfig_dlpDiscoveryConfigStartAwsS3(context), + }, + { + ResourceName: "google_data_loss_prevention_discovery_config.basic", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"location", "parent", "last_run_time", "update_time", "errors"}, + }, + { + Config: testAccDataLossPreventionDiscoveryConfig_dlpDiscoveryConfigUpdateAwsS3(context), + }, + { + ResourceName: "google_data_loss_prevention_discovery_config.basic", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"location", "parent", "last_run_time", "update_time", "errors"}, + }, + }, + }) +} + +func testAccDataLossPreventionDiscoveryConfig_AwsS3SingleBucket(t *testing.T) { + + context := map[string]interface{}{ + "project": envvar.GetTestProjectFromEnv(), + "organization": envvar.GetTestOrgFromEnv(t), + "location": envvar.GetTestRegionFromEnv(), + "random_suffix": acctest.RandString(t, 10), + } + + acctest.VcrTest(t, resource.TestCase{ + PreCheck: func() { acctest.AccTestPreCheck(t) }, + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t), + CheckDestroy: testAccCheckDataLossPreventionDiscoveryConfigDestroyProducer(t), + Steps: []resource.TestStep{ + { + Config: testAccDataLossPreventionDiscoveryConfig_dlpDiscoveryConfigStartAwsS3(context), + }, + { + ResourceName: "google_data_loss_prevention_discovery_config.basic", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"location", "parent", "last_run_time", "update_time", "errors"}, + }, + { + Config: testAccDataLossPreventionDiscoveryConfig_dlpDiscoveryConfigAwsS3SingleUpdate(context), + }, + { + ResourceName: "google_data_loss_prevention_discovery_config.basic", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"location", "parent", "last_run_time", "update_time", "errors"}, + }, + }, + }) +} + func testAccDataLossPreventionDiscoveryConfig_SecretsUpdate(t *testing.T) { context := map[string]interface{}{ @@ -1292,3 +1365,249 @@ resource "google_data_loss_prevention_discovery_config" "basic" { } `, context) } + +func testAccDataLossPreventionDiscoveryConfig_dlpDiscoveryConfigStartAwsS3(context map[string]interface{}) string { + return acctest.Nprintf(` +data "google_project" "project" { +} +resource "google_organization_iam_member" "service_agent" { + org_id = "%{organization}" + role = "roles/servicemanagement.admin" + member = "serviceAccount:service-${data.google_project.project.number}@dlp-api.iam.gserviceaccount.com" +} +resource "google_organization_iam_member" "org_admin" { + org_id = "%{organization}" + role = "roles/resourcemanager.organizationAdmin" + member = "serviceAccount:service-${data.google_project.project.number}@dlp-api.iam.gserviceaccount.com" +} +resource "google_organization_iam_member" "dlp_role" { + org_id = "%{organization}" + role = "roles/dlp.orgdriver" + member = "serviceAccount:service-${data.google_project.project.number}@dlp-api.iam.gserviceaccount.com" +} +resource "google_data_loss_prevention_inspect_template" "basic" { + parent = "projects/%{project}" + description = "Description" + display_name = "Display" + inspect_config { + info_types { + name = "EMAIL_ADDRESS" + } + } +} +resource "google_data_loss_prevention_discovery_config" "basic" { + parent = "organizations/%{organization}/locations/%{location}" + location = "%{location}" + org_config { + project_id = "%{project}" + location { + organization_id = "%{organization}" + } + } + other_cloud_starting_location { + aws_location { + account_id = "012345678910" + } + } + status = "RUNNING" + targets { + other_cloud_target { + data_source_type { + data_source = "aws/s3/bucket" + } + filter { + others {} + } + generation_cadence { + inspect_template_modified_cadence { + frequency = "UPDATE_FREQUENCY_MONTHLY" + } + refresh_frequency = "UPDATE_FREQUENCY_MONTHLY" + } + } + + } + inspect_templates = ["projects/%{project}/inspectTemplates/${google_data_loss_prevention_inspect_template.basic.name}"] +} +`, context) +} + +func testAccDataLossPreventionDiscoveryConfig_dlpDiscoveryConfigUpdateAwsS3(context map[string]interface{}) string { + return acctest.Nprintf(` +data "google_project" "project" { +} +resource "google_organization_iam_member" "org_admin" { + org_id = "%{organization}" + role = "roles/resourcemanager.organizationAdmin" + member = "serviceAccount:service-${data.google_project.project.number}@dlp-api.iam.gserviceaccount.com" +} +resource "google_organization_iam_member" "dlp_role" { + org_id = "%{organization}" + role = "roles/dlp.orgDriver" + member = "serviceAccount:service-${data.google_project.project.number}@dlp-api.iam.gserviceaccount.com" +} +resource "google_data_loss_prevention_inspect_template" "basic" { + parent = "projects/%{project}" + description = "Description" + display_name = "Display" + inspect_config { + info_types { + name = "EMAIL_ADDRESS" + } + } +} +resource "google_data_loss_prevention_discovery_config" "basic" { + parent = "organizations/%{organization}/locations/%{location}" + location = "%{location}" + org_config { + project_id = "%{project}" + location { + organization_id = "%{organization}" + } + } + other_cloud_starting_location { + aws_location { + account_id = "012345678901" + } + } + status = "RUNNING" + targets { + other_cloud_target { + data_source_type { + data_source = "aws/s3/bucket" + } + filter { + collection { + include_regexes { + patterns { + amazon_s3_bucket_regex { + aws_account_regex { + account_id_regex = "012345678901" + } + bucket_name_regex = "bucket" + } + } + } + } + } + conditions { + min_age = "10800s" + other_cloud_conditions { + amazon_s3_bucket_conditions { + bucket_types = ["TYPE_ALL_SUPPORTED"] + object_storage_classes = ["STANDARD", "STANDARD_INFREQUENT_ACCESS"] + } + } + } + generation_cadence { + inspect_template_modified_cadence { + frequency = "UPDATE_FREQUENCY_DAILY" + } + refresh_frequency = "UPDATE_FREQUENCY_MONTHLY" + } + } + } + targets { + other_cloud_target { + data_source_type { + data_source = "aws/s3/bucket" + } + filter { + collection { + include_regexes { + patterns { + amazon_s3_bucket_regex { + aws_account_regex { + account_id_regex = "012345678901" + } + bucket_name_regex = "do-not-scan" + } + } + } + } + } + disabled {} + } + } + targets { + other_cloud_target { + data_source_type { + data_source = "aws/s3/bucket" + } + filter { + others {} + } + generation_cadence { + inspect_template_modified_cadence { + frequency = "UPDATE_FREQUENCY_MONTHLY" + } + refresh_frequency = "UPDATE_FREQUENCY_MONTHLY" + } + } + } + inspect_templates = ["projects/%{project}/inspectTemplates/${google_data_loss_prevention_inspect_template.basic.name}"] +} +`, context) +} + +func testAccDataLossPreventionDiscoveryConfig_dlpDiscoveryConfigAwsS3SingleUpdate(context map[string]interface{}) string { + return acctest.Nprintf(` +resource "google_data_loss_prevention_inspect_template" "basic" { + parent = "projects/%{project}" + description = "Description" + display_name = "Display" + inspect_config { + info_types { + name = "EMAIL_ADDRESS" + } + } +} +data "google_project" "project" { +} +resource "google_organization_iam_member" "dlp_role" { + org_id = "%{organization}" + role = "roles/dlp.projectdriver" + member = "serviceAccount:service-${data.google_project.project.number}@dlp-api.iam.gserviceaccount.com" +} +resource "google_data_loss_prevention_discovery_config" "basic" { + parent = "organizations/%{organization}/locations/%{location}" + location = "%{location}" + org_config { + project_id = "%{project}" + location { + organization_id = "%{organization}" + } + } + other_cloud_starting_location { + aws_location { + all_asset_inventory_assets = true + } + } + status = "RUNNING" + targets { + other_cloud_target { + data_source_type { + data_source = "aws/s3/bucket" + } + filter { + single_resource { + amazon_s3_bucket { + aws_account { + account_id = "012345678901" + } + bucket_name = "aws_bucket" + } + } + } + generation_cadence { + inspect_template_modified_cadence { + frequency = "UPDATE_FREQUENCY_DAILY" + } + refresh_frequency = "UPDATE_FREQUENCY_DAILY" + } + } + } + inspect_templates = ["projects/%{project}/inspectTemplates/${google_data_loss_prevention_inspect_template.basic.name}"] +} +`, context) +}