From 275cf5d058177995e9e7ed6a753dffba5c8c52c4 Mon Sep 17 00:00:00 2001 From: Benjamin Elder Date: Tue, 28 Mar 2023 19:58:26 -0700 Subject: [PATCH 1/5] make oci-proxy verbosity configurable needed for deploying staging with production config --- infra/gcp/terraform/k8s-infra-oci-proxy-prod/oci-proxy.tf | 1 + .../gcp/terraform/k8s-infra-oci-proxy-prod/terraform.tfvars | 3 +++ infra/gcp/terraform/k8s-infra-oci-proxy-prod/variables.tf | 5 +++++ 3 files changed, 9 insertions(+) diff --git a/infra/gcp/terraform/k8s-infra-oci-proxy-prod/oci-proxy.tf b/infra/gcp/terraform/k8s-infra-oci-proxy-prod/oci-proxy.tf index 41998e572fc..c0e9b07f320 100644 --- a/infra/gcp/terraform/k8s-infra-oci-proxy-prod/oci-proxy.tf +++ b/infra/gcp/terraform/k8s-infra-oci-proxy-prod/oci-proxy.tf @@ -92,6 +92,7 @@ resource "google_cloud_run_service" "oci-proxy" { // - We will eventually auto-deploy staging by overriding the project and digest on the production config to avoid skew // If you're interested in running this image yourself releases are available at registry.k8s.io/infra-tools/archeio image = "gcr.io/k8s-staging-infra-tools/archeio@${var.digest}" + args = ["-v=${var.verbosity}"] dynamic "env" { for_each = each.value.environment_variables diff --git a/infra/gcp/terraform/k8s-infra-oci-proxy-prod/terraform.tfvars b/infra/gcp/terraform/k8s-infra-oci-proxy-prod/terraform.tfvars index 6f698c89ff5..e0b0f263c7a 100644 --- a/infra/gcp/terraform/k8s-infra-oci-proxy-prod/terraform.tfvars +++ b/infra/gcp/terraform/k8s-infra-oci-proxy-prod/terraform.tfvars @@ -19,6 +19,9 @@ project_id = "k8s-infra-oci-proxy-prod" // gcr.io/k8s-staging-infra-tools/archeio:v20230310-v0.2.0@sha256:bc742c5f47a69e21e828768991853faddbe13a7f69a9da4d7d2ad16e0e55892c // If you're interested in running this image yourself releases are available at registry.k8s.io/infra-tools/archeio digest = "sha256:bc742c5f47a69e21e828768991853faddbe13a7f69a9da4d7d2ad16e0e55892c" +// we increase this in staging, but not in production +// we already get a lot of info from build-in cloud run logs +verbosity = "0" cloud_run_config = { asia-east1 = { // TODO: switch DEFAULT_AWS_BASE_URL to cloudfront or else refine the region mapping diff --git a/infra/gcp/terraform/k8s-infra-oci-proxy-prod/variables.tf b/infra/gcp/terraform/k8s-infra-oci-proxy-prod/variables.tf index 0c4e1ecf4bd..2fa13ad27e3 100644 --- a/infra/gcp/terraform/k8s-infra-oci-proxy-prod/variables.tf +++ b/infra/gcp/terraform/k8s-infra-oci-proxy-prod/variables.tf @@ -23,6 +23,11 @@ variable "project_id" { variable "digest" { type = string } + +variable "verbosity" { + type = string +} + variable "cloud_run_config" { type = map(object({ environment_variables = list(object({ From 8d0eae1fb71ae107c9add98ecde6740eecaccdab Mon Sep 17 00:00:00 2001 From: Benjamin Elder Date: Sun, 2 Apr 2023 17:01:11 -0700 Subject: [PATCH 2/5] copy oci-proxy-prod to module --- infra/gcp/terraform/modules/oci-proxy/OWNERS | 3 + .../gcp/terraform/modules/oci-proxy/README.md | 9 + .../modules/oci-proxy/cloud-armor.tf | 170 ++++++++++++++++++ infra/gcp/terraform/modules/oci-proxy/logs.tf | 30 ++++ .../terraform/modules/oci-proxy/monitoring.tf | 38 ++++ .../terraform/modules/oci-proxy/oci-proxy.tf | 146 +++++++++++++++ .../terraform/modules/oci-proxy/provider.tf | 39 ++++ .../terraform/modules/oci-proxy/variables.tf | 38 ++++ .../terraform/modules/oci-proxy/versions.tf | 24 +++ 9 files changed, 497 insertions(+) create mode 100644 infra/gcp/terraform/modules/oci-proxy/OWNERS create mode 100644 infra/gcp/terraform/modules/oci-proxy/README.md create mode 100644 infra/gcp/terraform/modules/oci-proxy/cloud-armor.tf create mode 100644 infra/gcp/terraform/modules/oci-proxy/logs.tf create mode 100644 infra/gcp/terraform/modules/oci-proxy/monitoring.tf create mode 100644 infra/gcp/terraform/modules/oci-proxy/oci-proxy.tf create mode 100644 infra/gcp/terraform/modules/oci-proxy/provider.tf create mode 100644 infra/gcp/terraform/modules/oci-proxy/variables.tf create mode 100644 infra/gcp/terraform/modules/oci-proxy/versions.tf diff --git a/infra/gcp/terraform/modules/oci-proxy/OWNERS b/infra/gcp/terraform/modules/oci-proxy/OWNERS new file mode 100644 index 00000000000..27641792385 --- /dev/null +++ b/infra/gcp/terraform/modules/oci-proxy/OWNERS @@ -0,0 +1,3 @@ +approvers: +- ameukam +- bentheelder \ No newline at end of file diff --git a/infra/gcp/terraform/modules/oci-proxy/README.md b/infra/gcp/terraform/modules/oci-proxy/README.md new file mode 100644 index 00000000000..2238fa8d8b7 --- /dev/null +++ b/infra/gcp/terraform/modules/oci-proxy/README.md @@ -0,0 +1,9 @@ +# oci-proxy common module + +This module contains ~all of the config for oci-proxy / oci-proxy-staging. + +Staging is expected to continuously lead production rollouts and changes +will be vetted in staging before manually rolling out to production. + +The only differences between staging and production are inputs variables +to this module, such as domain and IP address. diff --git a/infra/gcp/terraform/modules/oci-proxy/cloud-armor.tf b/infra/gcp/terraform/modules/oci-proxy/cloud-armor.tf new file mode 100644 index 00000000000..1ecc29568a8 --- /dev/null +++ b/infra/gcp/terraform/modules/oci-proxy/cloud-armor.tf @@ -0,0 +1,170 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + + +# This file contains the Cloud Armor policies + +resource "google_compute_security_policy" "cloud-armor" { + project = google_project.project.project_id + name = "security-policy-oci-proxy" + + + rule { + action = "deny(403)" + priority = "910" + match { + expr { + expression = "evaluatePreconfiguredWaf('methodenforcement-v33-stable', {'sensitivity': 1})" + } + } + description = "Method enforcement" + + preview = false + } + + rule { + action = "deny(403)" + priority = "900" + match { + expr { + expression = "evaluatePreconfiguredWaf('protocolattack-v33-stable', {'sensitivity': 3, 'opt_out_rule_ids': ['owasp-crs-v030301-id921170-protocolattack']})" + } + } + description = "Protocol Attack" + + preview = false + } + + rule { + action = "deny(403)" + priority = "920" + match { + expr { + expression = "evaluatePreconfiguredWaf('scannerdetection-v33-stable', {'sensitivity': 1})" + } + } + description = "Scanner detection" + + preview = false + } + + rule { + action = "deny(403)" + priority = "990" + match { + expr { + expression = "evaluatePreconfiguredWaf('xss-v33-stable', {'sensitivity': 1})" + } + } + description = "Cross-site scripting (XSS)" + + preview = false + } + + rule { + action = "deny(403)" + priority = "960" + match { + expr { + expression = "evaluatePreconfiguredWaf('lfi-v33-stable', {'sensitivity': 1})" + } + } + description = "Local file inclusion (LFI)" + + preview = false + } + + rule { + action = "deny(403)" + priority = "930" + match { + expr { + expression = "evaluatePreconfiguredExpr('rce-stable')" + } + } + + preview = false + } + + rule { + action = "deny(403)" + priority = "940" + match { + expr { + expression = "evaluatePreconfiguredWaf('rfi-v33-stable', {'sensitivity': 2})" + } + } + description = "Remote file inclusion (RFI)" + + preview = false + } + + rule { + action = "deny(403)" + priority = "950" + match { + expr { + expression = "evaluatePreconfiguredWaf('sessionfixation-v33-stable', {'sensitivity': 1})" + } + } + description = "Session fixation" + + preview = false + } + + rule { + action = "deny(403)" + priority = "980" + match { + expr { + expression = "evaluatePreconfiguredWaf('php-v33-stable', {'sensitivity': 3})" + } + } + description = "PHP" + + preview = false + } + + rule { + action = "deny(403)" + priority = "1010" + match { + expr { + expression = "evaluatePreconfiguredExpr('cve-canary')" + } + } + description = "CVEs and other vulnerabilities" + + preview = false + } + + # Reject all traffic that hasn't been whitelisted. + rule { + action = "allow" + description = "Default rule, higher priority overrides it" + priority = "2147483647" + + match { + config { + src_ip_ranges = ["*"] + } + versioned_expr = "SRC_IPS_V1" + } + + preview = false + } +} + diff --git a/infra/gcp/terraform/modules/oci-proxy/logs.tf b/infra/gcp/terraform/modules/oci-proxy/logs.tf new file mode 100644 index 00000000000..473e34d75ac --- /dev/null +++ b/infra/gcp/terraform/modules/oci-proxy/logs.tf @@ -0,0 +1,30 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +resource "google_logging_project_sink" "bigquery_sink" { + project = google_project.project.project_id + name = "registry-k8s-io-logs-sink" + destination = "bigquery.googleapis.com/projects/k8s-infra-public-pii/datasets/registry_k8s_io_logs" + + bigquery_options { + use_partitioned_tables = false + } + + unique_writer_identity = true + + filter = "resource.type = \"cloud_run_revision\" AND log_name= \"projects/${google_project.project.project_id}/logs/run.googleapis.com%2Frequests\"" + +} diff --git a/infra/gcp/terraform/modules/oci-proxy/monitoring.tf b/infra/gcp/terraform/modules/oci-proxy/monitoring.tf new file mode 100644 index 00000000000..93c51ac815f --- /dev/null +++ b/infra/gcp/terraform/modules/oci-proxy/monitoring.tf @@ -0,0 +1,38 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +resource "google_monitoring_notification_channel" "emails" { + display_name = "k8s-infra-alerts@kubernetes.io" + project = google_project.project.project_id + type = "email" + labels = { + email_address = "k8s-infra-alerts@kubernetes.io" + } +} + +module "alerts" { + project_id = google_project.project.project_id + source = "../modules/monitoring/uptime-alert" + documentation_text = "${var.domain} is down" + domain = var.domain + + notification_channels = [ + # Manually created. Monitoring channels can't be created with Terraform. + # See: https://github.com/hashicorp/terraform-provider-google/issues/1134 + "${google_project.project.id}/notificationChannels/15334306215710275143", + google_monitoring_notification_channel.emails.name, + ] +} diff --git a/infra/gcp/terraform/modules/oci-proxy/oci-proxy.tf b/infra/gcp/terraform/modules/oci-proxy/oci-proxy.tf new file mode 100644 index 00000000000..c0e9b07f320 --- /dev/null +++ b/infra/gcp/terraform/modules/oci-proxy/oci-proxy.tf @@ -0,0 +1,146 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +data "google_organization" "org" { + domain = "kubernetes.io" +} + +resource "google_project" "project" { + name = var.project_id + project_id = var.project_id + org_id = data.google_organization.org.org_id + billing_account = "018801-93540E-22A20E" +} + + +// Enable services needed for the project +resource "google_project_service" "project" { + project = google_project.project.id + + for_each = toset([ + "compute.googleapis.com", + "containerregistry.googleapis.com", + "logging.googleapis.com", + "monitoring.googleapis.com", + "oslogin.googleapis.com", + "pubsub.googleapis.com", + "run.googleapis.com", + "storage-api.googleapis.com", + "storage-component.googleapis.com" + ]) + + service = each.key +} + +// Ensure k8s-infra-oci-proxy-admins@kubernetes.io has admin access to this project +resource "google_project_iam_member" "k8s_infra_oci_proxy_admins" { + project = google_project.project.id + role = "roles/owner" + member = "group:k8s-infra-oci-proxy-admins@kubernetes.io" +} + + +resource "google_service_account" "oci-proxy" { + project = google_project.project.project_id + account_id = "oci-proxy-prod" + display_name = "Minimal Service Account for OCI Proxy" +} + +// Make each service invokable by all users. +resource "google_cloud_run_service_iam_member" "allUsers" { + project = google_project.project.project_id + for_each = google_cloud_run_service.oci-proxy + + service = google_cloud_run_service.oci-proxy[each.key].name + location = google_cloud_run_service.oci-proxy[each.key].location + role = "roles/run.invoker" + member = "allUsers" +} + +resource "google_cloud_run_service" "oci-proxy" { + project = google_project.project.project_id + for_each = var.cloud_run_config + name = "${var.project_id}-${each.key}" + location = each.key + + template { + metadata { + annotations = { + "autoscaling.knative.dev/maxScale" = "10" // TODO: adjust to control costs + "run.googleapis.com/launch-stage" = "BETA" + } + } + spec { + service_account_name = google_service_account.oci-proxy.email + containers { + // NOTE: We deploy from staging because: + // - We pin by digest anyhow (so it's comparably secure) + // - We need to be able to deploy registry fixes ASAP + // - We will eventually auto-deploy staging by overriding the project and digest on the production config to avoid skew + // If you're interested in running this image yourself releases are available at registry.k8s.io/infra-tools/archeio + image = "gcr.io/k8s-staging-infra-tools/archeio@${var.digest}" + args = ["-v=${var.verbosity}"] + + dynamic "env" { + for_each = each.value.environment_variables + content { + name = env.value["name"] + value = env.value["value"] + } + } + + // ensure this match the value for template.spec.containers.resources.limits + env { + name = "GOMAXPROCS" + value = "1" + } + + resources { + limits = { + "cpu" = "1000m" + } + } + } + + # we can probably hit 1k QPS/core (cloud run's maximum configurable) + # but we are leaving in a little overhead, if we actually hit 1k qps in + # a region we can scale to another 1 core instance + container_concurrency = 800 + + // we only serve cheap redirects, 60s is a rather long request + timeout_seconds = 60 + } + } + + traffic { + percent = 100 + latest_revision = true + } + + depends_on = [ + google_project_service.project["run.googleapis.com"] + ] + + lifecycle { + ignore_changes = [ + // This gets added by the Cloud Run API post deploy and causes diffs, can be ignored... + template[0].metadata[0].annotations["client.knative.dev/sandbox"], + template[0].metadata[0].annotations["run.googleapis.com/user-image"], + template[0].metadata[0].annotations["run.googleapis.com/client-name"], + template[0].metadata[0].annotations["run.googleapis.com/client-version"], + ] + } +} diff --git a/infra/gcp/terraform/modules/oci-proxy/provider.tf b/infra/gcp/terraform/modules/oci-proxy/provider.tf new file mode 100644 index 00000000000..1084a8262c6 --- /dev/null +++ b/infra/gcp/terraform/modules/oci-proxy/provider.tf @@ -0,0 +1,39 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +This file defines: +- Required provider versions +- Storage backend details +*/ + +terraform { + backend "gcs" { + bucket = "k8s-infra-tf-oci-proxy" + prefix = "prod" + } + + required_providers { + google = { + source = "hashicorp/google" + version = "~> 4.38.0" + } + google-beta = { + source = "hashicorp/google-beta" + version = "~> 4.38.0" + } + } +} diff --git a/infra/gcp/terraform/modules/oci-proxy/variables.tf b/infra/gcp/terraform/modules/oci-proxy/variables.tf new file mode 100644 index 00000000000..2fa13ad27e3 --- /dev/null +++ b/infra/gcp/terraform/modules/oci-proxy/variables.tf @@ -0,0 +1,38 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +variable "domain" { + type = string +} +variable "project_id" { + type = string +} +variable "digest" { + type = string +} + +variable "verbosity" { + type = string +} + +variable "cloud_run_config" { + type = map(object({ + environment_variables = list(object({ + value = string + name = string + })) + })) +} diff --git a/infra/gcp/terraform/modules/oci-proxy/versions.tf b/infra/gcp/terraform/modules/oci-proxy/versions.tf new file mode 100644 index 00000000000..03a7450e90d --- /dev/null +++ b/infra/gcp/terraform/modules/oci-proxy/versions.tf @@ -0,0 +1,24 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +This file defines: +- Required Terraform version +*/ + +terraform { + required_version = "~> 1.2.0" +} From ffbaa4eb3b652a9cc7520593cae83a8004ef88a3 Mon Sep 17 00:00:00 2001 From: Benjamin Elder Date: Sun, 2 Apr 2023 17:41:43 -0700 Subject: [PATCH 3/5] refactor to use module --- .../k8s-infra-oci-proxy-prod/cloud-armor.tf | 170 -------- .../k8s-infra-oci-proxy-prod/logs.tf | 30 -- .../k8s-infra-oci-proxy-prod/monitoring.tf | 38 -- .../k8s-infra-oci-proxy-prod/oci-proxy.tf | 147 ++----- .../k8s-infra-oci-proxy-prod/terraform.tfvars | 388 ------------------ .../k8s-infra-oci-proxy-prod/variables.tf | 38 -- infra/gcp/terraform/modules/oci-proxy/logs.tf | 30 -- .../terraform/modules/oci-proxy/monitoring.tf | 4 +- .../oci-proxy}/network.tf | 8 +- .../terraform/modules/oci-proxy/oci-proxy.tf | 369 ++++++++++++++++- .../terraform/modules/oci-proxy/provider.tf | 39 -- .../terraform/modules/oci-proxy/variables.tf | 11 +- 12 files changed, 402 insertions(+), 870 deletions(-) delete mode 100644 infra/gcp/terraform/k8s-infra-oci-proxy-prod/cloud-armor.tf delete mode 100644 infra/gcp/terraform/k8s-infra-oci-proxy-prod/logs.tf delete mode 100644 infra/gcp/terraform/k8s-infra-oci-proxy-prod/monitoring.tf delete mode 100644 infra/gcp/terraform/k8s-infra-oci-proxy-prod/terraform.tfvars delete mode 100644 infra/gcp/terraform/k8s-infra-oci-proxy-prod/variables.tf delete mode 100644 infra/gcp/terraform/modules/oci-proxy/logs.tf rename infra/gcp/terraform/{k8s-infra-oci-proxy-prod => modules/oci-proxy}/network.tf (94%) delete mode 100644 infra/gcp/terraform/modules/oci-proxy/provider.tf diff --git a/infra/gcp/terraform/k8s-infra-oci-proxy-prod/cloud-armor.tf b/infra/gcp/terraform/k8s-infra-oci-proxy-prod/cloud-armor.tf deleted file mode 100644 index 1ecc29568a8..00000000000 --- a/infra/gcp/terraform/k8s-infra-oci-proxy-prod/cloud-armor.tf +++ /dev/null @@ -1,170 +0,0 @@ -/* -Copyright 2022 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - - -# This file contains the Cloud Armor policies - -resource "google_compute_security_policy" "cloud-armor" { - project = google_project.project.project_id - name = "security-policy-oci-proxy" - - - rule { - action = "deny(403)" - priority = "910" - match { - expr { - expression = "evaluatePreconfiguredWaf('methodenforcement-v33-stable', {'sensitivity': 1})" - } - } - description = "Method enforcement" - - preview = false - } - - rule { - action = "deny(403)" - priority = "900" - match { - expr { - expression = "evaluatePreconfiguredWaf('protocolattack-v33-stable', {'sensitivity': 3, 'opt_out_rule_ids': ['owasp-crs-v030301-id921170-protocolattack']})" - } - } - description = "Protocol Attack" - - preview = false - } - - rule { - action = "deny(403)" - priority = "920" - match { - expr { - expression = "evaluatePreconfiguredWaf('scannerdetection-v33-stable', {'sensitivity': 1})" - } - } - description = "Scanner detection" - - preview = false - } - - rule { - action = "deny(403)" - priority = "990" - match { - expr { - expression = "evaluatePreconfiguredWaf('xss-v33-stable', {'sensitivity': 1})" - } - } - description = "Cross-site scripting (XSS)" - - preview = false - } - - rule { - action = "deny(403)" - priority = "960" - match { - expr { - expression = "evaluatePreconfiguredWaf('lfi-v33-stable', {'sensitivity': 1})" - } - } - description = "Local file inclusion (LFI)" - - preview = false - } - - rule { - action = "deny(403)" - priority = "930" - match { - expr { - expression = "evaluatePreconfiguredExpr('rce-stable')" - } - } - - preview = false - } - - rule { - action = "deny(403)" - priority = "940" - match { - expr { - expression = "evaluatePreconfiguredWaf('rfi-v33-stable', {'sensitivity': 2})" - } - } - description = "Remote file inclusion (RFI)" - - preview = false - } - - rule { - action = "deny(403)" - priority = "950" - match { - expr { - expression = "evaluatePreconfiguredWaf('sessionfixation-v33-stable', {'sensitivity': 1})" - } - } - description = "Session fixation" - - preview = false - } - - rule { - action = "deny(403)" - priority = "980" - match { - expr { - expression = "evaluatePreconfiguredWaf('php-v33-stable', {'sensitivity': 3})" - } - } - description = "PHP" - - preview = false - } - - rule { - action = "deny(403)" - priority = "1010" - match { - expr { - expression = "evaluatePreconfiguredExpr('cve-canary')" - } - } - description = "CVEs and other vulnerabilities" - - preview = false - } - - # Reject all traffic that hasn't been whitelisted. - rule { - action = "allow" - description = "Default rule, higher priority overrides it" - priority = "2147483647" - - match { - config { - src_ip_ranges = ["*"] - } - versioned_expr = "SRC_IPS_V1" - } - - preview = false - } -} - diff --git a/infra/gcp/terraform/k8s-infra-oci-proxy-prod/logs.tf b/infra/gcp/terraform/k8s-infra-oci-proxy-prod/logs.tf deleted file mode 100644 index 473e34d75ac..00000000000 --- a/infra/gcp/terraform/k8s-infra-oci-proxy-prod/logs.tf +++ /dev/null @@ -1,30 +0,0 @@ -/* -Copyright 2022 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -resource "google_logging_project_sink" "bigquery_sink" { - project = google_project.project.project_id - name = "registry-k8s-io-logs-sink" - destination = "bigquery.googleapis.com/projects/k8s-infra-public-pii/datasets/registry_k8s_io_logs" - - bigquery_options { - use_partitioned_tables = false - } - - unique_writer_identity = true - - filter = "resource.type = \"cloud_run_revision\" AND log_name= \"projects/${google_project.project.project_id}/logs/run.googleapis.com%2Frequests\"" - -} diff --git a/infra/gcp/terraform/k8s-infra-oci-proxy-prod/monitoring.tf b/infra/gcp/terraform/k8s-infra-oci-proxy-prod/monitoring.tf deleted file mode 100644 index 93c51ac815f..00000000000 --- a/infra/gcp/terraform/k8s-infra-oci-proxy-prod/monitoring.tf +++ /dev/null @@ -1,38 +0,0 @@ -/* -Copyright 2022 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -resource "google_monitoring_notification_channel" "emails" { - display_name = "k8s-infra-alerts@kubernetes.io" - project = google_project.project.project_id - type = "email" - labels = { - email_address = "k8s-infra-alerts@kubernetes.io" - } -} - -module "alerts" { - project_id = google_project.project.project_id - source = "../modules/monitoring/uptime-alert" - documentation_text = "${var.domain} is down" - domain = var.domain - - notification_channels = [ - # Manually created. Monitoring channels can't be created with Terraform. - # See: https://github.com/hashicorp/terraform-provider-google/issues/1134 - "${google_project.project.id}/notificationChannels/15334306215710275143", - google_monitoring_notification_channel.emails.name, - ] -} diff --git a/infra/gcp/terraform/k8s-infra-oci-proxy-prod/oci-proxy.tf b/infra/gcp/terraform/k8s-infra-oci-proxy-prod/oci-proxy.tf index c0e9b07f320..65d76e52fd1 100644 --- a/infra/gcp/terraform/k8s-infra-oci-proxy-prod/oci-proxy.tf +++ b/infra/gcp/terraform/k8s-infra-oci-proxy-prod/oci-proxy.tf @@ -14,133 +14,38 @@ See the License for the specific language governing permissions and limitations under the License. */ -data "google_organization" "org" { - domain = "kubernetes.io" +locals { + project_id = "k8s-infra-oci-proxy-prod" } -resource "google_project" "project" { - name = var.project_id - project_id = var.project_id - org_id = data.google_organization.org.org_id - billing_account = "018801-93540E-22A20E" +module "oci-proxy" { + source = "../modules/oci-proxy" + // ***** production vs staging variables inputs ***** + // + // gcr.io/k8s-staging-infra-tools/archeio:v20230310-v0.2.0@sha256:bc742c5f47a69e21e828768991853faddbe13a7f69a9da4d7d2ad16e0e55892c + // If you're interested in running this image yourself releases are available at registry.k8s.io/infra-tools/archeio + digest = "sha256:bc742c5f47a69e21e828768991853faddbe13a7f69a9da4d7d2ad16e0e55892c" + domain = "registry.k8s.io" + project_id = local.project_id + // we increase this in staging, but not in production + // we already get a lot of info from build-in cloud run logs + verbosity = "0" + // Manually created. Monitoring channels can't be created with Terraform. + // See: https://github.com/hashicorp/terraform-provider-google/issues/1134 + notification_channel_id = "15334306215710275143" } +// we only sink logs to bigquery in production +resource "google_logging_project_sink" "bigquery_sink" { + project = local.project_id + name = "registry-k8s-io-logs-sink" + destination = "bigquery.googleapis.com/projects/k8s-infra-public-pii/datasets/registry_k8s_io_logs" -// Enable services needed for the project -resource "google_project_service" "project" { - project = google_project.project.id - - for_each = toset([ - "compute.googleapis.com", - "containerregistry.googleapis.com", - "logging.googleapis.com", - "monitoring.googleapis.com", - "oslogin.googleapis.com", - "pubsub.googleapis.com", - "run.googleapis.com", - "storage-api.googleapis.com", - "storage-component.googleapis.com" - ]) - - service = each.key -} - -// Ensure k8s-infra-oci-proxy-admins@kubernetes.io has admin access to this project -resource "google_project_iam_member" "k8s_infra_oci_proxy_admins" { - project = google_project.project.id - role = "roles/owner" - member = "group:k8s-infra-oci-proxy-admins@kubernetes.io" -} - - -resource "google_service_account" "oci-proxy" { - project = google_project.project.project_id - account_id = "oci-proxy-prod" - display_name = "Minimal Service Account for OCI Proxy" -} - -// Make each service invokable by all users. -resource "google_cloud_run_service_iam_member" "allUsers" { - project = google_project.project.project_id - for_each = google_cloud_run_service.oci-proxy - - service = google_cloud_run_service.oci-proxy[each.key].name - location = google_cloud_run_service.oci-proxy[each.key].location - role = "roles/run.invoker" - member = "allUsers" -} - -resource "google_cloud_run_service" "oci-proxy" { - project = google_project.project.project_id - for_each = var.cloud_run_config - name = "${var.project_id}-${each.key}" - location = each.key - - template { - metadata { - annotations = { - "autoscaling.knative.dev/maxScale" = "10" // TODO: adjust to control costs - "run.googleapis.com/launch-stage" = "BETA" - } - } - spec { - service_account_name = google_service_account.oci-proxy.email - containers { - // NOTE: We deploy from staging because: - // - We pin by digest anyhow (so it's comparably secure) - // - We need to be able to deploy registry fixes ASAP - // - We will eventually auto-deploy staging by overriding the project and digest on the production config to avoid skew - // If you're interested in running this image yourself releases are available at registry.k8s.io/infra-tools/archeio - image = "gcr.io/k8s-staging-infra-tools/archeio@${var.digest}" - args = ["-v=${var.verbosity}"] - - dynamic "env" { - for_each = each.value.environment_variables - content { - name = env.value["name"] - value = env.value["value"] - } - } - - // ensure this match the value for template.spec.containers.resources.limits - env { - name = "GOMAXPROCS" - value = "1" - } - - resources { - limits = { - "cpu" = "1000m" - } - } - } - - # we can probably hit 1k QPS/core (cloud run's maximum configurable) - # but we are leaving in a little overhead, if we actually hit 1k qps in - # a region we can scale to another 1 core instance - container_concurrency = 800 - - // we only serve cheap redirects, 60s is a rather long request - timeout_seconds = 60 - } + bigquery_options { + use_partitioned_tables = false } - traffic { - percent = 100 - latest_revision = true - } + unique_writer_identity = true - depends_on = [ - google_project_service.project["run.googleapis.com"] - ] - - lifecycle { - ignore_changes = [ - // This gets added by the Cloud Run API post deploy and causes diffs, can be ignored... - template[0].metadata[0].annotations["client.knative.dev/sandbox"], - template[0].metadata[0].annotations["run.googleapis.com/user-image"], - template[0].metadata[0].annotations["run.googleapis.com/client-name"], - template[0].metadata[0].annotations["run.googleapis.com/client-version"], - ] - } + filter = "resource.type = \"cloud_run_revision\" AND log_name= \"projects/${local.project_id}/logs/run.googleapis.com%2Frequests\"" } diff --git a/infra/gcp/terraform/k8s-infra-oci-proxy-prod/terraform.tfvars b/infra/gcp/terraform/k8s-infra-oci-proxy-prod/terraform.tfvars deleted file mode 100644 index e0b0f263c7a..00000000000 --- a/infra/gcp/terraform/k8s-infra-oci-proxy-prod/terraform.tfvars +++ /dev/null @@ -1,388 +0,0 @@ -/* -Copyright 2022 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -domain = "registry.k8s.io" -project_id = "k8s-infra-oci-proxy-prod" -// gcr.io/k8s-staging-infra-tools/archeio:v20230310-v0.2.0@sha256:bc742c5f47a69e21e828768991853faddbe13a7f69a9da4d7d2ad16e0e55892c -// If you're interested in running this image yourself releases are available at registry.k8s.io/infra-tools/archeio -digest = "sha256:bc742c5f47a69e21e828768991853faddbe13a7f69a9da4d7d2ad16e0e55892c" -// we increase this in staging, but not in production -// we already get a lot of info from build-in cloud run logs -verbosity = "0" -cloud_run_config = { - asia-east1 = { - // TODO: switch DEFAULT_AWS_BASE_URL to cloudfront or else refine the region mapping - // GCP asia-east1 is Changhua County, Taiwan - environment_variables = [ - { - name = "DEFAULT_AWS_BASE_URL", - // AWS ap-southeast-1 is Singapore - value = "https://prod-registry-k8s-io-ap-southeast-1.s3.dualstack.ap-southeast-1.amazonaws.com", - }, - { - name = "UPSTREAM_REGISTRY_ENDPOINT", - value = "https://asia-east1-docker.pkg.dev" - }, - { - name = "UPSTREAM_REGISTRY_PATH", - value = "k8s-artifacts-prod/images" - } - ] - } - // GCP asia-northeast1 is Tokyo, Japan - asia-northeast1 = { - environment_variables = [ - { - name = "DEFAULT_AWS_BASE_URL", - // AWS ap-northeast-1 is Tokyo - value = "https://prod-registry-k8s-io-ap-northeast-1.s3.dualstack.ap-northeast-1.amazonaws.com", - }, - { - name = "UPSTREAM_REGISTRY_ENDPOINT", - value = "https://asia-northeast1-docker.pkg.dev" - }, - { - name = "UPSTREAM_REGISTRY_PATH", - value = "k8s-artifacts-prod/images" - } - ] - } - // GCP asia-northeast2 is Osaka, Japan - asia-northeast2 = { - environment_variables = [ - { - name = "DEFAULT_AWS_BASE_URL", - // AWS ap-northeast-1 is Tokyo - value = "https://prod-registry-k8s-io-ap-northeast-1.s3.dualstack.ap-northeast-1.amazonaws.com", - }, - { - name = "UPSTREAM_REGISTRY_ENDPOINT", - value = "https://asia-northeast2-docker.pkg.dev" - }, - { - name = "UPSTREAM_REGISTRY_PATH", - value = "k8s-artifacts-prod/images" - } - ] - } - // GCP asia-south1 is Mumbai, India - asia-south1 = { - environment_variables = [ - { - name = "DEFAULT_AWS_BASE_URL", - // AWS ap-south-1 is Mumbai - value = "https://prod-registry-k8s-io-ap-south-1.s3.dualstack.ap-south-1.amazonaws.com", - }, - { - name = "UPSTREAM_REGISTRY_ENDPOINT", - value = "https://asia-south1-docker.pkg.dev" - }, - { - name = "UPSTREAM_REGISTRY_PATH", - value = "k8s-artifacts-prod/images" - } - ] - } - // GCP australia-southeast1 is Sydney - australia-southeast1 = { - environment_variables = [ - { - name = "DEFAULT_AWS_BASE_URL", - // AWS ap-southeast-1 is Singapore - value = "https://prod-registry-k8s-io-ap-southeast-1.s3.dualstack.ap-southeast-1.amazonaws.com", - }, - { - name = "UPSTREAM_REGISTRY_ENDPOINT", - value = "https://australia-southeast1-docker.pkg.dev" - }, - { - name = "UPSTREAM_REGISTRY_PATH", - value = "k8s-artifacts-prod/images" - } - ] - } - // GCP europe-north1 is Hamina, Finland - europe-north1 = { - environment_variables = [ - { - name = "DEFAULT_AWS_BASE_URL", - // AWS eu-central-1 is Frankfurt - value = "https://prod-registry-k8s-io-eu-central-1.s3.dualstack.eu-central-1.amazonaws.com", - }, - { - name = "UPSTREAM_REGISTRY_ENDPOINT", - value = "https://europe-north1-docker.pkg.dev" - }, - { - name = "UPSTREAM_REGISTRY_PATH", - value = "k8s-artifacts-prod/images" - } - ] - } - // GCP europe-southwest1 is Madrid, Spain - europe-southwest1 = { - environment_variables = [ - { - name = "DEFAULT_AWS_BASE_URL", - // AWS eu-central-1 is Frankfurt - value = "https://prod-registry-k8s-io-eu-central-1.s3.dualstack.eu-central-1.amazonaws.com", - }, - { - name = "UPSTREAM_REGISTRY_ENDPOINT", - value = "https://europe-southwest1-docker.pkg.dev" - }, - { - name = "UPSTREAM_REGISTRY_PATH", - value = "k8s-artifacts-prod/images" - } - ] - } - // GCP europe-west1 is St. Ghislain, Belgium - europe-west1 = { - environment_variables = [ - { - name = "DEFAULT_AWS_BASE_URL", - // AWS eu-central-1 is Frankfurt - value = "https://prod-registry-k8s-io-eu-central-1.s3.dualstack.eu-central-1.amazonaws.com", - }, - { - name = "UPSTREAM_REGISTRY_ENDPOINT", - value = "https://europe-west1-docker.pkg.dev" - }, - { - name = "UPSTREAM_REGISTRY_PATH", - value = "k8s-artifacts-prod/images" - } - ] - } - // GCP europe-west2 is London, UK - europe-west2 = { - environment_variables = [ - { - name = "DEFAULT_AWS_BASE_URL", - // AWS eu-west-2 is London - value = "https://prod-registry-k8s-io-eu-west-2.s3.dualstack.eu-west-2.amazonaws.com", - }, - { - name = "UPSTREAM_REGISTRY_ENDPOINT", - value = "https://europe-west2-docker.pkg.dev" - }, - { - name = "UPSTREAM_REGISTRY_PATH", - value = "k8s-artifacts-prod/images" - } - ] - } - // GCP europe-west4 is Eemshaven, Netherlands - europe-west4 = { - environment_variables = [ - { - name = "DEFAULT_AWS_BASE_URL", - // AWS eu-central-1 is Frankfurt - value = "https://prod-registry-k8s-io-eu-central-1.s3.dualstack.eu-central-1.amazonaws.com", - }, - { - name = "UPSTREAM_REGISTRY_ENDPOINT", - value = "https://europe-west4-docker.pkg.dev" - }, - { - name = "UPSTREAM_REGISTRY_PATH", - value = "k8s-artifacts-prod/images" - } - ] - } - // GCP europe-west8 is Milan, Italy - europe-west8 = { - environment_variables = [ - { - name = "DEFAULT_AWS_BASE_URL", - // AWS eu-central-1 is Frankfurt - value = "https://prod-registry-k8s-io-eu-central-1.s3.dualstack.eu-central-1.amazonaws.com", - }, - { - name = "UPSTREAM_REGISTRY_ENDPOINT", - value = "https://europe-west8-docker.pkg.dev" - }, - { - name = "UPSTREAM_REGISTRY_PATH", - value = "k8s-artifacts-prod/images" - } - ] - } - // GCP europe-west9 is Paris, France - europe-west9 = { - environment_variables = [ - { - name = "DEFAULT_AWS_BASE_URL", - // AWS eu-west-2 is London - value = "https://prod-registry-k8s-io-eu-west-2.s3.dualstack.eu-west-2.amazonaws.com", - }, - { - name = "UPSTREAM_REGISTRY_ENDPOINT", - value = "https://europe-west9-docker.pkg.dev" - }, - { - name = "UPSTREAM_REGISTRY_PATH", - value = "k8s-artifacts-prod/images" - } - ] - } - // GCP southamerica-west1 is Santiago, Chile - southamerica-west1 = { - environment_variables = [ - { - name = "DEFAULT_AWS_BASE_URL", - // AWS us-east-1 is Virginia, USA - // See: https://github.com/kubernetes/k8s.io/pull/4739/files#r1100667255 - value = "https://prod-registry-k8s-io-us-east-1.s3.dualstack.us-east-1.amazonaws.com", - }, - { - name = "UPSTREAM_REGISTRY_ENDPOINT", - value = "https://southamerica-west1-docker.pkg.dev" - }, - { - name = "UPSTREAM_REGISTRY_PATH", - value = "k8s-artifacts-prod/images" - } - ] - } - // GCP us-central1 is Iowa, USA - us-central1 = { - environment_variables = [ - { - name = "DEFAULT_AWS_BASE_URL", - // AWS us-east-2 is Ohio, USA - value = "https://prod-registry-k8s-io-us-east-2.s3.dualstack.us-east-2.amazonaws.com", - }, - { - name = "UPSTREAM_REGISTRY_ENDPOINT", - value = "https://us-central1-docker.pkg.dev" - }, - { - name = "UPSTREAM_REGISTRY_PATH", - value = "k8s-artifacts-prod/images" - } - ] - } - // GCP us-east1 is South Carolina, USA - us-east1 = { - environment_variables = [ - { - name = "DEFAULT_AWS_BASE_URL", - // AWS us-east-1 is Virginia, USA - value = "https://prod-registry-k8s-io-us-east-1.s3.dualstack.us-east-1.amazonaws.com", - }, - { - name = "UPSTREAM_REGISTRY_ENDPOINT", - value = "https://us-east1-docker.pkg.dev" - }, - { - name = "UPSTREAM_REGISTRY_PATH", - value = "k8s-artifacts-prod/images" - } - ] - } - // GCP us-east4 is Virginia, USA - us-east4 = { - environment_variables = [ - { - name = "DEFAULT_AWS_BASE_URL", - // AWS us-east-1 is Virginia, USA - value = "https://prod-registry-k8s-io-us-east-1.s3.dualstack.us-east-1.amazonaws.com", - }, - { - name = "UPSTREAM_REGISTRY_ENDPOINT", - value = "https://us-east4-docker.pkg.dev" - }, - { - name = "UPSTREAM_REGISTRY_PATH", - value = "k8s-artifacts-prod/images" - } - ] - } - // GCP us-east5 is Ohio, USA - us-east5 = { - environment_variables = [ - { - name = "DEFAULT_AWS_BASE_URL", - // AWS us-east-2 is Ohio, USA - value = "https://prod-registry-k8s-io-us-east-2.s3.dualstack.us-east-2.amazonaws.com", - }, - { - name = "UPSTREAM_REGISTRY_ENDPOINT", - value = "https://us-east5-docker.pkg.dev" - }, - { - name = "UPSTREAM_REGISTRY_PATH", - value = "k8s-artifacts-prod/images" - } - ] - } - // GCP us-south1 is Texas, USA - us-south1 = { - environment_variables = [ - { - name = "DEFAULT_AWS_BASE_URL", - // AWS us-east-2 is Ohio, USA - value = "https://prod-registry-k8s-io-us-east-2.s3.dualstack.us-east-2.amazonaws.com", - }, - { - name = "UPSTREAM_REGISTRY_ENDPOINT", - value = "https://us-south1-docker.pkg.dev" - }, - { - name = "UPSTREAM_REGISTRY_PATH", - value = "k8s-artifacts-prod/images" - } - ] - } - // GCP us-west1 is Oregon, USA - us-west1 = { - environment_variables = [ - { - name = "DEFAULT_AWS_BASE_URL", - // AWS us-west-2 is Oregon, USA - value = "https://prod-registry-k8s-io-us-west-2.s3.dualstack.us-west-2.amazonaws.com", - }, - { - name = "UPSTREAM_REGISTRY_ENDPOINT", - value = "https://us-west1-docker.pkg.dev" - }, - { - name = "UPSTREAM_REGISTRY_PATH", - value = "k8s-artifacts-prod/images" - } - ] - } - // GCP us-west2 is California, USA - us-west2 = { - environment_variables = [ - { - name = "DEFAULT_AWS_BASE_URL", - // AWS us-west-1 is California, USA - value = "https://prod-registry-k8s-io-us-west-1.s3.dualstack.us-west-1.amazonaws.com", - }, - { - name = "UPSTREAM_REGISTRY_ENDPOINT", - value = "https://us-west2-docker.pkg.dev" - }, - { - name = "UPSTREAM_REGISTRY_PATH", - value = "k8s-artifacts-prod/images" - } - ] - } -} diff --git a/infra/gcp/terraform/k8s-infra-oci-proxy-prod/variables.tf b/infra/gcp/terraform/k8s-infra-oci-proxy-prod/variables.tf deleted file mode 100644 index 2fa13ad27e3..00000000000 --- a/infra/gcp/terraform/k8s-infra-oci-proxy-prod/variables.tf +++ /dev/null @@ -1,38 +0,0 @@ -/* -Copyright 2022 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -variable "domain" { - type = string -} -variable "project_id" { - type = string -} -variable "digest" { - type = string -} - -variable "verbosity" { - type = string -} - -variable "cloud_run_config" { - type = map(object({ - environment_variables = list(object({ - value = string - name = string - })) - })) -} diff --git a/infra/gcp/terraform/modules/oci-proxy/logs.tf b/infra/gcp/terraform/modules/oci-proxy/logs.tf deleted file mode 100644 index 473e34d75ac..00000000000 --- a/infra/gcp/terraform/modules/oci-proxy/logs.tf +++ /dev/null @@ -1,30 +0,0 @@ -/* -Copyright 2022 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -resource "google_logging_project_sink" "bigquery_sink" { - project = google_project.project.project_id - name = "registry-k8s-io-logs-sink" - destination = "bigquery.googleapis.com/projects/k8s-infra-public-pii/datasets/registry_k8s_io_logs" - - bigquery_options { - use_partitioned_tables = false - } - - unique_writer_identity = true - - filter = "resource.type = \"cloud_run_revision\" AND log_name= \"projects/${google_project.project.project_id}/logs/run.googleapis.com%2Frequests\"" - -} diff --git a/infra/gcp/terraform/modules/oci-proxy/monitoring.tf b/infra/gcp/terraform/modules/oci-proxy/monitoring.tf index 93c51ac815f..145506ddd65 100644 --- a/infra/gcp/terraform/modules/oci-proxy/monitoring.tf +++ b/infra/gcp/terraform/modules/oci-proxy/monitoring.tf @@ -25,14 +25,14 @@ resource "google_monitoring_notification_channel" "emails" { module "alerts" { project_id = google_project.project.project_id - source = "../modules/monitoring/uptime-alert" + source = "../monitoring/uptime-alert" documentation_text = "${var.domain} is down" domain = var.domain notification_channels = [ # Manually created. Monitoring channels can't be created with Terraform. # See: https://github.com/hashicorp/terraform-provider-google/issues/1134 - "${google_project.project.id}/notificationChannels/15334306215710275143", + "${google_project.project.id}/notificationChannels/${var.notification_channel_id}", google_monitoring_notification_channel.emails.name, ] } diff --git a/infra/gcp/terraform/k8s-infra-oci-proxy-prod/network.tf b/infra/gcp/terraform/modules/oci-proxy/network.tf similarity index 94% rename from infra/gcp/terraform/k8s-infra-oci-proxy-prod/network.tf rename to infra/gcp/terraform/modules/oci-proxy/network.tf index 41b3dd671cf..0d4e13d4aba 100644 --- a/infra/gcp/terraform/k8s-infra-oci-proxy-prod/network.tf +++ b/infra/gcp/terraform/modules/oci-proxy/network.tf @@ -16,7 +16,7 @@ limitations under the License. resource "google_compute_global_address" "default_ipv4" { project = google_project.project.project_id - name = "k8s-infra-oci-proxy-prod" + name = google_project.project.project_id address_type = "EXTERNAL" ip_version = "IPV4" @@ -27,7 +27,7 @@ resource "google_compute_global_address" "default_ipv4" { resource "google_compute_global_address" "default_ipv6" { project = google_project.project.project_id - name = "k8s-infra-oci-proxy-prod-v6" + name = "${google_project.project.project_id}-v6" address_type = "EXTERNAL" ip_version = "IPV6" @@ -38,12 +38,12 @@ resource "google_compute_global_address" "default_ipv6" { data "google_compute_global_address" "default_ipv4" { project = google_project.project.project_id - name = "k8s-infra-oci-proxy-prod" + name = google_project.project.project_id } data "google_compute_global_address" "default_ipv6" { project = google_project.project.project_id - name = "k8s-infra-oci-proxy-prod-v6" + name = "${google_project.project.project_id}-v6" } resource "google_compute_region_network_endpoint_group" "oci-proxy" { diff --git a/infra/gcp/terraform/modules/oci-proxy/oci-proxy.tf b/infra/gcp/terraform/modules/oci-proxy/oci-proxy.tf index c0e9b07f320..7508878ba1e 100644 --- a/infra/gcp/terraform/modules/oci-proxy/oci-proxy.tf +++ b/infra/gcp/terraform/modules/oci-proxy/oci-proxy.tf @@ -14,6 +14,373 @@ See the License for the specific language governing permissions and limitations under the License. */ +locals { + cloud_run_config = { + asia-east1 = { + // TODO: switch DEFAULT_AWS_BASE_URL to cloudfront or else refine the region mapping + // GCP asia-east1 is Changhua County, Taiwan + environment_variables = [ + { + name = "DEFAULT_AWS_BASE_URL", + // AWS ap-southeast-1 is Singapore + value = "https://prod-registry-k8s-io-ap-southeast-1.s3.dualstack.ap-southeast-1.amazonaws.com", + }, + { + name = "UPSTREAM_REGISTRY_ENDPOINT", + value = "https://asia-east1-docker.pkg.dev" + }, + { + name = "UPSTREAM_REGISTRY_PATH", + value = "k8s-artifacts-prod/images" + } + ] + } + // GCP asia-northeast1 is Tokyo, Japan + asia-northeast1 = { + environment_variables = [ + { + name = "DEFAULT_AWS_BASE_URL", + // AWS ap-northeast-1 is Tokyo + value = "https://prod-registry-k8s-io-ap-northeast-1.s3.dualstack.ap-northeast-1.amazonaws.com", + }, + { + name = "UPSTREAM_REGISTRY_ENDPOINT", + value = "https://asia-northeast1-docker.pkg.dev" + }, + { + name = "UPSTREAM_REGISTRY_PATH", + value = "k8s-artifacts-prod/images" + } + ] + } + // GCP asia-northeast2 is Osaka, Japan + asia-northeast2 = { + environment_variables = [ + { + name = "DEFAULT_AWS_BASE_URL", + // AWS ap-northeast-1 is Tokyo + value = "https://prod-registry-k8s-io-ap-northeast-1.s3.dualstack.ap-northeast-1.amazonaws.com", + }, + { + name = "UPSTREAM_REGISTRY_ENDPOINT", + value = "https://asia-northeast2-docker.pkg.dev" + }, + { + name = "UPSTREAM_REGISTRY_PATH", + value = "k8s-artifacts-prod/images" + } + ] + } + // GCP asia-south1 is Mumbai, India + asia-south1 = { + environment_variables = [ + { + name = "DEFAULT_AWS_BASE_URL", + // AWS ap-south-1 is Mumbai + value = "https://prod-registry-k8s-io-ap-south-1.s3.dualstack.ap-south-1.amazonaws.com", + }, + { + name = "UPSTREAM_REGISTRY_ENDPOINT", + value = "https://asia-south1-docker.pkg.dev" + }, + { + name = "UPSTREAM_REGISTRY_PATH", + value = "k8s-artifacts-prod/images" + } + ] + } + // GCP australia-southeast1 is Sydney + australia-southeast1 = { + environment_variables = [ + { + name = "DEFAULT_AWS_BASE_URL", + // AWS ap-southeast-1 is Singapore + value = "https://prod-registry-k8s-io-ap-southeast-1.s3.dualstack.ap-southeast-1.amazonaws.com", + }, + { + name = "UPSTREAM_REGISTRY_ENDPOINT", + value = "https://australia-southeast1-docker.pkg.dev" + }, + { + name = "UPSTREAM_REGISTRY_PATH", + value = "k8s-artifacts-prod/images" + } + ] + } + // GCP europe-north1 is Hamina, Finland + europe-north1 = { + environment_variables = [ + { + name = "DEFAULT_AWS_BASE_URL", + // AWS eu-central-1 is Frankfurt + value = "https://prod-registry-k8s-io-eu-central-1.s3.dualstack.eu-central-1.amazonaws.com", + }, + { + name = "UPSTREAM_REGISTRY_ENDPOINT", + value = "https://europe-north1-docker.pkg.dev" + }, + { + name = "UPSTREAM_REGISTRY_PATH", + value = "k8s-artifacts-prod/images" + } + ] + } + // GCP europe-southwest1 is Madrid, Spain + europe-southwest1 = { + environment_variables = [ + { + name = "DEFAULT_AWS_BASE_URL", + // AWS eu-central-1 is Frankfurt + value = "https://prod-registry-k8s-io-eu-central-1.s3.dualstack.eu-central-1.amazonaws.com", + }, + { + name = "UPSTREAM_REGISTRY_ENDPOINT", + value = "https://europe-southwest1-docker.pkg.dev" + }, + { + name = "UPSTREAM_REGISTRY_PATH", + value = "k8s-artifacts-prod/images" + } + ] + } + // GCP europe-west1 is St. Ghislain, Belgium + europe-west1 = { + environment_variables = [ + { + name = "DEFAULT_AWS_BASE_URL", + // AWS eu-central-1 is Frankfurt + value = "https://prod-registry-k8s-io-eu-central-1.s3.dualstack.eu-central-1.amazonaws.com", + }, + { + name = "UPSTREAM_REGISTRY_ENDPOINT", + value = "https://europe-west1-docker.pkg.dev" + }, + { + name = "UPSTREAM_REGISTRY_PATH", + value = "k8s-artifacts-prod/images" + } + ] + } + // GCP europe-west2 is London, UK + europe-west2 = { + environment_variables = [ + { + name = "DEFAULT_AWS_BASE_URL", + // AWS eu-west-2 is London + value = "https://prod-registry-k8s-io-eu-west-2.s3.dualstack.eu-west-2.amazonaws.com", + }, + { + name = "UPSTREAM_REGISTRY_ENDPOINT", + value = "https://europe-west2-docker.pkg.dev" + }, + { + name = "UPSTREAM_REGISTRY_PATH", + value = "k8s-artifacts-prod/images" + } + ] + } + // GCP europe-west4 is Eemshaven, Netherlands + europe-west4 = { + environment_variables = [ + { + name = "DEFAULT_AWS_BASE_URL", + // AWS eu-central-1 is Frankfurt + value = "https://prod-registry-k8s-io-eu-central-1.s3.dualstack.eu-central-1.amazonaws.com", + }, + { + name = "UPSTREAM_REGISTRY_ENDPOINT", + value = "https://europe-west4-docker.pkg.dev" + }, + { + name = "UPSTREAM_REGISTRY_PATH", + value = "k8s-artifacts-prod/images" + } + ] + } + // GCP europe-west8 is Milan, Italy + europe-west8 = { + environment_variables = [ + { + name = "DEFAULT_AWS_BASE_URL", + // AWS eu-central-1 is Frankfurt + value = "https://prod-registry-k8s-io-eu-central-1.s3.dualstack.eu-central-1.amazonaws.com", + }, + { + name = "UPSTREAM_REGISTRY_ENDPOINT", + value = "https://europe-west8-docker.pkg.dev" + }, + { + name = "UPSTREAM_REGISTRY_PATH", + value = "k8s-artifacts-prod/images" + } + ] + } + // GCP europe-west9 is Paris, France + europe-west9 = { + environment_variables = [ + { + name = "DEFAULT_AWS_BASE_URL", + // AWS eu-west-2 is London + value = "https://prod-registry-k8s-io-eu-west-2.s3.dualstack.eu-west-2.amazonaws.com", + }, + { + name = "UPSTREAM_REGISTRY_ENDPOINT", + value = "https://europe-west9-docker.pkg.dev" + }, + { + name = "UPSTREAM_REGISTRY_PATH", + value = "k8s-artifacts-prod/images" + } + ] + } + // GCP southamerica-west1 is Santiago, Chile + southamerica-west1 = { + environment_variables = [ + { + name = "DEFAULT_AWS_BASE_URL", + // AWS us-east-1 is Virginia, USA + // See: https://github.com/kubernetes/k8s.io/pull/4739/files#r1100667255 + value = "https://prod-registry-k8s-io-us-east-1.s3.dualstack.us-east-1.amazonaws.com", + }, + { + name = "UPSTREAM_REGISTRY_ENDPOINT", + value = "https://southamerica-west1-docker.pkg.dev" + }, + { + name = "UPSTREAM_REGISTRY_PATH", + value = "k8s-artifacts-prod/images" + } + ] + } + // GCP us-central1 is Iowa, USA + us-central1 = { + environment_variables = [ + { + name = "DEFAULT_AWS_BASE_URL", + // AWS us-east-2 is Ohio, USA + value = "https://prod-registry-k8s-io-us-east-2.s3.dualstack.us-east-2.amazonaws.com", + }, + { + name = "UPSTREAM_REGISTRY_ENDPOINT", + value = "https://us-central1-docker.pkg.dev" + }, + { + name = "UPSTREAM_REGISTRY_PATH", + value = "k8s-artifacts-prod/images" + } + ] + } + // GCP us-east1 is South Carolina, USA + us-east1 = { + environment_variables = [ + { + name = "DEFAULT_AWS_BASE_URL", + // AWS us-east-1 is Virginia, USA + value = "https://prod-registry-k8s-io-us-east-1.s3.dualstack.us-east-1.amazonaws.com", + }, + { + name = "UPSTREAM_REGISTRY_ENDPOINT", + value = "https://us-east1-docker.pkg.dev" + }, + { + name = "UPSTREAM_REGISTRY_PATH", + value = "k8s-artifacts-prod/images" + } + ] + } + // GCP us-east4 is Virginia, USA + us-east4 = { + environment_variables = [ + { + name = "DEFAULT_AWS_BASE_URL", + // AWS us-east-1 is Virginia, USA + value = "https://prod-registry-k8s-io-us-east-1.s3.dualstack.us-east-1.amazonaws.com", + }, + { + name = "UPSTREAM_REGISTRY_ENDPOINT", + value = "https://us-east4-docker.pkg.dev" + }, + { + name = "UPSTREAM_REGISTRY_PATH", + value = "k8s-artifacts-prod/images" + } + ] + } + // GCP us-east5 is Ohio, USA + us-east5 = { + environment_variables = [ + { + name = "DEFAULT_AWS_BASE_URL", + // AWS us-east-2 is Ohio, USA + value = "https://prod-registry-k8s-io-us-east-2.s3.dualstack.us-east-2.amazonaws.com", + }, + { + name = "UPSTREAM_REGISTRY_ENDPOINT", + value = "https://us-east5-docker.pkg.dev" + }, + { + name = "UPSTREAM_REGISTRY_PATH", + value = "k8s-artifacts-prod/images" + } + ] + } + // GCP us-south1 is Texas, USA + us-south1 = { + environment_variables = [ + { + name = "DEFAULT_AWS_BASE_URL", + // AWS us-east-2 is Ohio, USA + value = "https://prod-registry-k8s-io-us-east-2.s3.dualstack.us-east-2.amazonaws.com", + }, + { + name = "UPSTREAM_REGISTRY_ENDPOINT", + value = "https://us-south1-docker.pkg.dev" + }, + { + name = "UPSTREAM_REGISTRY_PATH", + value = "k8s-artifacts-prod/images" + } + ] + } + // GCP us-west1 is Oregon, USA + us-west1 = { + environment_variables = [ + { + name = "DEFAULT_AWS_BASE_URL", + // AWS us-west-2 is Oregon, USA + value = "https://prod-registry-k8s-io-us-west-2.s3.dualstack.us-west-2.amazonaws.com", + }, + { + name = "UPSTREAM_REGISTRY_ENDPOINT", + value = "https://us-west1-docker.pkg.dev" + }, + { + name = "UPSTREAM_REGISTRY_PATH", + value = "k8s-artifacts-prod/images" + } + ] + } + // GCP us-west2 is California, USA + us-west2 = { + environment_variables = [ + { + name = "DEFAULT_AWS_BASE_URL", + // AWS us-west-1 is California, USA + value = "https://prod-registry-k8s-io-us-west-1.s3.dualstack.us-west-1.amazonaws.com", + }, + { + name = "UPSTREAM_REGISTRY_ENDPOINT", + value = "https://us-west2-docker.pkg.dev" + }, + { + name = "UPSTREAM_REGISTRY_PATH", + value = "k8s-artifacts-prod/images" + } + ] + } + } +} + data "google_organization" "org" { domain = "kubernetes.io" } @@ -72,7 +439,7 @@ resource "google_cloud_run_service_iam_member" "allUsers" { resource "google_cloud_run_service" "oci-proxy" { project = google_project.project.project_id - for_each = var.cloud_run_config + for_each = local.cloud_run_config name = "${var.project_id}-${each.key}" location = each.key diff --git a/infra/gcp/terraform/modules/oci-proxy/provider.tf b/infra/gcp/terraform/modules/oci-proxy/provider.tf deleted file mode 100644 index 1084a8262c6..00000000000 --- a/infra/gcp/terraform/modules/oci-proxy/provider.tf +++ /dev/null @@ -1,39 +0,0 @@ -/* -Copyright 2022 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -/* -This file defines: -- Required provider versions -- Storage backend details -*/ - -terraform { - backend "gcs" { - bucket = "k8s-infra-tf-oci-proxy" - prefix = "prod" - } - - required_providers { - google = { - source = "hashicorp/google" - version = "~> 4.38.0" - } - google-beta = { - source = "hashicorp/google-beta" - version = "~> 4.38.0" - } - } -} diff --git a/infra/gcp/terraform/modules/oci-proxy/variables.tf b/infra/gcp/terraform/modules/oci-proxy/variables.tf index 2fa13ad27e3..048f6714a24 100644 --- a/infra/gcp/terraform/modules/oci-proxy/variables.tf +++ b/infra/gcp/terraform/modules/oci-proxy/variables.tf @@ -23,16 +23,9 @@ variable "project_id" { variable "digest" { type = string } - variable "verbosity" { type = string } - -variable "cloud_run_config" { - type = map(object({ - environment_variables = list(object({ - value = string - name = string - })) - })) +variable "notification_channel_id" { + type = string } From 2b5d6b8a16b455c7e6757ed1d13941a4bb6112fb Mon Sep 17 00:00:00 2001 From: Benjamin Elder Date: Sun, 2 Apr 2023 17:58:40 -0700 Subject: [PATCH 4/5] tell terraform everything moved --- .../k8s-infra-oci-proxy-prod/moved.tf | 482 ++++++++++++++++++ 1 file changed, 482 insertions(+) create mode 100644 infra/gcp/terraform/k8s-infra-oci-proxy-prod/moved.tf diff --git a/infra/gcp/terraform/k8s-infra-oci-proxy-prod/moved.tf b/infra/gcp/terraform/k8s-infra-oci-proxy-prod/moved.tf new file mode 100644 index 00000000000..b128f146db6 --- /dev/null +++ b/infra/gcp/terraform/k8s-infra-oci-proxy-prod/moved.tf @@ -0,0 +1,482 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// All of the variables below were moved when refactoring to a common +// module between prod and staging + +/* we have to do this once per region ... */ + +moved { + from = google_cloud_run_service.oci-proxy["asia-east1"] + to = module.oci-proxy.google_cloud_run_service.oci-proxy["asia-east1"] +} + +moved { + from = google_cloud_run_service.oci-proxy["asia-northeast1"] + to = module.oci-proxy.google_cloud_run_service.oci-proxy["asia-northeast1"] +} + +moved { + from = google_cloud_run_service.oci-proxy["asia-northeast2"] + to = module.oci-proxy.google_cloud_run_service.oci-proxy["asia-northeast2"] +} + +moved { + from = google_cloud_run_service.oci-proxy["asia-south1"] + to = module.oci-proxy.google_cloud_run_service.oci-proxy["asia-south1"] +} + +moved { + from = google_cloud_run_service.oci-proxy["europe-north1"] + to = module.oci-proxy.google_cloud_run_service.oci-proxy["europe-north1"] +} + +moved { + from = google_cloud_run_service.oci-proxy["europe-southwest1"] + to = module.oci-proxy.google_cloud_run_service.oci-proxy["europe-southwest1"] +} + +moved { + from = google_cloud_run_service.oci-proxy["europe-west1"] + to = module.oci-proxy.google_cloud_run_service.oci-proxy["europe-west1"] +} + +moved { + from = google_cloud_run_service.oci-proxy["europe-west2"] + to = module.oci-proxy.google_cloud_run_service.oci-proxy["europe-west2"] +} + +moved { + from = google_cloud_run_service.oci-proxy["europe-west4"] + to = module.oci-proxy.google_cloud_run_service.oci-proxy["europe-west4"] +} + +moved { + from = google_cloud_run_service.oci-proxy["europe-west8"] + to = module.oci-proxy.google_cloud_run_service.oci-proxy["europe-west8"] +} + +moved { + from = google_cloud_run_service.oci-proxy["europe-west9"] + to = module.oci-proxy.google_cloud_run_service.oci-proxy["europe-west9"] +} + +moved { + from = google_cloud_run_service.oci-proxy["southamerica-west1"] + to = module.oci-proxy.google_cloud_run_service.oci-proxy["southamerica-west1"] +} + +moved { + from = google_cloud_run_service.oci-proxy["us-central1"] + to = module.oci-proxy.google_cloud_run_service.oci-proxy["us-central1"] +} + +moved { + from = google_cloud_run_service.oci-proxy["us-east1"] + to = module.oci-proxy.google_cloud_run_service.oci-proxy["us-east1"] +} + +moved { + from = google_cloud_run_service.oci-proxy["us-east4"] + to = module.oci-proxy.google_cloud_run_service.oci-proxy["us-east4"] +} + +moved { + from = google_cloud_run_service.oci-proxy["us-east5"] + to = module.oci-proxy.google_cloud_run_service.oci-proxy["us-east5"] +} + +moved { + from = google_cloud_run_service.oci-proxy["us-south1"] + to = module.oci-proxy.google_cloud_run_service.oci-proxy["us-south1"] +} + +moved { + from = google_cloud_run_service.oci-proxy["us-west1"] + to = module.oci-proxy.google_cloud_run_service.oci-proxy["us-west1"] +} + +moved { + from = google_cloud_run_service.oci-proxy["us-west2"] + to = module.oci-proxy.google_cloud_run_service.oci-proxy["us-west2"] +} + +moved { + from = google_cloud_run_service.oci-proxy["australia-southeast1"] + to = module.oci-proxy.google_cloud_run_service.oci-proxy["australia-southeast1"] +} + +/* again but for iam */ + +moved { + from = google_cloud_run_service_iam_member.allUsers["asia-east1"] + to = module.oci-proxy.google_cloud_run_service_iam_member.allUsers["asia-east1"] +} + +moved { + from = google_cloud_run_service_iam_member.allUsers["asia-northeast1"] + to = module.oci-proxy.google_cloud_run_service_iam_member.allUsers["asia-northeast1"] +} + +moved { + from = google_cloud_run_service_iam_member.allUsers["asia-northeast2"] + to = module.oci-proxy.google_cloud_run_service_iam_member.allUsers["asia-northeast2"] +} + +moved { + from = google_cloud_run_service_iam_member.allUsers["asia-south1"] + to = module.oci-proxy.google_cloud_run_service_iam_member.allUsers["asia-south1"] +} + +moved { + from = google_cloud_run_service_iam_member.allUsers["europe-north1"] + to = module.oci-proxy.google_cloud_run_service_iam_member.allUsers["europe-north1"] +} + +moved { + from = google_cloud_run_service_iam_member.allUsers["europe-southwest1"] + to = module.oci-proxy.google_cloud_run_service_iam_member.allUsers["europe-southwest1"] +} + +moved { + from = google_cloud_run_service_iam_member.allUsers["europe-west1"] + to = module.oci-proxy.google_cloud_run_service_iam_member.allUsers["europe-west1"] +} + +moved { + from = google_cloud_run_service_iam_member.allUsers["europe-west2"] + to = module.oci-proxy.google_cloud_run_service_iam_member.allUsers["europe-west2"] +} + +moved { + from = google_cloud_run_service_iam_member.allUsers["europe-west4"] + to = module.oci-proxy.google_cloud_run_service_iam_member.allUsers["europe-west4"] +} + +moved { + from = google_cloud_run_service_iam_member.allUsers["europe-west8"] + to = module.oci-proxy.google_cloud_run_service_iam_member.allUsers["europe-west8"] +} + +moved { + from = google_cloud_run_service_iam_member.allUsers["europe-west9"] + to = module.oci-proxy.google_cloud_run_service_iam_member.allUsers["europe-west9"] +} + +moved { + from = google_cloud_run_service_iam_member.allUsers["southamerica-west1"] + to = module.oci-proxy.google_cloud_run_service_iam_member.allUsers["southamerica-west1"] +} + +moved { + from = google_cloud_run_service_iam_member.allUsers["us-central1"] + to = module.oci-proxy.google_cloud_run_service_iam_member.allUsers["us-central1"] +} + +moved { + from = google_cloud_run_service_iam_member.allUsers["us-east1"] + to = module.oci-proxy.google_cloud_run_service_iam_member.allUsers["us-east1"] +} + +moved { + from = google_cloud_run_service_iam_member.allUsers["us-east4"] + to = module.oci-proxy.google_cloud_run_service_iam_member.allUsers["us-east4"] +} + +moved { + from = google_cloud_run_service_iam_member.allUsers["us-east5"] + to = module.oci-proxy.google_cloud_run_service_iam_member.allUsers["us-east5"] +} + +moved { + from = google_cloud_run_service_iam_member.allUsers["us-south1"] + to = module.oci-proxy.google_cloud_run_service_iam_member.allUsers["us-south1"] +} + +moved { + from = google_cloud_run_service_iam_member.allUsers["us-west1"] + to = module.oci-proxy.google_cloud_run_service_iam_member.allUsers["us-west1"] +} + +moved { + from = google_cloud_run_service_iam_member.allUsers["us-west2"] + to = module.oci-proxy.google_cloud_run_service_iam_member.allUsers["us-west2"] +} + +moved { + from = google_cloud_run_service_iam_member.allUsers["australia-southeast1"] + to = module.oci-proxy.google_cloud_run_service_iam_member.allUsers["australia-southeast1"] +} + +/* again but for network endpoint groups */ + +moved { + from = google_compute_region_network_endpoint_group.oci-proxy["asia-east1"] + to = module.oci-proxy.google_compute_region_network_endpoint_group.oci-proxy["asia-east1"] +} + +moved { + from = google_compute_region_network_endpoint_group.oci-proxy["asia-northeast1"] + to = module.oci-proxy.google_compute_region_network_endpoint_group.oci-proxy["asia-northeast1"] +} + +moved { + from = google_compute_region_network_endpoint_group.oci-proxy["asia-northeast2"] + to = module.oci-proxy.google_compute_region_network_endpoint_group.oci-proxy["asia-northeast2"] +} + +moved { + from = google_compute_region_network_endpoint_group.oci-proxy["asia-south1"] + to = module.oci-proxy.google_compute_region_network_endpoint_group.oci-proxy["asia-south1"] +} + +moved { + from = google_compute_region_network_endpoint_group.oci-proxy["europe-north1"] + to = module.oci-proxy.google_compute_region_network_endpoint_group.oci-proxy["europe-north1"] +} + +moved { + from = google_compute_region_network_endpoint_group.oci-proxy["europe-southwest1"] + to = module.oci-proxy.google_compute_region_network_endpoint_group.oci-proxy["europe-southwest1"] +} + +moved { + from = google_compute_region_network_endpoint_group.oci-proxy["europe-west1"] + to = module.oci-proxy.google_compute_region_network_endpoint_group.oci-proxy["europe-west1"] +} + +moved { + from = google_compute_region_network_endpoint_group.oci-proxy["europe-west2"] + to = module.oci-proxy.google_compute_region_network_endpoint_group.oci-proxy["europe-west2"] +} + +moved { + from = google_compute_region_network_endpoint_group.oci-proxy["europe-west4"] + to = module.oci-proxy.google_compute_region_network_endpoint_group.oci-proxy["europe-west4"] +} + +moved { + from = google_compute_region_network_endpoint_group.oci-proxy["europe-west8"] + to = module.oci-proxy.google_compute_region_network_endpoint_group.oci-proxy["europe-west8"] +} + +moved { + from = google_compute_region_network_endpoint_group.oci-proxy["europe-west9"] + to = module.oci-proxy.google_compute_region_network_endpoint_group.oci-proxy["europe-west9"] +} + +moved { + from = google_compute_region_network_endpoint_group.oci-proxy["southamerica-west1"] + to = module.oci-proxy.google_compute_region_network_endpoint_group.oci-proxy["southamerica-west1"] +} + +moved { + from = google_compute_region_network_endpoint_group.oci-proxy["us-central1"] + to = module.oci-proxy.google_compute_region_network_endpoint_group.oci-proxy["us-central1"] +} + +moved { + from = google_compute_region_network_endpoint_group.oci-proxy["us-east1"] + to = module.oci-proxy.google_compute_region_network_endpoint_group.oci-proxy["us-east1"] +} + +moved { + from = google_compute_region_network_endpoint_group.oci-proxy["us-east4"] + to = module.oci-proxy.google_compute_region_network_endpoint_group.oci-proxy["us-east4"] +} + +moved { + from = google_compute_region_network_endpoint_group.oci-proxy["us-east5"] + to = module.oci-proxy.google_compute_region_network_endpoint_group.oci-proxy["us-east5"] +} + +moved { + from = google_compute_region_network_endpoint_group.oci-proxy["us-south1"] + to = module.oci-proxy.google_compute_region_network_endpoint_group.oci-proxy["us-south1"] +} + +moved { + from = google_compute_region_network_endpoint_group.oci-proxy["us-west1"] + to = module.oci-proxy.google_compute_region_network_endpoint_group.oci-proxy["us-west1"] +} + +moved { + from = google_compute_region_network_endpoint_group.oci-proxy["us-west2"] + to = module.oci-proxy.google_compute_region_network_endpoint_group.oci-proxy["us-west2"] +} + +moved { + from = google_compute_region_network_endpoint_group.oci-proxy["australia-southeast1"] + to = module.oci-proxy.google_compute_region_network_endpoint_group.oci-proxy["australia-southeast1"] +} + + + +moved { + from = google_compute_security_policy.cloud-armor + to = module.oci-proxy.google_compute_security_policy.cloud-armor +} + +moved { + from = google_project_iam_member.k8s_infra_oci_proxy_admins + to = module.oci-proxy.google_project_iam_member.k8s_infra_oci_proxy_admins +} + +moved { + from = google_project_service.project["compute.googleapis.com"] + to = module.oci-proxy.google_project_service.project["compute.googleapis.com"] +} + +moved { + from = google_project_service.project["containerregistry.googleapis.com"] + to = module.oci-proxy.google_project_service.project["containerregistry.googleapis.com"] +} + +moved { + from = google_project_service.project["logging.googleapis.com"] + to = module.oci-proxy.google_project_service.project["logging.googleapis.com"] +} + +moved { + from = google_project_service.project["monitoring.googleapis.com"] + to = module.oci-proxy.google_project_service.project["monitoring.googleapis.com"] +} + +moved { + from = google_project_service.project["oslogin.googleapis.com"] + to = module.oci-proxy.google_project_service.project["oslogin.googleapis.com"] +} + +moved { + from = google_project_service.project["pubsub.googleapis.com"] + to = module.oci-proxy.google_project_service.project["pubsub.googleapis.com"] +} + +moved { + from = google_project_service.project["run.googleapis.com"] + to = module.oci-proxy.google_project_service.project["run.googleapis.com"] +} + + +moved { + from = google_project_service.project["storage-component.googleapis.com"] + to = module.oci-proxy.google_project_service.project["storage-component.googleapis.com"] +} + +moved { + from = google_project_service.project["storage-api.googleapis.com"] + to = module.oci-proxy.google_project_service.project["storage-api.googleapis.com"] +} + +moved { + from = google_project_service.project["storage-component.googleapis.com"] + to = module.oci-proxy.google_project_service.project["storage-component.googleapis.com"] +} + +moved { + from = google_project.project + to = module.oci-proxy.google_project.project +} + +moved { + from = google_service_account.oci-proxy + to = module.oci-proxy.google_service_account.oci-proxy +} + +moved { + from = google_monitoring_notification_channel.emails + to = module.oci-proxy.google_monitoring_notification_channel.emails +} + +moved { + from = module.alerts.google_monitoring_alert_policy.ssl_cert_expiration_alert + to = module.oci-proxy.module.alerts.google_monitoring_alert_policy.ssl_cert_expiration_alert +} + +moved { + from = module.alerts.google_monitoring_uptime_check_config.uptime_check + to = module.oci-proxy.module.alerts.google_monitoring_uptime_check_config.uptime_check +} + +moved { + from = module.alerts.google_monitoring_alert_policy.uptime_alert + to = module.oci-proxy.module.alerts.google_monitoring_alert_policy.uptime_alert +} + +moved { + from = module.lb-http.google_compute_backend_service.default["default"] + to = module.oci-proxy.module.lb-http.google_compute_backend_service.default["default"] +} + +moved { + from = module.lb-http.google_compute_global_forwarding_rule.http[0] + to = module.oci-proxy.module.lb-http.google_compute_global_forwarding_rule.http[0] +} + +moved { + from = module.lb-http.google_compute_global_forwarding_rule.http_ipv6[0] + to = module.oci-proxy.module.lb-http.google_compute_global_forwarding_rule.http_ipv6[0] +} + +moved { + from = module.lb-http.google_compute_global_forwarding_rule.https[0] + to = module.oci-proxy.module.lb-http.google_compute_global_forwarding_rule.https[0] +} + +moved { + from = module.lb-http.google_compute_global_forwarding_rule.https_ipv6[0] + to = module.oci-proxy.module.lb-http.google_compute_global_forwarding_rule.https_ipv6[0] +} + +moved { + from = module.lb-http.google_compute_managed_ssl_certificate.default[0] + to = module.oci-proxy.module.lb-http.google_compute_managed_ssl_certificate.default[0] +} + +moved { + from = module.lb-http.google_compute_target_http_proxy.default[0] + to = module.oci-proxy.module.lb-http.google_compute_target_http_proxy.default[0] +} + +moved { + from = module.lb-http.google_compute_target_https_proxy.default[0] + to = module.oci-proxy.module.lb-http.google_compute_target_https_proxy.default[0] +} + +moved { + from = module.lb-http.google_compute_url_map.default[0] + to = module.oci-proxy.module.lb-http.google_compute_url_map.default[0] +} + +moved { + from = module.lb-http.google_compute_url_map.https_redirect[0] + to = module.oci-proxy.module.lb-http.google_compute_url_map.https_redirect[0] +} + +moved { + from = module.lb-http.random_id.certificate[0] + to = module.oci-proxy.module.lb-http.random_id.certificate[0] +} + +moved { + from = google_compute_global_address.default_ipv4 + to = module.oci-proxy.google_compute_global_address.default_ipv4 +} + +moved { + from = google_compute_global_address.default_ipv6 + to = module.oci-proxy.google_compute_global_address.default_ipv6 +} From 0e7ab55d1ce8cbf1eebd11cbe9d951cc2d73a18f Mon Sep 17 00:00:00 2001 From: Benjamin Elder Date: Sun, 2 Apr 2023 19:38:20 -0700 Subject: [PATCH 5/5] update to 1.3.0 to support cross-module moves --- infra/gcp/terraform/k8s-infra-oci-proxy-prod/versions.tf | 2 +- infra/gcp/terraform/modules/monitoring/uptime-alert/versions.tf | 2 +- infra/gcp/terraform/modules/oci-proxy/versions.tf | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/infra/gcp/terraform/k8s-infra-oci-proxy-prod/versions.tf b/infra/gcp/terraform/k8s-infra-oci-proxy-prod/versions.tf index 03a7450e90d..5a04349ae3b 100644 --- a/infra/gcp/terraform/k8s-infra-oci-proxy-prod/versions.tf +++ b/infra/gcp/terraform/k8s-infra-oci-proxy-prod/versions.tf @@ -20,5 +20,5 @@ This file defines: */ terraform { - required_version = "~> 1.2.0" + required_version = "~> 1.3.0" } diff --git a/infra/gcp/terraform/modules/monitoring/uptime-alert/versions.tf b/infra/gcp/terraform/modules/monitoring/uptime-alert/versions.tf index ed4b4bb0486..9868dcc8437 100644 --- a/infra/gcp/terraform/modules/monitoring/uptime-alert/versions.tf +++ b/infra/gcp/terraform/modules/monitoring/uptime-alert/versions.tf @@ -15,7 +15,7 @@ limitations under the License. */ terraform { - required_version = "~> 1.2.0" + required_version = ">= 1.2.0" required_providers { google = { diff --git a/infra/gcp/terraform/modules/oci-proxy/versions.tf b/infra/gcp/terraform/modules/oci-proxy/versions.tf index 03a7450e90d..5a04349ae3b 100644 --- a/infra/gcp/terraform/modules/oci-proxy/versions.tf +++ b/infra/gcp/terraform/modules/oci-proxy/versions.tf @@ -20,5 +20,5 @@ This file defines: */ terraform { - required_version = "~> 1.2.0" + required_version = "~> 1.3.0" }