diff --git a/infra/gcp/terraform/k8s-infra-oci-proxy-prod/moved.tf b/infra/gcp/terraform/k8s-infra-oci-proxy-prod/moved.tf new file mode 100644 index 00000000000..b128f146db6 --- /dev/null +++ b/infra/gcp/terraform/k8s-infra-oci-proxy-prod/moved.tf @@ -0,0 +1,482 @@ +/* +Copyright 2023 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// All of the variables below were moved when refactoring to a common +// module between prod and staging + +/* we have to do this once per region ... */ + +moved { + from = google_cloud_run_service.oci-proxy["asia-east1"] + to = module.oci-proxy.google_cloud_run_service.oci-proxy["asia-east1"] +} + +moved { + from = google_cloud_run_service.oci-proxy["asia-northeast1"] + to = module.oci-proxy.google_cloud_run_service.oci-proxy["asia-northeast1"] +} + +moved { + from = google_cloud_run_service.oci-proxy["asia-northeast2"] + to = module.oci-proxy.google_cloud_run_service.oci-proxy["asia-northeast2"] +} + +moved { + from = google_cloud_run_service.oci-proxy["asia-south1"] + to = module.oci-proxy.google_cloud_run_service.oci-proxy["asia-south1"] +} + +moved { + from = google_cloud_run_service.oci-proxy["europe-north1"] + to = module.oci-proxy.google_cloud_run_service.oci-proxy["europe-north1"] +} + +moved { + from = google_cloud_run_service.oci-proxy["europe-southwest1"] + to = module.oci-proxy.google_cloud_run_service.oci-proxy["europe-southwest1"] +} + +moved { + from = google_cloud_run_service.oci-proxy["europe-west1"] + to = module.oci-proxy.google_cloud_run_service.oci-proxy["europe-west1"] +} + +moved { + from = google_cloud_run_service.oci-proxy["europe-west2"] + to = module.oci-proxy.google_cloud_run_service.oci-proxy["europe-west2"] +} + +moved { + from = google_cloud_run_service.oci-proxy["europe-west4"] + to = module.oci-proxy.google_cloud_run_service.oci-proxy["europe-west4"] +} + +moved { + from = google_cloud_run_service.oci-proxy["europe-west8"] + to = module.oci-proxy.google_cloud_run_service.oci-proxy["europe-west8"] +} + +moved { + from = google_cloud_run_service.oci-proxy["europe-west9"] + to = module.oci-proxy.google_cloud_run_service.oci-proxy["europe-west9"] +} + +moved { + from = google_cloud_run_service.oci-proxy["southamerica-west1"] + to = module.oci-proxy.google_cloud_run_service.oci-proxy["southamerica-west1"] +} + +moved { + from = google_cloud_run_service.oci-proxy["us-central1"] + to = module.oci-proxy.google_cloud_run_service.oci-proxy["us-central1"] +} + +moved { + from = google_cloud_run_service.oci-proxy["us-east1"] + to = module.oci-proxy.google_cloud_run_service.oci-proxy["us-east1"] +} + +moved { + from = google_cloud_run_service.oci-proxy["us-east4"] + to = module.oci-proxy.google_cloud_run_service.oci-proxy["us-east4"] +} + +moved { + from = google_cloud_run_service.oci-proxy["us-east5"] + to = module.oci-proxy.google_cloud_run_service.oci-proxy["us-east5"] +} + +moved { + from = google_cloud_run_service.oci-proxy["us-south1"] + to = module.oci-proxy.google_cloud_run_service.oci-proxy["us-south1"] +} + +moved { + from = google_cloud_run_service.oci-proxy["us-west1"] + to = module.oci-proxy.google_cloud_run_service.oci-proxy["us-west1"] +} + +moved { + from = google_cloud_run_service.oci-proxy["us-west2"] + to = module.oci-proxy.google_cloud_run_service.oci-proxy["us-west2"] +} + +moved { + from = google_cloud_run_service.oci-proxy["australia-southeast1"] + to = module.oci-proxy.google_cloud_run_service.oci-proxy["australia-southeast1"] +} + +/* again but for iam */ + +moved { + from = google_cloud_run_service_iam_member.allUsers["asia-east1"] + to = module.oci-proxy.google_cloud_run_service_iam_member.allUsers["asia-east1"] +} + +moved { + from = google_cloud_run_service_iam_member.allUsers["asia-northeast1"] + to = module.oci-proxy.google_cloud_run_service_iam_member.allUsers["asia-northeast1"] +} + +moved { + from = google_cloud_run_service_iam_member.allUsers["asia-northeast2"] + to = module.oci-proxy.google_cloud_run_service_iam_member.allUsers["asia-northeast2"] +} + +moved { + from = google_cloud_run_service_iam_member.allUsers["asia-south1"] + to = module.oci-proxy.google_cloud_run_service_iam_member.allUsers["asia-south1"] +} + +moved { + from = google_cloud_run_service_iam_member.allUsers["europe-north1"] + to = module.oci-proxy.google_cloud_run_service_iam_member.allUsers["europe-north1"] +} + +moved { + from = google_cloud_run_service_iam_member.allUsers["europe-southwest1"] + to = module.oci-proxy.google_cloud_run_service_iam_member.allUsers["europe-southwest1"] +} + +moved { + from = google_cloud_run_service_iam_member.allUsers["europe-west1"] + to = module.oci-proxy.google_cloud_run_service_iam_member.allUsers["europe-west1"] +} + +moved { + from = google_cloud_run_service_iam_member.allUsers["europe-west2"] + to = module.oci-proxy.google_cloud_run_service_iam_member.allUsers["europe-west2"] +} + +moved { + from = google_cloud_run_service_iam_member.allUsers["europe-west4"] + to = module.oci-proxy.google_cloud_run_service_iam_member.allUsers["europe-west4"] +} + +moved { + from = google_cloud_run_service_iam_member.allUsers["europe-west8"] + to = module.oci-proxy.google_cloud_run_service_iam_member.allUsers["europe-west8"] +} + +moved { + from = google_cloud_run_service_iam_member.allUsers["europe-west9"] + to = module.oci-proxy.google_cloud_run_service_iam_member.allUsers["europe-west9"] +} + +moved { + from = google_cloud_run_service_iam_member.allUsers["southamerica-west1"] + to = module.oci-proxy.google_cloud_run_service_iam_member.allUsers["southamerica-west1"] +} + +moved { + from = google_cloud_run_service_iam_member.allUsers["us-central1"] + to = module.oci-proxy.google_cloud_run_service_iam_member.allUsers["us-central1"] +} + +moved { + from = google_cloud_run_service_iam_member.allUsers["us-east1"] + to = module.oci-proxy.google_cloud_run_service_iam_member.allUsers["us-east1"] +} + +moved { + from = google_cloud_run_service_iam_member.allUsers["us-east4"] + to = module.oci-proxy.google_cloud_run_service_iam_member.allUsers["us-east4"] +} + +moved { + from = google_cloud_run_service_iam_member.allUsers["us-east5"] + to = module.oci-proxy.google_cloud_run_service_iam_member.allUsers["us-east5"] +} + +moved { + from = google_cloud_run_service_iam_member.allUsers["us-south1"] + to = module.oci-proxy.google_cloud_run_service_iam_member.allUsers["us-south1"] +} + +moved { + from = google_cloud_run_service_iam_member.allUsers["us-west1"] + to = module.oci-proxy.google_cloud_run_service_iam_member.allUsers["us-west1"] +} + +moved { + from = google_cloud_run_service_iam_member.allUsers["us-west2"] + to = module.oci-proxy.google_cloud_run_service_iam_member.allUsers["us-west2"] +} + +moved { + from = google_cloud_run_service_iam_member.allUsers["australia-southeast1"] + to = module.oci-proxy.google_cloud_run_service_iam_member.allUsers["australia-southeast1"] +} + +/* again but for network endpoint groups */ + +moved { + from = google_compute_region_network_endpoint_group.oci-proxy["asia-east1"] + to = module.oci-proxy.google_compute_region_network_endpoint_group.oci-proxy["asia-east1"] +} + +moved { + from = google_compute_region_network_endpoint_group.oci-proxy["asia-northeast1"] + to = module.oci-proxy.google_compute_region_network_endpoint_group.oci-proxy["asia-northeast1"] +} + +moved { + from = google_compute_region_network_endpoint_group.oci-proxy["asia-northeast2"] + to = module.oci-proxy.google_compute_region_network_endpoint_group.oci-proxy["asia-northeast2"] +} + +moved { + from = google_compute_region_network_endpoint_group.oci-proxy["asia-south1"] + to = module.oci-proxy.google_compute_region_network_endpoint_group.oci-proxy["asia-south1"] +} + +moved { + from = google_compute_region_network_endpoint_group.oci-proxy["europe-north1"] + to = module.oci-proxy.google_compute_region_network_endpoint_group.oci-proxy["europe-north1"] +} + +moved { + from = google_compute_region_network_endpoint_group.oci-proxy["europe-southwest1"] + to = module.oci-proxy.google_compute_region_network_endpoint_group.oci-proxy["europe-southwest1"] +} + +moved { + from = google_compute_region_network_endpoint_group.oci-proxy["europe-west1"] + to = module.oci-proxy.google_compute_region_network_endpoint_group.oci-proxy["europe-west1"] +} + +moved { + from = google_compute_region_network_endpoint_group.oci-proxy["europe-west2"] + to = module.oci-proxy.google_compute_region_network_endpoint_group.oci-proxy["europe-west2"] +} + +moved { + from = google_compute_region_network_endpoint_group.oci-proxy["europe-west4"] + to = module.oci-proxy.google_compute_region_network_endpoint_group.oci-proxy["europe-west4"] +} + +moved { + from = google_compute_region_network_endpoint_group.oci-proxy["europe-west8"] + to = module.oci-proxy.google_compute_region_network_endpoint_group.oci-proxy["europe-west8"] +} + +moved { + from = google_compute_region_network_endpoint_group.oci-proxy["europe-west9"] + to = module.oci-proxy.google_compute_region_network_endpoint_group.oci-proxy["europe-west9"] +} + +moved { + from = google_compute_region_network_endpoint_group.oci-proxy["southamerica-west1"] + to = module.oci-proxy.google_compute_region_network_endpoint_group.oci-proxy["southamerica-west1"] +} + +moved { + from = google_compute_region_network_endpoint_group.oci-proxy["us-central1"] + to = module.oci-proxy.google_compute_region_network_endpoint_group.oci-proxy["us-central1"] +} + +moved { + from = google_compute_region_network_endpoint_group.oci-proxy["us-east1"] + to = module.oci-proxy.google_compute_region_network_endpoint_group.oci-proxy["us-east1"] +} + +moved { + from = google_compute_region_network_endpoint_group.oci-proxy["us-east4"] + to = module.oci-proxy.google_compute_region_network_endpoint_group.oci-proxy["us-east4"] +} + +moved { + from = google_compute_region_network_endpoint_group.oci-proxy["us-east5"] + to = module.oci-proxy.google_compute_region_network_endpoint_group.oci-proxy["us-east5"] +} + +moved { + from = google_compute_region_network_endpoint_group.oci-proxy["us-south1"] + to = module.oci-proxy.google_compute_region_network_endpoint_group.oci-proxy["us-south1"] +} + +moved { + from = google_compute_region_network_endpoint_group.oci-proxy["us-west1"] + to = module.oci-proxy.google_compute_region_network_endpoint_group.oci-proxy["us-west1"] +} + +moved { + from = google_compute_region_network_endpoint_group.oci-proxy["us-west2"] + to = module.oci-proxy.google_compute_region_network_endpoint_group.oci-proxy["us-west2"] +} + +moved { + from = google_compute_region_network_endpoint_group.oci-proxy["australia-southeast1"] + to = module.oci-proxy.google_compute_region_network_endpoint_group.oci-proxy["australia-southeast1"] +} + + + +moved { + from = google_compute_security_policy.cloud-armor + to = module.oci-proxy.google_compute_security_policy.cloud-armor +} + +moved { + from = google_project_iam_member.k8s_infra_oci_proxy_admins + to = module.oci-proxy.google_project_iam_member.k8s_infra_oci_proxy_admins +} + +moved { + from = google_project_service.project["compute.googleapis.com"] + to = module.oci-proxy.google_project_service.project["compute.googleapis.com"] +} + +moved { + from = google_project_service.project["containerregistry.googleapis.com"] + to = module.oci-proxy.google_project_service.project["containerregistry.googleapis.com"] +} + +moved { + from = google_project_service.project["logging.googleapis.com"] + to = module.oci-proxy.google_project_service.project["logging.googleapis.com"] +} + +moved { + from = google_project_service.project["monitoring.googleapis.com"] + to = module.oci-proxy.google_project_service.project["monitoring.googleapis.com"] +} + +moved { + from = google_project_service.project["oslogin.googleapis.com"] + to = module.oci-proxy.google_project_service.project["oslogin.googleapis.com"] +} + +moved { + from = google_project_service.project["pubsub.googleapis.com"] + to = module.oci-proxy.google_project_service.project["pubsub.googleapis.com"] +} + +moved { + from = google_project_service.project["run.googleapis.com"] + to = module.oci-proxy.google_project_service.project["run.googleapis.com"] +} + + +moved { + from = google_project_service.project["storage-component.googleapis.com"] + to = module.oci-proxy.google_project_service.project["storage-component.googleapis.com"] +} + +moved { + from = google_project_service.project["storage-api.googleapis.com"] + to = module.oci-proxy.google_project_service.project["storage-api.googleapis.com"] +} + +moved { + from = google_project_service.project["storage-component.googleapis.com"] + to = module.oci-proxy.google_project_service.project["storage-component.googleapis.com"] +} + +moved { + from = google_project.project + to = module.oci-proxy.google_project.project +} + +moved { + from = google_service_account.oci-proxy + to = module.oci-proxy.google_service_account.oci-proxy +} + +moved { + from = google_monitoring_notification_channel.emails + to = module.oci-proxy.google_monitoring_notification_channel.emails +} + +moved { + from = module.alerts.google_monitoring_alert_policy.ssl_cert_expiration_alert + to = module.oci-proxy.module.alerts.google_monitoring_alert_policy.ssl_cert_expiration_alert +} + +moved { + from = module.alerts.google_monitoring_uptime_check_config.uptime_check + to = module.oci-proxy.module.alerts.google_monitoring_uptime_check_config.uptime_check +} + +moved { + from = module.alerts.google_monitoring_alert_policy.uptime_alert + to = module.oci-proxy.module.alerts.google_monitoring_alert_policy.uptime_alert +} + +moved { + from = module.lb-http.google_compute_backend_service.default["default"] + to = module.oci-proxy.module.lb-http.google_compute_backend_service.default["default"] +} + +moved { + from = module.lb-http.google_compute_global_forwarding_rule.http[0] + to = module.oci-proxy.module.lb-http.google_compute_global_forwarding_rule.http[0] +} + +moved { + from = module.lb-http.google_compute_global_forwarding_rule.http_ipv6[0] + to = module.oci-proxy.module.lb-http.google_compute_global_forwarding_rule.http_ipv6[0] +} + +moved { + from = module.lb-http.google_compute_global_forwarding_rule.https[0] + to = module.oci-proxy.module.lb-http.google_compute_global_forwarding_rule.https[0] +} + +moved { + from = module.lb-http.google_compute_global_forwarding_rule.https_ipv6[0] + to = module.oci-proxy.module.lb-http.google_compute_global_forwarding_rule.https_ipv6[0] +} + +moved { + from = module.lb-http.google_compute_managed_ssl_certificate.default[0] + to = module.oci-proxy.module.lb-http.google_compute_managed_ssl_certificate.default[0] +} + +moved { + from = module.lb-http.google_compute_target_http_proxy.default[0] + to = module.oci-proxy.module.lb-http.google_compute_target_http_proxy.default[0] +} + +moved { + from = module.lb-http.google_compute_target_https_proxy.default[0] + to = module.oci-proxy.module.lb-http.google_compute_target_https_proxy.default[0] +} + +moved { + from = module.lb-http.google_compute_url_map.default[0] + to = module.oci-proxy.module.lb-http.google_compute_url_map.default[0] +} + +moved { + from = module.lb-http.google_compute_url_map.https_redirect[0] + to = module.oci-proxy.module.lb-http.google_compute_url_map.https_redirect[0] +} + +moved { + from = module.lb-http.random_id.certificate[0] + to = module.oci-proxy.module.lb-http.random_id.certificate[0] +} + +moved { + from = google_compute_global_address.default_ipv4 + to = module.oci-proxy.google_compute_global_address.default_ipv4 +} + +moved { + from = google_compute_global_address.default_ipv6 + to = module.oci-proxy.google_compute_global_address.default_ipv6 +} diff --git a/infra/gcp/terraform/k8s-infra-oci-proxy-prod/oci-proxy.tf b/infra/gcp/terraform/k8s-infra-oci-proxy-prod/oci-proxy.tf index 41998e572fc..65d76e52fd1 100644 --- a/infra/gcp/terraform/k8s-infra-oci-proxy-prod/oci-proxy.tf +++ b/infra/gcp/terraform/k8s-infra-oci-proxy-prod/oci-proxy.tf @@ -14,132 +14,38 @@ See the License for the specific language governing permissions and limitations under the License. */ -data "google_organization" "org" { - domain = "kubernetes.io" +locals { + project_id = "k8s-infra-oci-proxy-prod" } -resource "google_project" "project" { - name = var.project_id - project_id = var.project_id - org_id = data.google_organization.org.org_id - billing_account = "018801-93540E-22A20E" +module "oci-proxy" { + source = "../modules/oci-proxy" + // ***** production vs staging variables inputs ***** + // + // gcr.io/k8s-staging-infra-tools/archeio:v20230310-v0.2.0@sha256:bc742c5f47a69e21e828768991853faddbe13a7f69a9da4d7d2ad16e0e55892c + // If you're interested in running this image yourself releases are available at registry.k8s.io/infra-tools/archeio + digest = "sha256:bc742c5f47a69e21e828768991853faddbe13a7f69a9da4d7d2ad16e0e55892c" + domain = "registry.k8s.io" + project_id = local.project_id + // we increase this in staging, but not in production + // we already get a lot of info from build-in cloud run logs + verbosity = "0" + // Manually created. Monitoring channels can't be created with Terraform. + // See: https://github.com/hashicorp/terraform-provider-google/issues/1134 + notification_channel_id = "15334306215710275143" } +// we only sink logs to bigquery in production +resource "google_logging_project_sink" "bigquery_sink" { + project = local.project_id + name = "registry-k8s-io-logs-sink" + destination = "bigquery.googleapis.com/projects/k8s-infra-public-pii/datasets/registry_k8s_io_logs" -// Enable services needed for the project -resource "google_project_service" "project" { - project = google_project.project.id - - for_each = toset([ - "compute.googleapis.com", - "containerregistry.googleapis.com", - "logging.googleapis.com", - "monitoring.googleapis.com", - "oslogin.googleapis.com", - "pubsub.googleapis.com", - "run.googleapis.com", - "storage-api.googleapis.com", - "storage-component.googleapis.com" - ]) - - service = each.key -} - -// Ensure k8s-infra-oci-proxy-admins@kubernetes.io has admin access to this project -resource "google_project_iam_member" "k8s_infra_oci_proxy_admins" { - project = google_project.project.id - role = "roles/owner" - member = "group:k8s-infra-oci-proxy-admins@kubernetes.io" -} - - -resource "google_service_account" "oci-proxy" { - project = google_project.project.project_id - account_id = "oci-proxy-prod" - display_name = "Minimal Service Account for OCI Proxy" -} - -// Make each service invokable by all users. -resource "google_cloud_run_service_iam_member" "allUsers" { - project = google_project.project.project_id - for_each = google_cloud_run_service.oci-proxy - - service = google_cloud_run_service.oci-proxy[each.key].name - location = google_cloud_run_service.oci-proxy[each.key].location - role = "roles/run.invoker" - member = "allUsers" -} - -resource "google_cloud_run_service" "oci-proxy" { - project = google_project.project.project_id - for_each = var.cloud_run_config - name = "${var.project_id}-${each.key}" - location = each.key - - template { - metadata { - annotations = { - "autoscaling.knative.dev/maxScale" = "10" // TODO: adjust to control costs - "run.googleapis.com/launch-stage" = "BETA" - } - } - spec { - service_account_name = google_service_account.oci-proxy.email - containers { - // NOTE: We deploy from staging because: - // - We pin by digest anyhow (so it's comparably secure) - // - We need to be able to deploy registry fixes ASAP - // - We will eventually auto-deploy staging by overriding the project and digest on the production config to avoid skew - // If you're interested in running this image yourself releases are available at registry.k8s.io/infra-tools/archeio - image = "gcr.io/k8s-staging-infra-tools/archeio@${var.digest}" - - dynamic "env" { - for_each = each.value.environment_variables - content { - name = env.value["name"] - value = env.value["value"] - } - } - - // ensure this match the value for template.spec.containers.resources.limits - env { - name = "GOMAXPROCS" - value = "1" - } - - resources { - limits = { - "cpu" = "1000m" - } - } - } - - # we can probably hit 1k QPS/core (cloud run's maximum configurable) - # but we are leaving in a little overhead, if we actually hit 1k qps in - # a region we can scale to another 1 core instance - container_concurrency = 800 - - // we only serve cheap redirects, 60s is a rather long request - timeout_seconds = 60 - } + bigquery_options { + use_partitioned_tables = false } - traffic { - percent = 100 - latest_revision = true - } + unique_writer_identity = true - depends_on = [ - google_project_service.project["run.googleapis.com"] - ] - - lifecycle { - ignore_changes = [ - // This gets added by the Cloud Run API post deploy and causes diffs, can be ignored... - template[0].metadata[0].annotations["client.knative.dev/sandbox"], - template[0].metadata[0].annotations["run.googleapis.com/user-image"], - template[0].metadata[0].annotations["run.googleapis.com/client-name"], - template[0].metadata[0].annotations["run.googleapis.com/client-version"], - ] - } + filter = "resource.type = \"cloud_run_revision\" AND log_name= \"projects/${local.project_id}/logs/run.googleapis.com%2Frequests\"" } diff --git a/infra/gcp/terraform/k8s-infra-oci-proxy-prod/terraform.tfvars b/infra/gcp/terraform/k8s-infra-oci-proxy-prod/terraform.tfvars deleted file mode 100644 index 6f698c89ff5..00000000000 --- a/infra/gcp/terraform/k8s-infra-oci-proxy-prod/terraform.tfvars +++ /dev/null @@ -1,385 +0,0 @@ -/* -Copyright 2022 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -domain = "registry.k8s.io" -project_id = "k8s-infra-oci-proxy-prod" -// gcr.io/k8s-staging-infra-tools/archeio:v20230310-v0.2.0@sha256:bc742c5f47a69e21e828768991853faddbe13a7f69a9da4d7d2ad16e0e55892c -// If you're interested in running this image yourself releases are available at registry.k8s.io/infra-tools/archeio -digest = "sha256:bc742c5f47a69e21e828768991853faddbe13a7f69a9da4d7d2ad16e0e55892c" -cloud_run_config = { - asia-east1 = { - // TODO: switch DEFAULT_AWS_BASE_URL to cloudfront or else refine the region mapping - // GCP asia-east1 is Changhua County, Taiwan - environment_variables = [ - { - name = "DEFAULT_AWS_BASE_URL", - // AWS ap-southeast-1 is Singapore - value = "https://prod-registry-k8s-io-ap-southeast-1.s3.dualstack.ap-southeast-1.amazonaws.com", - }, - { - name = "UPSTREAM_REGISTRY_ENDPOINT", - value = "https://asia-east1-docker.pkg.dev" - }, - { - name = "UPSTREAM_REGISTRY_PATH", - value = "k8s-artifacts-prod/images" - } - ] - } - // GCP asia-northeast1 is Tokyo, Japan - asia-northeast1 = { - environment_variables = [ - { - name = "DEFAULT_AWS_BASE_URL", - // AWS ap-northeast-1 is Tokyo - value = "https://prod-registry-k8s-io-ap-northeast-1.s3.dualstack.ap-northeast-1.amazonaws.com", - }, - { - name = "UPSTREAM_REGISTRY_ENDPOINT", - value = "https://asia-northeast1-docker.pkg.dev" - }, - { - name = "UPSTREAM_REGISTRY_PATH", - value = "k8s-artifacts-prod/images" - } - ] - } - // GCP asia-northeast2 is Osaka, Japan - asia-northeast2 = { - environment_variables = [ - { - name = "DEFAULT_AWS_BASE_URL", - // AWS ap-northeast-1 is Tokyo - value = "https://prod-registry-k8s-io-ap-northeast-1.s3.dualstack.ap-northeast-1.amazonaws.com", - }, - { - name = "UPSTREAM_REGISTRY_ENDPOINT", - value = "https://asia-northeast2-docker.pkg.dev" - }, - { - name = "UPSTREAM_REGISTRY_PATH", - value = "k8s-artifacts-prod/images" - } - ] - } - // GCP asia-south1 is Mumbai, India - asia-south1 = { - environment_variables = [ - { - name = "DEFAULT_AWS_BASE_URL", - // AWS ap-south-1 is Mumbai - value = "https://prod-registry-k8s-io-ap-south-1.s3.dualstack.ap-south-1.amazonaws.com", - }, - { - name = "UPSTREAM_REGISTRY_ENDPOINT", - value = "https://asia-south1-docker.pkg.dev" - }, - { - name = "UPSTREAM_REGISTRY_PATH", - value = "k8s-artifacts-prod/images" - } - ] - } - // GCP australia-southeast1 is Sydney - australia-southeast1 = { - environment_variables = [ - { - name = "DEFAULT_AWS_BASE_URL", - // AWS ap-southeast-1 is Singapore - value = "https://prod-registry-k8s-io-ap-southeast-1.s3.dualstack.ap-southeast-1.amazonaws.com", - }, - { - name = "UPSTREAM_REGISTRY_ENDPOINT", - value = "https://australia-southeast1-docker.pkg.dev" - }, - { - name = "UPSTREAM_REGISTRY_PATH", - value = "k8s-artifacts-prod/images" - } - ] - } - // GCP europe-north1 is Hamina, Finland - europe-north1 = { - environment_variables = [ - { - name = "DEFAULT_AWS_BASE_URL", - // AWS eu-central-1 is Frankfurt - value = "https://prod-registry-k8s-io-eu-central-1.s3.dualstack.eu-central-1.amazonaws.com", - }, - { - name = "UPSTREAM_REGISTRY_ENDPOINT", - value = "https://europe-north1-docker.pkg.dev" - }, - { - name = "UPSTREAM_REGISTRY_PATH", - value = "k8s-artifacts-prod/images" - } - ] - } - // GCP europe-southwest1 is Madrid, Spain - europe-southwest1 = { - environment_variables = [ - { - name = "DEFAULT_AWS_BASE_URL", - // AWS eu-central-1 is Frankfurt - value = "https://prod-registry-k8s-io-eu-central-1.s3.dualstack.eu-central-1.amazonaws.com", - }, - { - name = "UPSTREAM_REGISTRY_ENDPOINT", - value = "https://europe-southwest1-docker.pkg.dev" - }, - { - name = "UPSTREAM_REGISTRY_PATH", - value = "k8s-artifacts-prod/images" - } - ] - } - // GCP europe-west1 is St. Ghislain, Belgium - europe-west1 = { - environment_variables = [ - { - name = "DEFAULT_AWS_BASE_URL", - // AWS eu-central-1 is Frankfurt - value = "https://prod-registry-k8s-io-eu-central-1.s3.dualstack.eu-central-1.amazonaws.com", - }, - { - name = "UPSTREAM_REGISTRY_ENDPOINT", - value = "https://europe-west1-docker.pkg.dev" - }, - { - name = "UPSTREAM_REGISTRY_PATH", - value = "k8s-artifacts-prod/images" - } - ] - } - // GCP europe-west2 is London, UK - europe-west2 = { - environment_variables = [ - { - name = "DEFAULT_AWS_BASE_URL", - // AWS eu-west-2 is London - value = "https://prod-registry-k8s-io-eu-west-2.s3.dualstack.eu-west-2.amazonaws.com", - }, - { - name = "UPSTREAM_REGISTRY_ENDPOINT", - value = "https://europe-west2-docker.pkg.dev" - }, - { - name = "UPSTREAM_REGISTRY_PATH", - value = "k8s-artifacts-prod/images" - } - ] - } - // GCP europe-west4 is Eemshaven, Netherlands - europe-west4 = { - environment_variables = [ - { - name = "DEFAULT_AWS_BASE_URL", - // AWS eu-central-1 is Frankfurt - value = "https://prod-registry-k8s-io-eu-central-1.s3.dualstack.eu-central-1.amazonaws.com", - }, - { - name = "UPSTREAM_REGISTRY_ENDPOINT", - value = "https://europe-west4-docker.pkg.dev" - }, - { - name = "UPSTREAM_REGISTRY_PATH", - value = "k8s-artifacts-prod/images" - } - ] - } - // GCP europe-west8 is Milan, Italy - europe-west8 = { - environment_variables = [ - { - name = "DEFAULT_AWS_BASE_URL", - // AWS eu-central-1 is Frankfurt - value = "https://prod-registry-k8s-io-eu-central-1.s3.dualstack.eu-central-1.amazonaws.com", - }, - { - name = "UPSTREAM_REGISTRY_ENDPOINT", - value = "https://europe-west8-docker.pkg.dev" - }, - { - name = "UPSTREAM_REGISTRY_PATH", - value = "k8s-artifacts-prod/images" - } - ] - } - // GCP europe-west9 is Paris, France - europe-west9 = { - environment_variables = [ - { - name = "DEFAULT_AWS_BASE_URL", - // AWS eu-west-2 is London - value = "https://prod-registry-k8s-io-eu-west-2.s3.dualstack.eu-west-2.amazonaws.com", - }, - { - name = "UPSTREAM_REGISTRY_ENDPOINT", - value = "https://europe-west9-docker.pkg.dev" - }, - { - name = "UPSTREAM_REGISTRY_PATH", - value = "k8s-artifacts-prod/images" - } - ] - } - // GCP southamerica-west1 is Santiago, Chile - southamerica-west1 = { - environment_variables = [ - { - name = "DEFAULT_AWS_BASE_URL", - // AWS us-east-1 is Virginia, USA - // See: https://github.com/kubernetes/k8s.io/pull/4739/files#r1100667255 - value = "https://prod-registry-k8s-io-us-east-1.s3.dualstack.us-east-1.amazonaws.com", - }, - { - name = "UPSTREAM_REGISTRY_ENDPOINT", - value = "https://southamerica-west1-docker.pkg.dev" - }, - { - name = "UPSTREAM_REGISTRY_PATH", - value = "k8s-artifacts-prod/images" - } - ] - } - // GCP us-central1 is Iowa, USA - us-central1 = { - environment_variables = [ - { - name = "DEFAULT_AWS_BASE_URL", - // AWS us-east-2 is Ohio, USA - value = "https://prod-registry-k8s-io-us-east-2.s3.dualstack.us-east-2.amazonaws.com", - }, - { - name = "UPSTREAM_REGISTRY_ENDPOINT", - value = "https://us-central1-docker.pkg.dev" - }, - { - name = "UPSTREAM_REGISTRY_PATH", - value = "k8s-artifacts-prod/images" - } - ] - } - // GCP us-east1 is South Carolina, USA - us-east1 = { - environment_variables = [ - { - name = "DEFAULT_AWS_BASE_URL", - // AWS us-east-1 is Virginia, USA - value = "https://prod-registry-k8s-io-us-east-1.s3.dualstack.us-east-1.amazonaws.com", - }, - { - name = "UPSTREAM_REGISTRY_ENDPOINT", - value = "https://us-east1-docker.pkg.dev" - }, - { - name = "UPSTREAM_REGISTRY_PATH", - value = "k8s-artifacts-prod/images" - } - ] - } - // GCP us-east4 is Virginia, USA - us-east4 = { - environment_variables = [ - { - name = "DEFAULT_AWS_BASE_URL", - // AWS us-east-1 is Virginia, USA - value = "https://prod-registry-k8s-io-us-east-1.s3.dualstack.us-east-1.amazonaws.com", - }, - { - name = "UPSTREAM_REGISTRY_ENDPOINT", - value = "https://us-east4-docker.pkg.dev" - }, - { - name = "UPSTREAM_REGISTRY_PATH", - value = "k8s-artifacts-prod/images" - } - ] - } - // GCP us-east5 is Ohio, USA - us-east5 = { - environment_variables = [ - { - name = "DEFAULT_AWS_BASE_URL", - // AWS us-east-2 is Ohio, USA - value = "https://prod-registry-k8s-io-us-east-2.s3.dualstack.us-east-2.amazonaws.com", - }, - { - name = "UPSTREAM_REGISTRY_ENDPOINT", - value = "https://us-east5-docker.pkg.dev" - }, - { - name = "UPSTREAM_REGISTRY_PATH", - value = "k8s-artifacts-prod/images" - } - ] - } - // GCP us-south1 is Texas, USA - us-south1 = { - environment_variables = [ - { - name = "DEFAULT_AWS_BASE_URL", - // AWS us-east-2 is Ohio, USA - value = "https://prod-registry-k8s-io-us-east-2.s3.dualstack.us-east-2.amazonaws.com", - }, - { - name = "UPSTREAM_REGISTRY_ENDPOINT", - value = "https://us-south1-docker.pkg.dev" - }, - { - name = "UPSTREAM_REGISTRY_PATH", - value = "k8s-artifacts-prod/images" - } - ] - } - // GCP us-west1 is Oregon, USA - us-west1 = { - environment_variables = [ - { - name = "DEFAULT_AWS_BASE_URL", - // AWS us-west-2 is Oregon, USA - value = "https://prod-registry-k8s-io-us-west-2.s3.dualstack.us-west-2.amazonaws.com", - }, - { - name = "UPSTREAM_REGISTRY_ENDPOINT", - value = "https://us-west1-docker.pkg.dev" - }, - { - name = "UPSTREAM_REGISTRY_PATH", - value = "k8s-artifacts-prod/images" - } - ] - } - // GCP us-west2 is California, USA - us-west2 = { - environment_variables = [ - { - name = "DEFAULT_AWS_BASE_URL", - // AWS us-west-1 is California, USA - value = "https://prod-registry-k8s-io-us-west-1.s3.dualstack.us-west-1.amazonaws.com", - }, - { - name = "UPSTREAM_REGISTRY_ENDPOINT", - value = "https://us-west2-docker.pkg.dev" - }, - { - name = "UPSTREAM_REGISTRY_PATH", - value = "k8s-artifacts-prod/images" - } - ] - } -} diff --git a/infra/gcp/terraform/k8s-infra-oci-proxy-prod/versions.tf b/infra/gcp/terraform/k8s-infra-oci-proxy-prod/versions.tf index 03a7450e90d..5a04349ae3b 100644 --- a/infra/gcp/terraform/k8s-infra-oci-proxy-prod/versions.tf +++ b/infra/gcp/terraform/k8s-infra-oci-proxy-prod/versions.tf @@ -20,5 +20,5 @@ This file defines: */ terraform { - required_version = "~> 1.2.0" + required_version = "~> 1.3.0" } diff --git a/infra/gcp/terraform/modules/monitoring/uptime-alert/versions.tf b/infra/gcp/terraform/modules/monitoring/uptime-alert/versions.tf index ed4b4bb0486..9868dcc8437 100644 --- a/infra/gcp/terraform/modules/monitoring/uptime-alert/versions.tf +++ b/infra/gcp/terraform/modules/monitoring/uptime-alert/versions.tf @@ -15,7 +15,7 @@ limitations under the License. */ terraform { - required_version = "~> 1.2.0" + required_version = ">= 1.2.0" required_providers { google = { diff --git a/infra/gcp/terraform/modules/oci-proxy/OWNERS b/infra/gcp/terraform/modules/oci-proxy/OWNERS new file mode 100644 index 00000000000..27641792385 --- /dev/null +++ b/infra/gcp/terraform/modules/oci-proxy/OWNERS @@ -0,0 +1,3 @@ +approvers: +- ameukam +- bentheelder \ No newline at end of file diff --git a/infra/gcp/terraform/modules/oci-proxy/README.md b/infra/gcp/terraform/modules/oci-proxy/README.md new file mode 100644 index 00000000000..2238fa8d8b7 --- /dev/null +++ b/infra/gcp/terraform/modules/oci-proxy/README.md @@ -0,0 +1,9 @@ +# oci-proxy common module + +This module contains ~all of the config for oci-proxy / oci-proxy-staging. + +Staging is expected to continuously lead production rollouts and changes +will be vetted in staging before manually rolling out to production. + +The only differences between staging and production are inputs variables +to this module, such as domain and IP address. diff --git a/infra/gcp/terraform/k8s-infra-oci-proxy-prod/cloud-armor.tf b/infra/gcp/terraform/modules/oci-proxy/cloud-armor.tf similarity index 100% rename from infra/gcp/terraform/k8s-infra-oci-proxy-prod/cloud-armor.tf rename to infra/gcp/terraform/modules/oci-proxy/cloud-armor.tf diff --git a/infra/gcp/terraform/k8s-infra-oci-proxy-prod/monitoring.tf b/infra/gcp/terraform/modules/oci-proxy/monitoring.tf similarity index 89% rename from infra/gcp/terraform/k8s-infra-oci-proxy-prod/monitoring.tf rename to infra/gcp/terraform/modules/oci-proxy/monitoring.tf index 93c51ac815f..145506ddd65 100644 --- a/infra/gcp/terraform/k8s-infra-oci-proxy-prod/monitoring.tf +++ b/infra/gcp/terraform/modules/oci-proxy/monitoring.tf @@ -25,14 +25,14 @@ resource "google_monitoring_notification_channel" "emails" { module "alerts" { project_id = google_project.project.project_id - source = "../modules/monitoring/uptime-alert" + source = "../monitoring/uptime-alert" documentation_text = "${var.domain} is down" domain = var.domain notification_channels = [ # Manually created. Monitoring channels can't be created with Terraform. # See: https://github.com/hashicorp/terraform-provider-google/issues/1134 - "${google_project.project.id}/notificationChannels/15334306215710275143", + "${google_project.project.id}/notificationChannels/${var.notification_channel_id}", google_monitoring_notification_channel.emails.name, ] } diff --git a/infra/gcp/terraform/k8s-infra-oci-proxy-prod/network.tf b/infra/gcp/terraform/modules/oci-proxy/network.tf similarity index 94% rename from infra/gcp/terraform/k8s-infra-oci-proxy-prod/network.tf rename to infra/gcp/terraform/modules/oci-proxy/network.tf index 41b3dd671cf..0d4e13d4aba 100644 --- a/infra/gcp/terraform/k8s-infra-oci-proxy-prod/network.tf +++ b/infra/gcp/terraform/modules/oci-proxy/network.tf @@ -16,7 +16,7 @@ limitations under the License. resource "google_compute_global_address" "default_ipv4" { project = google_project.project.project_id - name = "k8s-infra-oci-proxy-prod" + name = google_project.project.project_id address_type = "EXTERNAL" ip_version = "IPV4" @@ -27,7 +27,7 @@ resource "google_compute_global_address" "default_ipv4" { resource "google_compute_global_address" "default_ipv6" { project = google_project.project.project_id - name = "k8s-infra-oci-proxy-prod-v6" + name = "${google_project.project.project_id}-v6" address_type = "EXTERNAL" ip_version = "IPV6" @@ -38,12 +38,12 @@ resource "google_compute_global_address" "default_ipv6" { data "google_compute_global_address" "default_ipv4" { project = google_project.project.project_id - name = "k8s-infra-oci-proxy-prod" + name = google_project.project.project_id } data "google_compute_global_address" "default_ipv6" { project = google_project.project.project_id - name = "k8s-infra-oci-proxy-prod-v6" + name = "${google_project.project.project_id}-v6" } resource "google_compute_region_network_endpoint_group" "oci-proxy" { diff --git a/infra/gcp/terraform/modules/oci-proxy/oci-proxy.tf b/infra/gcp/terraform/modules/oci-proxy/oci-proxy.tf new file mode 100644 index 00000000000..7508878ba1e --- /dev/null +++ b/infra/gcp/terraform/modules/oci-proxy/oci-proxy.tf @@ -0,0 +1,513 @@ +/* +Copyright 2022 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +locals { + cloud_run_config = { + asia-east1 = { + // TODO: switch DEFAULT_AWS_BASE_URL to cloudfront or else refine the region mapping + // GCP asia-east1 is Changhua County, Taiwan + environment_variables = [ + { + name = "DEFAULT_AWS_BASE_URL", + // AWS ap-southeast-1 is Singapore + value = "https://prod-registry-k8s-io-ap-southeast-1.s3.dualstack.ap-southeast-1.amazonaws.com", + }, + { + name = "UPSTREAM_REGISTRY_ENDPOINT", + value = "https://asia-east1-docker.pkg.dev" + }, + { + name = "UPSTREAM_REGISTRY_PATH", + value = "k8s-artifacts-prod/images" + } + ] + } + // GCP asia-northeast1 is Tokyo, Japan + asia-northeast1 = { + environment_variables = [ + { + name = "DEFAULT_AWS_BASE_URL", + // AWS ap-northeast-1 is Tokyo + value = "https://prod-registry-k8s-io-ap-northeast-1.s3.dualstack.ap-northeast-1.amazonaws.com", + }, + { + name = "UPSTREAM_REGISTRY_ENDPOINT", + value = "https://asia-northeast1-docker.pkg.dev" + }, + { + name = "UPSTREAM_REGISTRY_PATH", + value = "k8s-artifacts-prod/images" + } + ] + } + // GCP asia-northeast2 is Osaka, Japan + asia-northeast2 = { + environment_variables = [ + { + name = "DEFAULT_AWS_BASE_URL", + // AWS ap-northeast-1 is Tokyo + value = "https://prod-registry-k8s-io-ap-northeast-1.s3.dualstack.ap-northeast-1.amazonaws.com", + }, + { + name = "UPSTREAM_REGISTRY_ENDPOINT", + value = "https://asia-northeast2-docker.pkg.dev" + }, + { + name = "UPSTREAM_REGISTRY_PATH", + value = "k8s-artifacts-prod/images" + } + ] + } + // GCP asia-south1 is Mumbai, India + asia-south1 = { + environment_variables = [ + { + name = "DEFAULT_AWS_BASE_URL", + // AWS ap-south-1 is Mumbai + value = "https://prod-registry-k8s-io-ap-south-1.s3.dualstack.ap-south-1.amazonaws.com", + }, + { + name = "UPSTREAM_REGISTRY_ENDPOINT", + value = "https://asia-south1-docker.pkg.dev" + }, + { + name = "UPSTREAM_REGISTRY_PATH", + value = "k8s-artifacts-prod/images" + } + ] + } + // GCP australia-southeast1 is Sydney + australia-southeast1 = { + environment_variables = [ + { + name = "DEFAULT_AWS_BASE_URL", + // AWS ap-southeast-1 is Singapore + value = "https://prod-registry-k8s-io-ap-southeast-1.s3.dualstack.ap-southeast-1.amazonaws.com", + }, + { + name = "UPSTREAM_REGISTRY_ENDPOINT", + value = "https://australia-southeast1-docker.pkg.dev" + }, + { + name = "UPSTREAM_REGISTRY_PATH", + value = "k8s-artifacts-prod/images" + } + ] + } + // GCP europe-north1 is Hamina, Finland + europe-north1 = { + environment_variables = [ + { + name = "DEFAULT_AWS_BASE_URL", + // AWS eu-central-1 is Frankfurt + value = "https://prod-registry-k8s-io-eu-central-1.s3.dualstack.eu-central-1.amazonaws.com", + }, + { + name = "UPSTREAM_REGISTRY_ENDPOINT", + value = "https://europe-north1-docker.pkg.dev" + }, + { + name = "UPSTREAM_REGISTRY_PATH", + value = "k8s-artifacts-prod/images" + } + ] + } + // GCP europe-southwest1 is Madrid, Spain + europe-southwest1 = { + environment_variables = [ + { + name = "DEFAULT_AWS_BASE_URL", + // AWS eu-central-1 is Frankfurt + value = "https://prod-registry-k8s-io-eu-central-1.s3.dualstack.eu-central-1.amazonaws.com", + }, + { + name = "UPSTREAM_REGISTRY_ENDPOINT", + value = "https://europe-southwest1-docker.pkg.dev" + }, + { + name = "UPSTREAM_REGISTRY_PATH", + value = "k8s-artifacts-prod/images" + } + ] + } + // GCP europe-west1 is St. Ghislain, Belgium + europe-west1 = { + environment_variables = [ + { + name = "DEFAULT_AWS_BASE_URL", + // AWS eu-central-1 is Frankfurt + value = "https://prod-registry-k8s-io-eu-central-1.s3.dualstack.eu-central-1.amazonaws.com", + }, + { + name = "UPSTREAM_REGISTRY_ENDPOINT", + value = "https://europe-west1-docker.pkg.dev" + }, + { + name = "UPSTREAM_REGISTRY_PATH", + value = "k8s-artifacts-prod/images" + } + ] + } + // GCP europe-west2 is London, UK + europe-west2 = { + environment_variables = [ + { + name = "DEFAULT_AWS_BASE_URL", + // AWS eu-west-2 is London + value = "https://prod-registry-k8s-io-eu-west-2.s3.dualstack.eu-west-2.amazonaws.com", + }, + { + name = "UPSTREAM_REGISTRY_ENDPOINT", + value = "https://europe-west2-docker.pkg.dev" + }, + { + name = "UPSTREAM_REGISTRY_PATH", + value = "k8s-artifacts-prod/images" + } + ] + } + // GCP europe-west4 is Eemshaven, Netherlands + europe-west4 = { + environment_variables = [ + { + name = "DEFAULT_AWS_BASE_URL", + // AWS eu-central-1 is Frankfurt + value = "https://prod-registry-k8s-io-eu-central-1.s3.dualstack.eu-central-1.amazonaws.com", + }, + { + name = "UPSTREAM_REGISTRY_ENDPOINT", + value = "https://europe-west4-docker.pkg.dev" + }, + { + name = "UPSTREAM_REGISTRY_PATH", + value = "k8s-artifacts-prod/images" + } + ] + } + // GCP europe-west8 is Milan, Italy + europe-west8 = { + environment_variables = [ + { + name = "DEFAULT_AWS_BASE_URL", + // AWS eu-central-1 is Frankfurt + value = "https://prod-registry-k8s-io-eu-central-1.s3.dualstack.eu-central-1.amazonaws.com", + }, + { + name = "UPSTREAM_REGISTRY_ENDPOINT", + value = "https://europe-west8-docker.pkg.dev" + }, + { + name = "UPSTREAM_REGISTRY_PATH", + value = "k8s-artifacts-prod/images" + } + ] + } + // GCP europe-west9 is Paris, France + europe-west9 = { + environment_variables = [ + { + name = "DEFAULT_AWS_BASE_URL", + // AWS eu-west-2 is London + value = "https://prod-registry-k8s-io-eu-west-2.s3.dualstack.eu-west-2.amazonaws.com", + }, + { + name = "UPSTREAM_REGISTRY_ENDPOINT", + value = "https://europe-west9-docker.pkg.dev" + }, + { + name = "UPSTREAM_REGISTRY_PATH", + value = "k8s-artifacts-prod/images" + } + ] + } + // GCP southamerica-west1 is Santiago, Chile + southamerica-west1 = { + environment_variables = [ + { + name = "DEFAULT_AWS_BASE_URL", + // AWS us-east-1 is Virginia, USA + // See: https://github.com/kubernetes/k8s.io/pull/4739/files#r1100667255 + value = "https://prod-registry-k8s-io-us-east-1.s3.dualstack.us-east-1.amazonaws.com", + }, + { + name = "UPSTREAM_REGISTRY_ENDPOINT", + value = "https://southamerica-west1-docker.pkg.dev" + }, + { + name = "UPSTREAM_REGISTRY_PATH", + value = "k8s-artifacts-prod/images" + } + ] + } + // GCP us-central1 is Iowa, USA + us-central1 = { + environment_variables = [ + { + name = "DEFAULT_AWS_BASE_URL", + // AWS us-east-2 is Ohio, USA + value = "https://prod-registry-k8s-io-us-east-2.s3.dualstack.us-east-2.amazonaws.com", + }, + { + name = "UPSTREAM_REGISTRY_ENDPOINT", + value = "https://us-central1-docker.pkg.dev" + }, + { + name = "UPSTREAM_REGISTRY_PATH", + value = "k8s-artifacts-prod/images" + } + ] + } + // GCP us-east1 is South Carolina, USA + us-east1 = { + environment_variables = [ + { + name = "DEFAULT_AWS_BASE_URL", + // AWS us-east-1 is Virginia, USA + value = "https://prod-registry-k8s-io-us-east-1.s3.dualstack.us-east-1.amazonaws.com", + }, + { + name = "UPSTREAM_REGISTRY_ENDPOINT", + value = "https://us-east1-docker.pkg.dev" + }, + { + name = "UPSTREAM_REGISTRY_PATH", + value = "k8s-artifacts-prod/images" + } + ] + } + // GCP us-east4 is Virginia, USA + us-east4 = { + environment_variables = [ + { + name = "DEFAULT_AWS_BASE_URL", + // AWS us-east-1 is Virginia, USA + value = "https://prod-registry-k8s-io-us-east-1.s3.dualstack.us-east-1.amazonaws.com", + }, + { + name = "UPSTREAM_REGISTRY_ENDPOINT", + value = "https://us-east4-docker.pkg.dev" + }, + { + name = "UPSTREAM_REGISTRY_PATH", + value = "k8s-artifacts-prod/images" + } + ] + } + // GCP us-east5 is Ohio, USA + us-east5 = { + environment_variables = [ + { + name = "DEFAULT_AWS_BASE_URL", + // AWS us-east-2 is Ohio, USA + value = "https://prod-registry-k8s-io-us-east-2.s3.dualstack.us-east-2.amazonaws.com", + }, + { + name = "UPSTREAM_REGISTRY_ENDPOINT", + value = "https://us-east5-docker.pkg.dev" + }, + { + name = "UPSTREAM_REGISTRY_PATH", + value = "k8s-artifacts-prod/images" + } + ] + } + // GCP us-south1 is Texas, USA + us-south1 = { + environment_variables = [ + { + name = "DEFAULT_AWS_BASE_URL", + // AWS us-east-2 is Ohio, USA + value = "https://prod-registry-k8s-io-us-east-2.s3.dualstack.us-east-2.amazonaws.com", + }, + { + name = "UPSTREAM_REGISTRY_ENDPOINT", + value = "https://us-south1-docker.pkg.dev" + }, + { + name = "UPSTREAM_REGISTRY_PATH", + value = "k8s-artifacts-prod/images" + } + ] + } + // GCP us-west1 is Oregon, USA + us-west1 = { + environment_variables = [ + { + name = "DEFAULT_AWS_BASE_URL", + // AWS us-west-2 is Oregon, USA + value = "https://prod-registry-k8s-io-us-west-2.s3.dualstack.us-west-2.amazonaws.com", + }, + { + name = "UPSTREAM_REGISTRY_ENDPOINT", + value = "https://us-west1-docker.pkg.dev" + }, + { + name = "UPSTREAM_REGISTRY_PATH", + value = "k8s-artifacts-prod/images" + } + ] + } + // GCP us-west2 is California, USA + us-west2 = { + environment_variables = [ + { + name = "DEFAULT_AWS_BASE_URL", + // AWS us-west-1 is California, USA + value = "https://prod-registry-k8s-io-us-west-1.s3.dualstack.us-west-1.amazonaws.com", + }, + { + name = "UPSTREAM_REGISTRY_ENDPOINT", + value = "https://us-west2-docker.pkg.dev" + }, + { + name = "UPSTREAM_REGISTRY_PATH", + value = "k8s-artifacts-prod/images" + } + ] + } + } +} + +data "google_organization" "org" { + domain = "kubernetes.io" +} + +resource "google_project" "project" { + name = var.project_id + project_id = var.project_id + org_id = data.google_organization.org.org_id + billing_account = "018801-93540E-22A20E" +} + + +// Enable services needed for the project +resource "google_project_service" "project" { + project = google_project.project.id + + for_each = toset([ + "compute.googleapis.com", + "containerregistry.googleapis.com", + "logging.googleapis.com", + "monitoring.googleapis.com", + "oslogin.googleapis.com", + "pubsub.googleapis.com", + "run.googleapis.com", + "storage-api.googleapis.com", + "storage-component.googleapis.com" + ]) + + service = each.key +} + +// Ensure k8s-infra-oci-proxy-admins@kubernetes.io has admin access to this project +resource "google_project_iam_member" "k8s_infra_oci_proxy_admins" { + project = google_project.project.id + role = "roles/owner" + member = "group:k8s-infra-oci-proxy-admins@kubernetes.io" +} + + +resource "google_service_account" "oci-proxy" { + project = google_project.project.project_id + account_id = "oci-proxy-prod" + display_name = "Minimal Service Account for OCI Proxy" +} + +// Make each service invokable by all users. +resource "google_cloud_run_service_iam_member" "allUsers" { + project = google_project.project.project_id + for_each = google_cloud_run_service.oci-proxy + + service = google_cloud_run_service.oci-proxy[each.key].name + location = google_cloud_run_service.oci-proxy[each.key].location + role = "roles/run.invoker" + member = "allUsers" +} + +resource "google_cloud_run_service" "oci-proxy" { + project = google_project.project.project_id + for_each = local.cloud_run_config + name = "${var.project_id}-${each.key}" + location = each.key + + template { + metadata { + annotations = { + "autoscaling.knative.dev/maxScale" = "10" // TODO: adjust to control costs + "run.googleapis.com/launch-stage" = "BETA" + } + } + spec { + service_account_name = google_service_account.oci-proxy.email + containers { + // NOTE: We deploy from staging because: + // - We pin by digest anyhow (so it's comparably secure) + // - We need to be able to deploy registry fixes ASAP + // - We will eventually auto-deploy staging by overriding the project and digest on the production config to avoid skew + // If you're interested in running this image yourself releases are available at registry.k8s.io/infra-tools/archeio + image = "gcr.io/k8s-staging-infra-tools/archeio@${var.digest}" + args = ["-v=${var.verbosity}"] + + dynamic "env" { + for_each = each.value.environment_variables + content { + name = env.value["name"] + value = env.value["value"] + } + } + + // ensure this match the value for template.spec.containers.resources.limits + env { + name = "GOMAXPROCS" + value = "1" + } + + resources { + limits = { + "cpu" = "1000m" + } + } + } + + # we can probably hit 1k QPS/core (cloud run's maximum configurable) + # but we are leaving in a little overhead, if we actually hit 1k qps in + # a region we can scale to another 1 core instance + container_concurrency = 800 + + // we only serve cheap redirects, 60s is a rather long request + timeout_seconds = 60 + } + } + + traffic { + percent = 100 + latest_revision = true + } + + depends_on = [ + google_project_service.project["run.googleapis.com"] + ] + + lifecycle { + ignore_changes = [ + // This gets added by the Cloud Run API post deploy and causes diffs, can be ignored... + template[0].metadata[0].annotations["client.knative.dev/sandbox"], + template[0].metadata[0].annotations["run.googleapis.com/user-image"], + template[0].metadata[0].annotations["run.googleapis.com/client-name"], + template[0].metadata[0].annotations["run.googleapis.com/client-version"], + ] + } +} diff --git a/infra/gcp/terraform/k8s-infra-oci-proxy-prod/variables.tf b/infra/gcp/terraform/modules/oci-proxy/variables.tf similarity index 82% rename from infra/gcp/terraform/k8s-infra-oci-proxy-prod/variables.tf rename to infra/gcp/terraform/modules/oci-proxy/variables.tf index 0c4e1ecf4bd..048f6714a24 100644 --- a/infra/gcp/terraform/k8s-infra-oci-proxy-prod/variables.tf +++ b/infra/gcp/terraform/modules/oci-proxy/variables.tf @@ -23,11 +23,9 @@ variable "project_id" { variable "digest" { type = string } -variable "cloud_run_config" { - type = map(object({ - environment_variables = list(object({ - value = string - name = string - })) - })) +variable "verbosity" { + type = string +} +variable "notification_channel_id" { + type = string } diff --git a/infra/gcp/terraform/k8s-infra-oci-proxy-prod/logs.tf b/infra/gcp/terraform/modules/oci-proxy/versions.tf similarity index 53% rename from infra/gcp/terraform/k8s-infra-oci-proxy-prod/logs.tf rename to infra/gcp/terraform/modules/oci-proxy/versions.tf index 473e34d75ac..5a04349ae3b 100644 --- a/infra/gcp/terraform/k8s-infra-oci-proxy-prod/logs.tf +++ b/infra/gcp/terraform/modules/oci-proxy/versions.tf @@ -14,17 +14,11 @@ See the License for the specific language governing permissions and limitations under the License. */ -resource "google_logging_project_sink" "bigquery_sink" { - project = google_project.project.project_id - name = "registry-k8s-io-logs-sink" - destination = "bigquery.googleapis.com/projects/k8s-infra-public-pii/datasets/registry_k8s_io_logs" - - bigquery_options { - use_partitioned_tables = false - } - - unique_writer_identity = true - - filter = "resource.type = \"cloud_run_revision\" AND log_name= \"projects/${google_project.project.project_id}/logs/run.googleapis.com%2Frequests\"" +/* +This file defines: +- Required Terraform version +*/ +terraform { + required_version = "~> 1.3.0" }