Merge branch 'GoogleCloudPlatform:main' into main

.ci/containers/README.md

@@ -3,17 +3,27 @@ The docker images located in this folder are used by multiple builds for magic m
 
 ## Naming Convention
 
-The images are named with the languages they contain and the images are versioned with tags that indicate the version of each language contained. eg: the image `go-ruby-python` with a tag of `1.11.5-2.6.0-2.7` indicates that the image has `go 1.11.5`, `ruby 2.6.0` and `python 2.7`.
+The images are named according to their use. We have a small number of images that get reused in multiple places, based around sets of requirements shared by different parts of the build pipeline. The images are:
 
-If there are multiple images with the same language version but different libraries (gems), a `v#` is appended to differentiate. eg: `1.11.5-2.6.0-2.7-v6`
+- `gcr.io/graphite-docker-images/bash-plus`
+- `gcr.io/graphite-docker-images/build-environment`
+- `gcr.io/graphite-docker-images/go-plus`
 
 ## Updating a docker image
-The Dockerfile should be updated, then the image rebuilt and pushed to the container registry stored at the `graphite-docker-images` GCP project. To update any of the images:
+
+Before you begin, set up Docker (including configuring it to [authenticate with gcloud](https://cloud.google.com/container-registry/docs/advanced-authentication#gcloud-helper)).
 
 1. Make changes to the Dockerfile
-2. Configure docker to use gcloud auth: `gcloud auth configure-docker`
-3. Build the image: `docker build . --tag gcr.io/graphite-docker-images/go-ruby-python`
-4. Find the new image's id: `docker images`
-5. Add the appropriate tag `docker tag ac37c0af8ce7 gcr.io/graphite-docker-images/go-ruby-python:1.11.5-2.6.0-2.7-v6`
-6. Push the image: `docker push gcr.io/graphite-docker-images/go-ruby-python:1.11.5-2.6.0-2.7-v6`
-7. Check the UI and ensure the new version is available and tagged at `latest`. It must be tagged `latest` for the Kokoro builds to get the correct version.
+2. Build the image with the `testing` tag:
+   ```bash
+   sudo docker build . --tag gcr.io/graphite-docker-images/bash-plus:testing
+   ```
+3. Push the image:
+   ```bash
+   sudo docker push gcr.io/graphite-docker-images/bash-plus:testing
+   ```
+4. Update cloudbuild yaml files to reference the image you just pushed by adding the `:testing` suffix
+5. Update files that will cause the cloudbuild yaml changes (and therefore your changes) to be exercised
+   - Tip: Modifying `mmv1/third_party/terraform/services/compute/metadata.go.erb` will trigger builds for TPG, TPGB, and TGC.
+6. Create a PR with these changes.
+7. Verify that the cloudbuild steps that should use your testing image _are_ using your testing image (in the Execution Details tab for the step.)

.ci/containers/build-environment/Dockerfile

@@ -36,10 +36,6 @@ RUN apt-get update && apt-get install -y --no-install-recommends git openssh-cli
 RUN git config --global user.name "Modular Magician"
 RUN git config --global user.email "[email protected]"
 
-# Set up Github SSH cloning.
-RUN ssh-keyscan github.com >> /known_hosts
-RUN echo "UserKnownHostsFile /known_hosts" >> /etc/ssh/ssh_config
-
 RUN go install golang.org/x/tools/cmd/goimports@d088b475e3360caabc032aaee1dc66351d4e729a
 RUN go install github.com/github/[email protected]+incompatible
 

.ci/containers/go-plus/Dockerfile

@@ -20,10 +20,6 @@ ENV GOCACHE=/go/cache
 COPY --from=builder /go/pkg/mod /go/pkg/mod
 COPY --from=builder /go/cache /go/cache
 
-# Set up Github SSH cloning.
-RUN ssh-keyscan github.com >> /known_hosts
-RUN echo "UserKnownHostsFile /known_hosts" >> /etc/ssh/ssh_config
-
 RUN apt-get update && \
     apt-get install -y git jq unzip parallel curl && \
     echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && \

.ci/gcb-generate-diffs-new.yml

@@ -255,7 +255,7 @@ steps:
     - name: 'gcr.io/graphite-docker-images/go-plus'
       id: gcb-tpg-vcr-test
       entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
-      secretEnv: ["GITHUB_TOKEN_DOWNSTREAMS", "GITHUB_TOKEN_MAGIC_MODULES", "GOOGLE_BILLING_ACCOUNT", "GOOGLE_CUST_ID", "GOOGLE_FIRESTORE_PROJECT", "GOOGLE_IDENTITY_USER", "GOOGLE_MASTER_BILLING_ACCOUNT", "GOOGLE_ORG", "GOOGLE_ORG_2", "GOOGLE_ORG_DOMAIN", "GOOGLE_PROJECT", "GOOGLE_PROJECT_NUMBER", "GOOGLE_SERVICE_ACCOUNT", "SA_KEY", "GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION", "GOOGLE_TPU_V2_VM_RUNTIME_VERSION"]
+      secretEnv: ["GITHUB_TOKEN_DOWNSTREAMS", "GITHUB_TOKEN_MAGIC_MODULES", "GOOGLE_BILLING_ACCOUNT", "GOOGLE_CUST_ID", "GOOGLE_FIRESTORE_PROJECT", "GOOGLE_IDENTITY_USER", "GOOGLE_MASTER_BILLING_ACCOUNT", "GOOGLE_ORG", "GOOGLE_ORG_2", "GOOGLE_ORG_DOMAIN", "GOOGLE_PROJECT", "GOOGLE_PROJECT_NUMBER", "GOOGLE_SERVICE_ACCOUNT", "SA_KEY", "GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION"]
       waitFor: ["diff"]
       env:
         - BASE_BRANCH=$_BASE_BRANCH
@@ -277,6 +277,8 @@ steps:
       args:
           - 'request-service-reviewers'
           - $_PR_NUMBER
+      env:
+        - COMMIT_SHA=$COMMIT_SHA
 
 # Long timeout to enable waiting on VCR test
 timeout: 20000s
@@ -316,5 +318,3 @@ availableSecrets:
       env: SA_KEY
     - versionName: projects/673497134629/secrets/ci-test-public-advertised-prefix-description/versions/latest
       env: GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION
-    - versionName: projects/673497134629/secrets/ci-test-tpu-v2-vm-runtime-version/versions/latest
-      env: GOOGLE_TPU_V2_VM_RUNTIME_VERSION

.ci/gcb-push-downstream.yml

@@ -181,7 +181,7 @@ steps:
       entrypoint: '/workspace/.ci/scripts/go-plus/vcr-cassette-merger/vcr_merge.sh'
       secretEnv: ["GITHUB_TOKEN_CLASSIC", "GOOGLE_PROJECT"]
       id: vcr-merge
-      waitFor: ["tpg-push"]
+      waitFor: ["tpg-push", "tpgb-push", "tgc-push", "tf-oics-push"]
       env:
         - BASE_BRANCH=$BRANCH_NAME
       args:
@@ -206,7 +206,6 @@ steps:
         - "GOOGLE_SERVICE_ACCOUNT"
         - "SA_KEY"
         - "GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION"
-        - "GOOGLE_TPU_V2_VM_RUNTIME_VERSION"
       env:
         - "COMMIT_SHA=$COMMIT_SHA"
         - "GOOGLE_REGION=us-central1"
@@ -253,5 +252,3 @@ availableSecrets:
       env: SA_KEY
     - versionName: projects/673497134629/secrets/ci-test-public-advertised-prefix-description/versions/latest
       env: GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION
-    - versionName: projects/673497134629/secrets/ci-test-tpu-v2-vm-runtime-version/versions/latest
-      env: GOOGLE_TPU_V2_VM_RUNTIME_VERSION

.ci/gcb-vcr-nightly.yml

@@ -3,7 +3,7 @@ steps:
     - name: 'gcr.io/graphite-docker-images/go-plus'
       id: gcb-vcr-nightly
       entrypoint: '/workspace/.ci/scripts/go-plus/vcr-cassette-update/vcr_cassette_update.sh'
-      secretEnv: ["GOOGLE_BILLING_ACCOUNT", "GOOGLE_CUST_ID", "GOOGLE_FIRESTORE_PROJECT", "GOOGLE_IDENTITY_USER", "GOOGLE_MASTER_BILLING_ACCOUNT", "GOOGLE_ORG", "GOOGLE_ORG_2", "GOOGLE_ORG_DOMAIN", "GOOGLE_PROJECT", "GOOGLE_PROJECT_NUMBER", "GOOGLE_SERVICE_ACCOUNT", "SA_KEY", "GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION", "GOOGLE_TPU_V2_VM_RUNTIME_VERSION"]
+      secretEnv: ["GOOGLE_BILLING_ACCOUNT", "GOOGLE_CUST_ID", "GOOGLE_FIRESTORE_PROJECT", "GOOGLE_IDENTITY_USER", "GOOGLE_MASTER_BILLING_ACCOUNT", "GOOGLE_ORG", "GOOGLE_ORG_2", "GOOGLE_ORG_DOMAIN", "GOOGLE_PROJECT", "GOOGLE_PROJECT_NUMBER", "GOOGLE_SERVICE_ACCOUNT", "SA_KEY", "GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION"]
       args:
           - $BUILD_ID
 
@@ -41,5 +41,3 @@ availableSecrets:
       env: SA_KEY
     - versionName: projects/673497134629/secrets/ci-test-public-advertised-prefix-description/versions/latest
       env: GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION
-    - versionName: projects/673497134629/secrets/ci-test-tpu-v2-vm-runtime-version/versions/latest
-      env: GOOGLE_TPU_V2_VM_RUNTIME_VERSION

.ci/infra/terraform/README.md

@@ -48,6 +48,7 @@ After applying this configuration:
 - Create a `support@` group in the Google Workspace Admin Console, add new service account as a member, and make it an owner
 - Enroll in Cloud Armor Managed Protection Plus tier
 - Add Cloud Identity Premium Plan to the Google Workspace domain
+- Perform the Privileged Access Manager set-up https://pantheon.corp.google.com/iam-admin/pam/setup
 
 Quotas that will need to be adjusted to support all tests:
 - Project quota for the new service account

.ci/infra/terraform/main.tf

@@ -33,6 +33,12 @@ resource "google_organization_iam_member" "sa_access_boundary_admin" {
   member = google_service_account.sa.member
 }
 
+resource "google_organization_iam_member" "sa_apphub_admin" {
+  org_id = data.google_organization.org.org_id
+  role   = "roles/apphub.admin"
+  member = google_service_account.sa.member
+}
+
 resource "google_organization_iam_member" "sa_assuredworkloads_admin" {
   org_id = data.google_organization.org.org_id
   role   = "roles/assuredworkloads.admin"
@@ -63,6 +69,18 @@ resource "google_organization_iam_member" "sa_compute_xpn_admin" {
   member = google_service_account.sa.member
 }
 
+resource "google_organization_iam_member" "sa_contentwarehouse_admin" {
+  org_id = data.google_organization.org.org_id
+  role   = "roles/contentwarehouse.admin"
+  member = google_service_account.sa.member
+}
+
+resource "google_organization_iam_member" "sa_contentwarehouse_document_admin" {
+  org_id = data.google_organization.org.org_id
+  role   = "roles/contentwarehouse.documentAdmin"
+  member = google_service_account.sa.member
+}
+
 resource "google_organization_iam_member" "sa_deny_admin" {
   org_id = data.google_organization.org.org_id
   role   = "roles/iam.denyAdmin"
@@ -135,12 +153,6 @@ resource "google_organization_iam_member" "sa_storage_admin" {
   member = google_service_account.sa.member
 }
 
-resource "google_organization_iam_member" "apphub_admin" {
-  org_id = data.google_organization.org.org_id
-  role               = "roles/apphub.admin"
-  member             = google_service_account.sa.member
-}
-
 resource "google_billing_account_iam_member" "sa_master_billing_admin" {
   billing_account_id = data.google_billing_account.master_acct.id
   role               = "roles/billing.admin"
@@ -286,6 +298,7 @@ module "project-services" {
     "oslogin.googleapis.com",
     "parallelstore.googleapis.com",
     "privateca.googleapis.com",
+    "privilegedaccessmanager.googleapis.com",
     "pubsub.googleapis.com",
     "pubsublite.googleapis.com",
     "publicca.googleapis.com",

.ci/magician/cloudbuild/community.go

@@ -41,21 +41,6 @@ func (cb *Client) ApproveCommunityChecker(prNumber, commitSha string) error {
 	return nil
 }
 
-func (cb *Client) GetAwaitingApprovalBuildLink(prNumber, commitSha string) (string, error) {
-	buildId, err := getPendingBuildId(PROJECT_ID, commitSha)
-	if err != nil {
-		return "", err
-	}
-
-	if buildId == "" {
-		return "", fmt.Errorf("failed to find pending build for PR %s", prNumber)
-	}
-
-	targetUrl := fmt.Sprintf("https://console.cloud.google.com/cloud-build/builds;region=global/%s?project=%s", buildId, PROJECT_ID)
-
-	return targetUrl, nil
-}
-
 func getPendingBuildId(projectId, commitSha string) (string, error) {
 	COMMUNITY_CHECKER_TRIGGER, ok := os.LookupEnv("COMMUNITY_CHECKER_TRIGGER")
 	if !ok {

.ci/magician/cmd/DIFF_COMMENT.md

@@ -7,7 +7,7 @@ Your PR hasn't generated any diffs, but I'll let you know if a future commit doe
 Your PR generated some diffs in downstreams - here they are.
 
 {{range .Diffs -}}
-{{.Title}}: [Diff](https://github.com/modular-magician/{{.Repo}}/compare/auto-pr-{{$.PrNumber}}-old..auto-pr-{{$.PrNumber}}) ({{.DiffStats}})
+{{.Title}}: [Diff](https://github.com/modular-magician/{{.Repo}}/compare/auto-pr-{{$.PrNumber}}-old..auto-pr-{{$.PrNumber}}) ({{.ShortStat}})
 {{end -}}
 {{end -}}
 

.ci/magician/cmd/check_cassettes.go

@@ -28,7 +28,6 @@ var ccEnvironmentVariables = [...]string{
 	"GOOGLE_REGION",
 	"GOOGLE_SERVICE_ACCOUNT",
 	"GOOGLE_PUBLIC_AVERTISED_PREFIX_DESCRIPTION",
-	"GOOGLE_TPU_V2_VM_RUNTIME_VERSION",
 	"GOOGLE_ZONE",
 	"PATH",
 	"SA_KEY",