diff --git a/.ci/gcb-community-checker.yml b/.ci/gcb-community-checker.yml deleted file mode 100644 index ba689d1307ff..000000000000 --- a/.ci/gcb-community-checker.yml +++ /dev/null @@ -1,81 +0,0 @@ ---- -steps: - # The GCB / GH integration uses a shallow clone of the repo. We need to convert - # that to a full clone in order to work with it properly. - # https://cloud.google.com/source-repositories/docs/integrating-with-cloud-build#unshallowing_clones - - name: "gcr.io/cloud-builders/git" - args: - - fetch - - --unshallow - - # Configure git - - name: "gcr.io/cloud-builders/git" - args: - - config - - --global - - user.email - - magic-modules+differ@google.com - - name: "gcr.io/cloud-builders/git" - args: - - config - - --global - - user.name - - "Modular Magician Diff Process" - - # Display commit log for clarity - - name: "gcr.io/cloud-builders/git" - args: - - log - - "--oneline" - - "-n 10" - - # Find common ancestor commit and apply diff for the .ci folder. - - name: "gcr.io/cloud-builders/git" - id: findMergeBase - entrypoint: "bash" - args: - - "-c" - - | - git fetch origin main - if [ "$_BASE_BRANCH" != "main" ]; then - echo "Checking out .ci/ folder from main" - git checkout origin/main -- .ci/ - else - base_commit=$(git merge-base origin/main HEAD) - echo "Common ancestor commit: $base_commit" - git diff $base_commit origin/main -- .ci/ - git diff $base_commit origin/main -- .ci/ > /workspace/ci.diff - git apply ./ci.diff --allow-empty - fi - if [ "$_BASE_BRANCH" != "main" ]; then - echo "Checking out tools/ folder from main" - git checkout origin/main -- tools/ - else - base_commit=$(git merge-base origin/main HEAD) - echo "Common ancestor commit: $base_commit" - git diff $base_commit origin/main -- tools/ - git diff $base_commit origin/main -- tools/ > /workspace/tools.diff - git apply ./tools.diff --allow-empty - fi - - - name: 'gcr.io/graphite-docker-images/go-plus' - entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh' - id: community-checker - secretEnv: ["GITHUB_TOKEN_MAGIC_MODULES", "GENERATE_DIFFS_TRIGGER"] - timeout: 8000s - args: - - "community-checker" - - $_PR_NUMBER - - $COMMIT_SHA - - $BRANCH_NAME - - $_HEAD_REPO_URL - - $_HEAD_BRANCH - - $_BASE_BRANCH - -logsBucket: 'gs://cloudbuild-community-checker-logs' -availableSecrets: - secretManager: - - versionName: projects/673497134629/secrets/github-magician-token-generate-diffs-magic-modules/versions/latest - env: GITHUB_TOKEN_MAGIC_MODULES - - versionName: projects/673497134629/secrets/ci-trigger-generate-diffs/versions/latest - env: GENERATE_DIFFS_TRIGGER diff --git a/.ci/gcb-contributor-membership-checker.yml b/.ci/gcb-contributor-membership-checker.yml index ce7260352953..dbb9fb1c902b 100644 --- a/.ci/gcb-contributor-membership-checker.yml +++ b/.ci/gcb-contributor-membership-checker.yml @@ -62,7 +62,7 @@ steps: entrypoint: "/workspace/.ci/scripts/go-plus/magician/exec.sh" id: contributor-membership-checker secretEnv: - ["GITHUB_TOKEN_MAGIC_MODULES", "GENERATE_DIFFS_TRIGGER", "COMMUNITY_CHECKER_TRIGGER", "DOWNSTREAM_GENERATION_AND_TEST_TRIGGER"] + ["GITHUB_TOKEN_MAGIC_MODULES", "DOWNSTREAM_GENERATION_AND_TEST_TRIGGER"] timeout: 8000s args: - "membership-checker" @@ -74,9 +74,5 @@ availableSecrets: secretManager: - versionName: projects/673497134629/secrets/github-magician-token-generate-diffs-magic-modules/versions/latest env: GITHUB_TOKEN_MAGIC_MODULES - - versionName: projects/673497134629/secrets/ci-trigger-generate-diffs/versions/latest - env: GENERATE_DIFFS_TRIGGER - - versionName: projects/673497134629/secrets/ci-trigger-community-checker/versions/latest - env: COMMUNITY_CHECKER_TRIGGER - versionName: projects/673497134629/secrets/ci-trigger-downstream-generation-and-test/versions/latest env: DOWNSTREAM_GENERATION_AND_TEST_TRIGGER diff --git a/.ci/magician/cloudbuild/build_trigger.go b/.ci/magician/cloudbuild/build_trigger.go deleted file mode 100644 index f776af1222cd..000000000000 --- a/.ci/magician/cloudbuild/build_trigger.go +++ /dev/null @@ -1,61 +0,0 @@ -/* -* Copyright 2023 Google LLC. All Rights Reserved. -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. - */ -package cloudbuild - -import ( - "context" - "fmt" - "os" - - cloudbuildv1 "google.golang.org/api/cloudbuild/v1" -) - -func (cb *Client) TriggerMMPresubmitRuns(commitSha string, substitutions map[string]string) error { - presubmitTriggerId, ok := os.LookupEnv("GENERATE_DIFFS_TRIGGER") - if !ok { - return fmt.Errorf("did not provide GENERATE_DIFFS_TRIGGER environment variable") - } - - err := triggerCloudBuildRun(PROJECT_ID, presubmitTriggerId, REPO_NAME, commitSha, substitutions) - if err != nil { - return err - } - - return nil -} - -func triggerCloudBuildRun(projectId, triggerId, repoName, commitSha string, substitutions map[string]string) error { - ctx := context.Background() - c, err := cloudbuildv1.NewService(ctx) - if err != nil { - return fmt.Errorf("failed to create Cloud Build service client: %s", err) - } - - repoSource := &cloudbuildv1.RepoSource{ - ProjectId: projectId, - RepoName: repoName, - CommitSha: commitSha, - Substitutions: substitutions, - } - - _, err = c.Projects.Triggers.Run(projectId, triggerId, repoSource).Do() - if err != nil { - return fmt.Errorf("failed to create Cloud Build run: %s", err) - } - - fmt.Println("Started Cloud Build Run: ", triggerId) - return nil -} diff --git a/.ci/magician/cloudbuild/community.go b/.ci/magician/cloudbuild/community.go index 5b352975c0e9..9c01d7c94f01 100644 --- a/.ci/magician/cloudbuild/community.go +++ b/.ci/magician/cloudbuild/community.go @@ -23,7 +23,7 @@ import ( cloudbuildv1 "google.golang.org/api/cloudbuild/v1" ) -func (cb *Client) ApproveCommunityChecker(prNumber, commitSha string) error { +func (cb *Client) ApproveDownstreamGenAndTest(prNumber, commitSha string) error { buildId, err := getPendingBuildId(PROJECT_ID, commitSha) if err != nil { return err @@ -42,9 +42,9 @@ func (cb *Client) ApproveCommunityChecker(prNumber, commitSha string) error { } func getPendingBuildId(projectId, commitSha string) (string, error) { - COMMUNITY_CHECKER_TRIGGER, ok := os.LookupEnv("COMMUNITY_CHECKER_TRIGGER") + COMMUNITY_CHECKER_TRIGGER, ok := os.LookupEnv("DOWNSTREAM_GENERATION_AND_TEST_TRIGGER") if !ok { - return "", fmt.Errorf("Did not provide COMMUNITY_CHECKER_TRIGGER environment variable") + return "", fmt.Errorf("Did not provide DOWNSTREAM_GENERATION_AND_TEST_TRIGGER environment variable") } ctx := context.Background() diff --git a/.ci/magician/cmd/community_checker.go b/.ci/magician/cmd/community_checker.go deleted file mode 100644 index f86aabdfba58..000000000000 --- a/.ci/magician/cmd/community_checker.go +++ /dev/null @@ -1,97 +0,0 @@ -/* -* Copyright 2023 Google LLC. All Rights Reserved. -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. - */ -package cmd - -import ( - "fmt" - "magician/cloudbuild" - "magician/github" - - "github.com/spf13/cobra" -) - -// communityApprovalCmd represents the communityApproval command -var communityApprovalCmd = &cobra.Command{ - Use: "community-checker", - Short: "Run presubmit generate diffs for untrusted users and remove awaiting-approval label", - Long: `This command processes pull requests and performs various validations and actions based on the PR's metadata and author. - - The following PR details are expected as arguments: - 1. PR Number - 2. Commit SHA - 3. Branch Name - 4. Head Repo URL - 5. Head Branch - 6. Base Branch - - The command performs the following steps: - 1. Trigger cloud presubmits with specific substitutions for the PR. - 2. Remove the 'awaiting-approval' label from the PR. - `, - RunE: func(cmd *cobra.Command, args []string) error { - prNumber := args[0] - fmt.Println("PR Number: ", prNumber) - - commitSha := args[1] - fmt.Println("Commit SHA: ", commitSha) - - branchName := args[2] - fmt.Println("Branch Name: ", branchName) - - headRepoUrl := args[3] - fmt.Println("Head Repo URL: ", headRepoUrl) - - headBranch := args[4] - fmt.Println("Head Branch: ", headBranch) - - baseBranch := args[5] - fmt.Println("Base Branch: ", baseBranch) - - githubToken, ok := lookupGithubTokenOrFallback("GITHUB_TOKEN_MAGIC_MODULES") - if !ok { - return fmt.Errorf("did not provide GITHUB_TOKEN_MAGIC_MODULES or GITHUB_TOKEN environment variables") - } - gh := github.NewClient(githubToken) - cb := cloudbuild.NewClient() - return execCommunityChecker(prNumber, commitSha, branchName, headRepoUrl, headBranch, baseBranch, gh, cb) - }, -} - -func execCommunityChecker(prNumber, commitSha, branchName, headRepoUrl, headBranch, baseBranch string, gh GithubClient, cb CloudbuildClient) error { - substitutions := map[string]string{ - "BRANCH_NAME": branchName, - "_PR_NUMBER": prNumber, - "_HEAD_REPO_URL": headRepoUrl, - "_HEAD_BRANCH": headBranch, - "_BASE_BRANCH": baseBranch, - } - - // trigger presubmit builds - community-checker requires approval - // (explicitly or via membership-checker) - err := cb.TriggerMMPresubmitRuns(commitSha, substitutions) - if err != nil { - return err - } - - // in community-checker job: - // remove awaiting-approval label from external contributor PRs - gh.RemoveLabel(prNumber, "awaiting-approval") - return nil -} - -func init() { - rootCmd.AddCommand(communityApprovalCmd) -} diff --git a/.ci/magician/cmd/community_checker_test.go b/.ci/magician/cmd/community_checker_test.go deleted file mode 100644 index 9880d83031a3..000000000000 --- a/.ci/magician/cmd/community_checker_test.go +++ /dev/null @@ -1,126 +0,0 @@ -/* -* Copyright 2023 Google LLC. All Rights Reserved. -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. - */ -package cmd - -import ( - "magician/github" - "reflect" - "testing" -) - -func TestExecCommunityChecker_CoreContributorFlow(t *testing.T) { - gh := &mockGithub{ - pullRequest: github.PullRequest{ - User: github.User{ - Login: "core_author", - }, - }, - userType: github.CoreContributorUserType, - calledMethods: make(map[string][][]any), - } - cb := &mockCloudBuild{ - calledMethods: make(map[string][][]any), - } - - execCommunityChecker("pr1", "sha1", "branch1", "url1", "head1", "base1", gh, cb) - - method := "TriggerMMPresubmitRuns" - expected := [][]any{{"sha1", map[string]string{"BRANCH_NAME": "branch1", "_BASE_BRANCH": "base1", "_HEAD_BRANCH": "head1", "_HEAD_REPO_URL": "url1", "_PR_NUMBER": "pr1"}}} - if calls, ok := cb.calledMethods[method]; !ok { - t.Fatal("Presubmit runs not triggered for core contributor") - } else if !reflect.DeepEqual(calls, expected) { - t.Fatalf("Wrong calls for %s, got %v, expected %v", method, calls, expected) - } - - method = "RemoveLabel" - expected = [][]any{{"pr1", "awaiting-approval"}} - if calls, ok := gh.calledMethods[method]; !ok { - t.Fatal("awaiting-approval label not removed for PR ") - } else if !reflect.DeepEqual(calls, expected) { - t.Fatalf("Wrong calls for %s, got %v, expected %v", method, calls, expected) - } - -} - -func TestExecCommunityChecker_GooglerFlow(t *testing.T) { - gh := &mockGithub{ - pullRequest: github.PullRequest{ - User: github.User{ - Login: "googler_author", - }, - }, - userType: github.GooglerUserType, - calledMethods: make(map[string][][]any), - requestedReviewers: []github.User{github.User{Login: "reviewer1"}}, - previousReviewers: []github.User{github.User{Login: github.GetRandomReviewer()}, github.User{Login: "reviewer3"}}, - } - cb := &mockCloudBuild{ - calledMethods: make(map[string][][]any), - } - - execCommunityChecker("pr1", "sha1", "branch1", "url1", "head1", "base1", gh, cb) - - method := "TriggerMMPresubmitRuns" - expected := [][]any{{"sha1", map[string]string{"BRANCH_NAME": "branch1", "_BASE_BRANCH": "base1", "_HEAD_BRANCH": "head1", "_HEAD_REPO_URL": "url1", "_PR_NUMBER": "pr1"}}} - if calls, ok := cb.calledMethods[method]; !ok { - t.Fatal("Presubmit runs not triggered for googler") - } else if !reflect.DeepEqual(calls, expected) { - t.Fatalf("Wrong calls for %s, got %v, expected %v", method, calls, expected) - } - - method = "RemoveLabel" - expected = [][]any{{"pr1", "awaiting-approval"}} - if calls, ok := gh.calledMethods[method]; !ok { - t.Fatal("awaiting-approval label not removed for PR ") - } else if !reflect.DeepEqual(calls, expected) { - t.Fatalf("Wrong calls for %s, got %v, expected %v", method, calls, expected) - } -} - -func TestExecCommunityChecker_AmbiguousUserFlow(t *testing.T) { - gh := &mockGithub{ - pullRequest: github.PullRequest{ - User: github.User{ - Login: "ambiguous_author", - }, - }, - userType: github.CommunityUserType, - calledMethods: make(map[string][][]any), - requestedReviewers: []github.User{github.User{Login: github.GetRandomReviewer()}}, - previousReviewers: []github.User{github.User{Login: github.GetRandomReviewer()}, github.User{Login: "reviewer3"}}, - } - cb := &mockCloudBuild{ - calledMethods: make(map[string][][]any), - } - - execCommunityChecker("pr1", "sha1", "branch1", "url1", "head1", "base1", gh, cb) - - method := "TriggerMMPresubmitRuns" - expected := [][]any{{"sha1", map[string]string{"BRANCH_NAME": "branch1", "_BASE_BRANCH": "base1", "_HEAD_BRANCH": "head1", "_HEAD_REPO_URL": "url1", "_PR_NUMBER": "pr1"}}} - if calls, ok := cb.calledMethods[method]; !ok { - t.Fatal("Presubmit runs not triggered for ambiguous user") - } else if !reflect.DeepEqual(calls, expected) { - t.Fatalf("Wrong calls for %s, got %v, expected %v", method, calls, expected) - } - - method = "RemoveLabel" - expected = [][]any{{"pr1", "awaiting-approval"}} - if calls, ok := gh.calledMethods[method]; !ok { - t.Fatal("awaiting-approval label not removed for PR ") - } else if !reflect.DeepEqual(calls, expected) { - t.Fatalf("Wrong calls for %s, got %v, expected %v", method, calls, expected) - } -} diff --git a/.ci/magician/cmd/interfaces.go b/.ci/magician/cmd/interfaces.go index 948ab794c907..b7007b89e934 100644 --- a/.ci/magician/cmd/interfaces.go +++ b/.ci/magician/cmd/interfaces.go @@ -36,8 +36,7 @@ type GithubClient interface { } type CloudbuildClient interface { - ApproveCommunityChecker(prNumber, commitSha string) error - TriggerMMPresubmitRuns(commitSha string, substitutions map[string]string) error + ApproveDownstreamGenAndTest(prNumber, commitSha string) error } type ExecRunner interface { diff --git a/.ci/magician/cmd/membership_checker.go b/.ci/magician/cmd/membership_checker.go index 7e149a122dad..0faa0fef8b1a 100644 --- a/.ci/magician/cmd/membership_checker.go +++ b/.ci/magician/cmd/membership_checker.go @@ -74,7 +74,7 @@ func execMembershipChecker(prNumber, commitSha string, gh GithubClient, cb Cloud // 1. auto approve community-checker run for trusted users // 2. add awaiting-approval label to external contributor PRs if trusted { - err = cb.ApproveCommunityChecker(prNumber, commitSha) + err = cb.ApproveDownstreamGenAndTest(prNumber, commitSha) if err != nil { return err } diff --git a/.ci/magician/cmd/membership_checker_test.go b/.ci/magician/cmd/membership_checker_test.go index d13ae2c3df0c..c32c05cff319 100644 --- a/.ci/magician/cmd/membership_checker_test.go +++ b/.ci/magician/cmd/membership_checker_test.go @@ -37,7 +37,7 @@ func TestExecMembershipChecker_CoreContributorFlow(t *testing.T) { execMembershipChecker("pr1", "sha1", gh, cb) - method := "ApproveCommunityChecker" + method := "ApproveDownstreamGenAndTest" expected := [][]any{{"pr1", "sha1"}} if calls, ok := cb.calledMethods[method]; !ok { t.Fatal("Community checker not approved for core author") @@ -65,7 +65,7 @@ func TestExecMembershipChecker_GooglerFlow(t *testing.T) { execMembershipChecker("pr1", "sha1", gh, cb) - method := "ApproveCommunityChecker" + method := "ApproveDownstreamGenAndTest" expected := [][]any{{"pr1", "sha1"}} if calls, ok := cb.calledMethods[method]; !ok { t.Fatal("Community checker not approved for googler") @@ -100,7 +100,7 @@ func TestExecMembershipChecker_AmbiguousUserFlow(t *testing.T) { t.Fatalf("Wrong calls for %s, got %v, expected %v", method, calls, expected) } - if _, ok := gh.calledMethods["ApproveCommunityChecker"]; ok { + if _, ok := gh.calledMethods["ApproveDownstreamGenAndTest"]; ok { t.Fatal("Incorrectly approved community checker for ambiguous user") } } diff --git a/.ci/magician/cmd/mock_cloudbuild_test.go b/.ci/magician/cmd/mock_cloudbuild_test.go index d95ae3f3ffea..37a5ab399e95 100644 --- a/.ci/magician/cmd/mock_cloudbuild_test.go +++ b/.ci/magician/cmd/mock_cloudbuild_test.go @@ -19,12 +19,7 @@ type mockCloudBuild struct { calledMethods map[string][][]any } -func (m *mockCloudBuild) ApproveCommunityChecker(prNumber, commitSha string) error { - m.calledMethods["ApproveCommunityChecker"] = append(m.calledMethods["ApproveCommunityChecker"], []any{prNumber, commitSha}) - return nil -} - -func (m *mockCloudBuild) TriggerMMPresubmitRuns(commitSha string, substitutions map[string]string) error { - m.calledMethods["TriggerMMPresubmitRuns"] = append(m.calledMethods["TriggerMMPresubmitRuns"], []any{commitSha, substitutions}) +func (m *mockCloudBuild) ApproveDownstreamGenAndTest(prNumber, commitSha string) error { + m.calledMethods["ApproveDownstreamGenAndTest"] = append(m.calledMethods["ApproveDownstreamGenAndTest"], []any{prNumber, commitSha}) return nil }