diff --git a/api/ghactions/notify.go b/api/ghactions/notify.go new file mode 100644 index 00000000000..c7963900e7e --- /dev/null +++ b/api/ghactions/notify.go @@ -0,0 +1,200 @@ +// Copyright 2019 The WPT Dashboard Project. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package ghactions + +import ( + "context" + "fmt" + "net/http" + "regexp" + "strconv" + + mapset "github.com/deckarep/golang-set" + "github.com/gobwas/glob" + "github.com/google/go-github/v47/github" + "github.com/gorilla/mux" + uc "github.com/web-platform-tests/wpt.fyi/api/receiver/client" + + "github.com/web-platform-tests/wpt.fyi/shared" +) + +const uploaderName = "github-actions" + +var ( + prHeadRegex = regexp.MustCompile(`\baffected-tests$`) + prBaseRegex = regexp.MustCompile(`\baffected-tests-without-changes$`) + epochBranchesRegex = regexp.MustCompile("^epochs/.*") +) + +func notifyHandler(w http.ResponseWriter, r *http.Request) { + vars := mux.Vars(r) + rawRunID := vars["run_id"] + var runID int64 + var err error + if runID, err = strconv.ParseInt(rawRunID, 0, 0); err != nil { + http.Error(w, fmt.Sprintf("Invalid run id: %s", rawRunID), http.StatusBadRequest) + + return + } + + owner := vars["owner"] + repo := vars["repo"] + + if owner != shared.WPTRepoOwner || repo != shared.WPTRepoName { + http.Error(w, fmt.Sprintf("Invalid repo: %s/%s", owner, repo), http.StatusBadRequest) + + return + } + + artifactName := vars["artifact_name"] + artifactNameGlob, err := glob.Compile(artifactName) + if err != nil { + http.Error(w, fmt.Sprintf("Invalid artifact name: %s", artifactName), http.StatusBadRequest) + + return + } + + ctx := r.Context() + aeAPI := shared.NewAppEngineAPI(ctx) + log := shared.GetLogger(ctx) + + ghClient, err := aeAPI.GetGitHubClient() + if err != nil { + log.Errorf("Failed to get GitHub client: %s", err.Error()) + http.Error(w, err.Error(), http.StatusInternalServerError) + + return + } + + processed, err := processBuild( + ctx, + aeAPI, + ghClient, + owner, + repo, + runID, + artifactNameGlob, + ) + + if err != nil { + log.Errorf("%v", err.Error()) + http.Error(w, err.Error(), http.StatusInternalServerError) + + return + } + + if processed { + w.WriteHeader(http.StatusOK) + fmt.Fprintln(w, "GitHub Actions workflow run artifacts retrieved successfully") + } else { + w.WriteHeader(http.StatusNoContent) + fmt.Fprintln(w, "Notification of workflow run artifacts was ignored") + } +} + +func processBuild( + ctx context.Context, + aeAPI shared.AppEngineAPI, + ghClient *github.Client, + owner string, + repo string, + runID int64, + artifactNameGlob glob.Glob, +) (bool, error) { + log := shared.GetLogger(ctx) + + workflowRun, _, err := ghClient.Actions.GetWorkflowRunByID(ctx, owner, repo, runID) + if err != nil { + return false, err + } + + headSha := workflowRun.HeadSHA + + opts := &github.ListOptions{PerPage: 100} + + uploadedAny := false + errors := make(chan (error)) + + for { + artifacts, resp, err := ghClient.Actions.ListWorkflowRunArtifacts(ctx, owner, repo, runID, opts) + + if err != nil { + return false, err + } + + for _, artifact := range artifacts.Artifacts { + + if !artifactNameGlob.Match(*artifact.Name) { + log.Infof("Skipping artifact %s", artifact.Name) + + continue + } + + log.Infof("Uploading %s for %s/%s run %v...", artifact.Name, owner, repo, runID) + + labels := chooseLabels(workflowRun, *artifact.Name, owner, repo) + + uploader, err := aeAPI.GetUploader(uploaderName) + if err != nil { + return false, fmt.Errorf("failed to get uploader creds from Datastore: %w", err) + } + + uploadClient := uc.NewClient(aeAPI) + err = uploadClient.CreateRun( + *headSha, + uploader.Username, + uploader.Password, + // Uploading a ZIP with both results and screenshots is special-cased by the receiver. + []string{*artifact.ArchiveDownloadURL}, + nil, + shared.ToStringSlice(labels)) + if err != nil { + errors <- fmt.Errorf("failed to create run: %w", err) + } else { + uploadedAny = true + } + + } + + if resp.NextPage == 0 { + break + } + opts.Page = resp.NextPage + } + + close(errors) + for err := range errors { + return uploadedAny, err + } + + return uploadedAny, nil +} + +func chooseLabels( + workflowRun *github.WorkflowRun, + artifactName string, + owner string, + repo string, +) mapset.Set { + labels := mapset.NewSet() + + if (*workflowRun.Event == "push" && + *workflowRun.HeadRepository.Owner.Login == owner && + *workflowRun.HeadRepository.Name == repo) && + (*workflowRun.HeadBranch == "master" || + epochBranchesRegex.MatchString(*workflowRun.HeadBranch)) { + labels.Add(shared.MasterLabel) + } + + if *workflowRun.Event == "pull_request" { + if prHeadRegex.MatchString(artifactName) { + labels.Add(shared.PRHeadLabel) + } else if prBaseRegex.MatchString(artifactName) { + labels.Add(shared.PRBaseLabel) + } + } + + return labels +} diff --git a/api/ghactions/notify_test.go b/api/ghactions/notify_test.go new file mode 100644 index 00000000000..3e995ab041c --- /dev/null +++ b/api/ghactions/notify_test.go @@ -0,0 +1,132 @@ +//go:build small +// +build small + +// Copyright 2018 The WPT Dashboard Project. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package ghactions + +import ( + "testing" + + "github.com/google/go-github/v47/github" + "github.com/stretchr/testify/assert" +) + +func PointerTo[T any](v T) *T { + return &v +} + +func TestArtifactRegexes(t *testing.T) { + assert.True(t, prHeadRegex.MatchString("safari-preview-1-affected-tests")) + assert.True(t, prBaseRegex.MatchString("safari-preview-1-affected-tests-without-changes")) + + // Base and Head could be confused with substring matching + assert.False(t, prBaseRegex.MatchString("safari-preview-1-affected-tests")) + assert.False(t, prHeadRegex.MatchString("safari-preview-1-affected-tests-without-changes")) +} + +func TestEpochBranchesRegex(t *testing.T) { + assert.True(t, epochBranchesRegex.MatchString("epochs/twelve_hourly")) + assert.True(t, epochBranchesRegex.MatchString("epochs/six_hourly")) + assert.True(t, epochBranchesRegex.MatchString("epochs/weekly")) + assert.True(t, epochBranchesRegex.MatchString("epochs/daily")) + + assert.False(t, epochBranchesRegex.MatchString("weekly")) + assert.False(t, epochBranchesRegex.MatchString("a/epochs/weekly")) +} + +func TestChooseLabels(t *testing.T) { + wptOrgUser := github.User{ + Login: PointerTo("web-platform-tests"), + } + + otherUser := github.User{ + Login: PointerTo("xxx"), + } + + wptRepo := github.Repository{ + Name: PointerTo("wpt"), + FullName: PointerTo("web-platform-tests/wpt"), + Owner: &wptOrgUser, + } + + otherRepo := github.Repository{ + Name: PointerTo("wpt"), + FullName: PointerTo("xxx/wpt"), + Owner: &otherUser, + } + + masterWorkflowRun := github.WorkflowRun{ + HeadBranch: PointerTo("master"), + Event: PointerTo("push"), + Status: PointerTo("completed"), + Conclusion: PointerTo("success"), + HeadSHA: PointerTo("74dc6f6f5b2ba16940e6b6075f0faf311361dbb2"), + Repository: &wptRepo, + HeadRepository: &wptRepo, + } + + masterOtherWorkflowRun := github.WorkflowRun{ + HeadBranch: PointerTo("master"), + Event: PointerTo("push"), + Status: PointerTo("completed"), + Conclusion: PointerTo("success"), + HeadSHA: PointerTo("74dc6f6f5b2ba16940e6b6075f0faf311361dbb2"), + Repository: &otherRepo, + HeadRepository: &otherRepo, + } + + prWorkflowRun := github.WorkflowRun{ + HeadBranch: PointerTo("new-branch"), + Event: PointerTo("pull_request"), + Status: PointerTo("completed"), + Conclusion: PointerTo("success"), + HeadSHA: PointerTo("74dc6f6f5b2ba16940e6b6075f0faf311361dbb2"), + Repository: &wptRepo, + HeadRepository: &wptRepo, + } + + prOtherWorkflowRun := github.WorkflowRun{ + HeadBranch: PointerTo("master"), + Event: PointerTo("pull_request"), + Status: PointerTo("completed"), + Conclusion: PointerTo("success"), + HeadSHA: PointerTo("74dc6f6f5b2ba16940e6b6075f0faf311361dbb2"), + Repository: &wptRepo, + HeadRepository: &otherRepo, + } + + assert.ElementsMatch( + t, + chooseLabels(&masterWorkflowRun, "results-safari-1", "web-platform-tests", "wpt").ToSlice(), + []string{"master"}, + ) + + assert.ElementsMatch(t, + chooseLabels(&masterOtherWorkflowRun, "results-safari-1", "web-platform-tests", "wpt").ToSlice(), + []string{}, + ) + + assert.ElementsMatch(t, + chooseLabels(&prWorkflowRun, "results-safari-1", "web-platform-tests", "wpt").ToSlice(), + []string{}, + ) + + assert.ElementsMatch(t, + chooseLabels(&prOtherWorkflowRun, "results-safari-1", "web-platform-tests", "wpt").ToSlice(), + []string{}, + ) + + assert.ElementsMatch( + t, + chooseLabels(&prWorkflowRun, "results-safari-1-affected-tests", "web-platform-tests", "wpt").ToSlice(), + []string{"pr_head"}, + ) + + assert.ElementsMatch(t, + chooseLabels(&prOtherWorkflowRun, "results-safari-1-affected-tests-without-changes", "web-platform-tests", "wpt").ToSlice(), + []string{"pr_base"}, + ) +} diff --git a/api/ghactions/routes.go b/api/ghactions/routes.go new file mode 100644 index 00000000000..415c340ad36 --- /dev/null +++ b/api/ghactions/routes.go @@ -0,0 +1,16 @@ +// Copyright 2019 The WPT Dashboard Project. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +package ghactions + +import "github.com/web-platform-tests/wpt.fyi/shared" + +// RegisterRoutes adds all the api route handlers. +func RegisterRoutes() { + // notifyHandler exposes an endpoint for notifying wpt.fyi that it can collect + // the results of an GitHub Actions workflow run. + // The endpoint is insecure, because we'll only try to fetch (specifically) a + // web-platform-tests/wpt build with the given ID. + shared.AddRoute("/api/checks/github-actions/", "github-actions-notify", notifyHandler).Methods("POST") +} diff --git a/go.mod b/go.mod index ffb0ea44164..e84f87b2d46 100644 --- a/go.mod +++ b/go.mod @@ -53,6 +53,7 @@ require ( github.com/go-logr/stdr v1.2.2 // indirect github.com/gobuffalo/logger v1.0.7 // indirect github.com/gobuffalo/packd v1.0.2 // indirect + github.com/gobwas/glob v0.2.3 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/google/go-querystring v1.1.0 // indirect github.com/google/s2a-go v0.1.8 // indirect diff --git a/go.sum b/go.sum index 6d5033385c7..7bbe3f60725 100644 --- a/go.sum +++ b/go.sum @@ -130,6 +130,8 @@ github.com/gobuffalo/packd v1.0.2 h1:Yg523YqnOxGIWCp69W12yYBKsoChwI7mtu6ceM9Bwfw github.com/gobuffalo/packd v1.0.2/go.mod h1:sUc61tDqGMXON80zpKGp92lDb86Km28jfvX7IAyxFT8= github.com/gobuffalo/packr/v2 v2.8.3 h1:xE1yzvnO56cUC0sTpKR3DIbxZgB54AftTFMhB2XEWlY= github.com/gobuffalo/packr/v2 v2.8.3/go.mod h1:0SahksCVcx4IMnigTjiFuyldmTrdTctXsOdiU5KwbKc= +github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= +github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=