From 68e58cc880cbc0b1dcc2f7c4edd01c6196b2fac4 Mon Sep 17 00:00:00 2001 From: yiyiyimu Date: Tue, 20 Apr 2021 21:06:51 +0800 Subject: [PATCH 1/2] feat: support token Signed-off-by: yiyiyimu --- api-go/cmd/contributor/main.go | 5 +++-- api-go/cmd/setupDB/main.go | 14 -------------- api-go/internal/contributor/contributor.go | 15 +++++++++------ api-go/internal/gcpdb/gcpdb.go | 20 ++++++++++++-------- api-go/internal/ghapi/ghapi.go | 10 ++++++++-- api-go/internal/graph/graph.go | 21 ++++++++++++--------- 6 files changed, 44 insertions(+), 41 deletions(-) delete mode 100644 api-go/cmd/setupDB/main.go diff --git a/api-go/cmd/contributor/main.go b/api-go/cmd/contributor/main.go index d3e25b0..0d208ee 100644 --- a/api-go/cmd/contributor/main.go +++ b/api-go/cmd/contributor/main.go @@ -59,10 +59,11 @@ func main() { func getContributor(w http.ResponseWriter, r *http.Request) { v := r.URL.Query() repo := v.Get("repo") + token := v.Get("token") w.Header().Set("Content-Type", "application/json; charset=utf-8") w.Header().Set("Access-Control-Allow-Origin", "*") - conList, code, err := contributor.GetContributorList(repo) + conList, code, err := contributor.GetContributorList(repo, token) if err != nil { w.WriteHeader(code) @@ -172,7 +173,7 @@ func getActivities(w http.ResponseWriter, r *http.Request) { } func refreshAll(w http.ResponseWriter, r *http.Request) { - _, code, err := gcpdb.UpdateDB("") + _, code, err := gcpdb.UpdateDB("", "") if err != nil { w.WriteHeader(code) diff --git a/api-go/cmd/setupDB/main.go b/api-go/cmd/setupDB/main.go deleted file mode 100644 index a66fa02..0000000 --- a/api-go/cmd/setupDB/main.go +++ /dev/null @@ -1,14 +0,0 @@ -package main - -import ( - "log" - - "github.com/api7/contributor-graph/api/internal/gcpdb" -) - -func main() { - _, _, err := gcpdb.UpdateDB("") - if err != nil { - log.Fatal(err) - } -} diff --git a/api-go/internal/contributor/contributor.go b/api-go/internal/contributor/contributor.go index 4069428..db0368a 100644 --- a/api-go/internal/contributor/contributor.go +++ b/api-go/internal/contributor/contributor.go @@ -17,14 +17,14 @@ import ( "github.com/api7/contributor-graph/api/internal/utils" ) -func GetContributorList(repoName string) ([]utils.ReturnCon, int, error) { +func GetContributorList(repoName string, token string) ([]utils.ReturnCon, int, error) { _, _, err := ghapi.SplitRepo(repoName) if err != nil { return nil, http.StatusNotFound, fmt.Errorf("Repo format error") } fmt.Printf("New request coming with %s\n", repoName) - returnCons, code, err := gcpdb.SingleCon(repoName) + returnCons, code, err := gcpdb.SingleCon(repoName, token) if err != nil { return nil, code, err } @@ -42,12 +42,15 @@ func GetContributorMonthly(repoInput string) ([]utils.MonthlyConList, int, error defer dbCli.Close() var repos []string + var isSearch bool if repoInput == "" { + isSearch = false repos, err = getUpdateRepoList(ctx, dbCli) if err != nil { return nil, http.StatusInternalServerError, err } } else { + isSearch = true repos = []string{strings.ToLower(repoInput)} } @@ -74,13 +77,13 @@ func GetContributorMonthly(repoInput string) ([]utils.MonthlyConList, int, error // get first commit of the repo and use it as the start listCommitOpts := &github.CommitsListOptions{} var firstCommitTime *time.Time - commits, resp, statusCode, err := ghapi.GetCommits(ctx, ghCli, repoName, listCommitOpts) + commits, resp, statusCode, err := ghapi.GetCommits(ctx, ghCli, repoName, listCommitOpts, isSearch) if err != nil { return nil, statusCode, err } if resp.NextPage != 0 { listCommitOpts.Page = resp.LastPage - commits, resp, statusCode, err = ghapi.GetCommits(ctx, ghCli, repoName, listCommitOpts) + commits, resp, statusCode, err = ghapi.GetCommits(ctx, ghCli, repoName, listCommitOpts, isSearch) if err != nil { return nil, statusCode, err } @@ -104,7 +107,7 @@ func GetContributorMonthly(repoInput string) ([]utils.MonthlyConList, int, error for firstCommitTime == nil { listCommitOpts.Page = resp.PrevPage - commits, resp, statusCode, err = ghapi.GetCommits(ctx, ghCli, repoName, listCommitOpts) + commits, resp, statusCode, err = ghapi.GetCommits(ctx, ghCli, repoName, listCommitOpts, isSearch) if err != nil { return nil, statusCode, err } @@ -136,7 +139,7 @@ func GetContributorMonthly(repoInput string) ([]utils.MonthlyConList, int, error comLists := make(map[string]bool) listCommitOpts := &github.CommitsListOptions{Since: firstDay, Until: firstDay.AddDate(0, 1, 0), ListOptions: ghapi.ListOpts} for { - commits, resp, statusCode, err := ghapi.GetCommits(ctx, ghCli, repoName, listCommitOpts) + commits, resp, statusCode, err := ghapi.GetCommits(ctx, ghCli, repoName, listCommitOpts, isSearch) if err != nil { return nil, statusCode, err } diff --git a/api-go/internal/gcpdb/gcpdb.go b/api-go/internal/gcpdb/gcpdb.go index b01ad85..2860796 100644 --- a/api-go/internal/gcpdb/gcpdb.go +++ b/api-go/internal/gcpdb/gcpdb.go @@ -21,7 +21,7 @@ import ( // if repoInput is not empty, fetch single repo and store it in db // else, use repo list to do daily update for all repos -func UpdateDB(repoInput string) ([]*utils.ConList, int, error) { +func UpdateDB(repoInput string, token string) ([]*utils.ConList, int, error) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -47,10 +47,13 @@ func UpdateDB(repoInput string) ([]*utils.ConList, int, error) { conLists = []*utils.ConList{} var ghToken string + var isSearch bool if repoInput == "" { ghToken = utils.UpdateToken[i%len(utils.UpdateToken)] + isSearch = false } else { - ghToken = utils.Token + ghToken = token + isSearch = true } ghCli := ghapi.GetGithubClient(ctx, ghToken) @@ -81,7 +84,7 @@ func UpdateDB(repoInput string) ([]*utils.ConList, int, error) { // get last page lastPage := 0 listCommitOpts := &github.CommitsListOptions{Since: lastModifiedTimeDB, ListOptions: ghapi.ListOpts} - _, resp, statusCode, err := ghapi.GetCommits(ctx, ghCli, repoName, listCommitOpts) + _, resp, statusCode, err := ghapi.GetCommits(ctx, ghCli, repoName, listCommitOpts, isSearch) if err != nil { return nil, statusCode, err } @@ -89,7 +92,7 @@ func UpdateDB(repoInput string) ([]*utils.ConList, int, error) { lastPage = resp.LastPage } - newConLists, code, err := updateContributorList(ctx, dbCli, ghCli, conMap, repoName, lastPage, listCommitOpts) + newConLists, code, err := updateContributorList(ctx, dbCli, ghCli, conMap, repoName, lastPage, listCommitOpts, isSearch) if err != nil { return nil, code, err } @@ -111,8 +114,8 @@ func UpdateDB(repoInput string) ([]*utils.ConList, int, error) { return conLists, http.StatusOK, nil } -func SingleCon(repoInput string) ([]utils.ReturnCon, int, error) { - conLists, code, err := UpdateDB(repoInput) +func SingleCon(repoInput string, token string) ([]utils.ReturnCon, int, error) { + conLists, code, err := UpdateDB(repoInput, token) if err != nil { return nil, code, err } @@ -129,7 +132,7 @@ func MultiCon(repoInput string) ([]utils.ReturnCon, int, error) { conMap := make(map[string]time.Time) for _, r := range repos { - conLists, code, err := UpdateDB(r) + conLists, code, err := UpdateDB(r, "") if err != nil { return nil, code, err } @@ -174,13 +177,14 @@ func updateContributorList( repoName string, lastPage int, listCommitOpts *github.CommitsListOptions, + isSearch bool, ) ([]*utils.ConList, int, error) { bar := progressbar.Default(int64(lastPage + 1)) var commitLists []*utils.ConList for i := lastPage; i >= 0; i-- { listCommitOpts.Page = i - commits, _, statusCode, err := ghapi.GetCommits(ctx, ghCli, repoName, listCommitOpts) + commits, _, statusCode, err := ghapi.GetCommits(ctx, ghCli, repoName, listCommitOpts, isSearch) if err != nil { return nil, statusCode, err } diff --git a/api-go/internal/ghapi/ghapi.go b/api-go/internal/ghapi/ghapi.go index ccb1a12..6486597 100644 --- a/api-go/internal/ghapi/ghapi.go +++ b/api-go/internal/ghapi/ghapi.go @@ -27,7 +27,10 @@ func SplitRepo(repo string) (string, string, error) { } func GetGithubClient(ctx context.Context, token string) *github.Client { - tc := getToken(ctx, token) + var tc *http.Client + if token != "" { + tc = getToken(ctx, token) + } return github.NewClient(tc) } @@ -58,7 +61,7 @@ func FormatCommits(ctx context.Context, comLists []*utils.ConList) ([]utils.Retu } // TODO: support goroutine -func GetCommits(ctx context.Context, ghCli *github.Client, repoName string, listCommitOpts *github.CommitsListOptions) ([]*github.RepositoryCommit, *github.Response, int, error) { +func GetCommits(ctx context.Context, ghCli *github.Client, repoName string, listCommitOpts *github.CommitsListOptions, isSearch bool) ([]*github.RepositoryCommit, *github.Response, int, error) { owner, repo, err := SplitRepo(repoName) if err != nil { return nil, nil, http.StatusBadRequest, err @@ -70,6 +73,9 @@ func GetCommits(ctx context.Context, ghCli *github.Client, repoName string, list return nil, nil, http.StatusNotFound, fmt.Errorf("Repo not found") } if _, ok := err.(*github.RateLimitError); ok || strings.Contains(err.Error(), "403 API rate limit exceeded") { + if isSearch { + return nil, nil, http.StatusForbidden, fmt.Errorf("Hit rate limit! Need Github Token to increase API quota!") + } // give it another random chance to see if magic happens *ghCli = *GetGithubClient(ctx, utils.UpdateToken[rand.Intn(len(utils.UpdateToken))]) commits, resp, err = ghCli.Repositories.ListCommits(ctx, owner, repo, listCommitOpts) diff --git a/api-go/internal/graph/graph.go b/api-go/internal/graph/graph.go index d1a2738..eb79312 100644 --- a/api-go/internal/graph/graph.go +++ b/api-go/internal/graph/graph.go @@ -7,6 +7,7 @@ import ( "io" "io/ioutil" "net/http" + "strings" "cloud.google.com/go/storage" "github.com/api7/contributor-graph/api/internal/utils" @@ -32,16 +33,18 @@ func GenerateAndSaveSVG(ctx context.Context, repo string) error { return err } - wc := client.Bucket(bucket).Object(object).NewWriter(ctx) - wc.ACL = []storage.ACLRule{{Entity: storage.AllUsers, Role: storage.RoleReader}} - wc.CacheControl = "public, max-age=86400" - wc.ContentType = "image/svg+xml;charset=utf-8" + if !strings.Contains(repo, ",") { + wc := client.Bucket(bucket).Object(object).NewWriter(ctx) + wc.ACL = []storage.ACLRule{{Entity: storage.AllUsers, Role: storage.RoleReader}} + wc.CacheControl = "public, max-age=86400" + wc.ContentType = "image/svg+xml;charset=utf-8" - if _, err = io.Copy(wc, bytes.NewReader(svg)); err != nil { - return fmt.Errorf("upload svg failed: io.Copy: %v", err) - } - if err := wc.Close(); err != nil { - return fmt.Errorf("upload svg failed: Writer.Close: %v", err) + if _, err = io.Copy(wc, bytes.NewReader(svg)); err != nil { + return fmt.Errorf("upload svg failed: io.Copy: %v", err) + } + if err := wc.Close(); err != nil { + return fmt.Errorf("upload svg failed: Writer.Close: %v", err) + } } fmt.Printf("New SVG generated with %s\n", repo) return nil From a7c3f7a0dfa2720ba11f4aba1d1e41a46d3f9c44 Mon Sep 17 00:00:00 2001 From: yiyiyimu Date: Wed, 21 Apr 2021 21:54:52 +0800 Subject: [PATCH 2/2] fix: use header to get token, to comply with FE Signed-off-by: yiyiyimu --- api-go/cmd/contributor/main.go | 7 ++++--- api-go/internal/contributor/contributor.go | 11 +++++++++-- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/api-go/cmd/contributor/main.go b/api-go/cmd/contributor/main.go index 0d208ee..3f3ee9d 100644 --- a/api-go/cmd/contributor/main.go +++ b/api-go/cmd/contributor/main.go @@ -57,9 +57,9 @@ func main() { } func getContributor(w http.ResponseWriter, r *http.Request) { + token := r.Header.Get("Authorization") v := r.URL.Query() repo := v.Get("repo") - token := v.Get("token") w.Header().Set("Content-Type", "application/json; charset=utf-8") w.Header().Set("Access-Control-Allow-Origin", "*") @@ -75,12 +75,13 @@ func getContributor(w http.ResponseWriter, r *http.Request) { } func getMonthlyContributor(w http.ResponseWriter, r *http.Request) { + token := r.Header.Get("Authorization") v := r.URL.Query() repo := v.Get("repo") w.Header().Set("Content-Type", "application/json; charset=utf-8") w.Header().Set("Access-Control-Allow-Origin", "*") - monthlyConLists, code, err := contributor.GetContributorMonthly(repo) + monthlyConLists, code, err := contributor.GetContributorMonthly(repo, token) if err != nil { w.WriteHeader(code) @@ -183,7 +184,7 @@ func refreshAll(w http.ResponseWriter, r *http.Request) { } func refreshMonthly(w http.ResponseWriter, r *http.Request) { - _, code, err := contributor.GetContributorMonthly("") + _, code, err := contributor.GetContributorMonthly("", "") if err != nil { w.WriteHeader(code) diff --git a/api-go/internal/contributor/contributor.go b/api-go/internal/contributor/contributor.go index 839c607..104efea 100644 --- a/api-go/internal/contributor/contributor.go +++ b/api-go/internal/contributor/contributor.go @@ -34,7 +34,7 @@ func GetContributorList(repoName string, token string) ([]utils.ReturnCon, int, return returnCons, http.StatusOK, nil } -func GetContributorMonthly(repoInput string) ([]utils.MonthlyConList, int, error) { +func GetContributorMonthly(repoInput string, token string) ([]utils.MonthlyConList, int, error) { ctx := context.Background() dbCli, err := datastore.NewClient(ctx, utils.ProjectID) @@ -71,7 +71,14 @@ func GetContributorMonthly(repoInput string) ([]utils.MonthlyConList, int, error }) if len(monthlyConLists) == 0 || monthlyConLists[len(monthlyConLists)-1].Month.AddDate(0, 2, 0).Before(time.Now()) { - ghCli := ghapi.GetGithubClient(ctx, utils.UpdateToken[i%len(utils.UpdateToken)]) + var ghToken string + if repoInput == "" { + ghToken = utils.UpdateToken[i%len(utils.UpdateToken)] + } else { + ghToken = token + } + + ghCli := ghapi.GetGithubClient(ctx, ghToken) var firstDay time.Time if len(monthlyConLists) > 0 {