Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: support token #31

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
Open

feat: support token #31

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
68e58cc
feat: support token
Yiyiyimu Apr 20, 2021
b166821
chore: merge master
Yiyiyimu Apr 21, 2021
a7c3f7a
fix: use header to get token, to comply with FE
Yiyiyimu Apr 21, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 6 additions & 4 deletions api-go/cmd/contributor/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,12 +57,13 @@ func main() {
}

func getContributor(w http.ResponseWriter, r *http.Request) {
token := r.Header.Get("Authorization")
v := r.URL.Query()
repo := v.Get("repo")
w.Header().Set("Content-Type", "application/json; charset=utf-8")
w.Header().Set("Access-Control-Allow-Origin", "*")

conList, code, err := contributor.GetContributorList(repo)
conList, code, err := contributor.GetContributorList(repo, token)

if err != nil {
w.WriteHeader(code)
Expand All @@ -74,12 +75,13 @@ func getContributor(w http.ResponseWriter, r *http.Request) {
}

func getMonthlyContributor(w http.ResponseWriter, r *http.Request) {
token := r.Header.Get("Authorization")
v := r.URL.Query()
repo := v.Get("repo")
w.Header().Set("Content-Type", "application/json; charset=utf-8")
w.Header().Set("Access-Control-Allow-Origin", "*")

monthlyConLists, code, err := contributor.GetContributorMonthly(repo)
monthlyConLists, code, err := contributor.GetContributorMonthly(repo, token)

if err != nil {
w.WriteHeader(code)
Expand Down Expand Up @@ -172,7 +174,7 @@ func getActivities(w http.ResponseWriter, r *http.Request) {
}

func refreshAll(w http.ResponseWriter, r *http.Request) {
_, code, err := gcpdb.UpdateDB("")
_, code, err := gcpdb.UpdateDB("", "")

if err != nil {
w.WriteHeader(code)
Expand All @@ -182,7 +184,7 @@ func refreshAll(w http.ResponseWriter, r *http.Request) {
}

func refreshMonthly(w http.ResponseWriter, r *http.Request) {
_, code, err := contributor.GetContributorMonthly("")
_, code, err := contributor.GetContributorMonthly("", "")

if err != nil {
w.WriteHeader(code)
Expand Down
14 changes: 0 additions & 14 deletions api-go/cmd/setupDB/main.go
View file
Open in desktop

This file was deleted.

19 changes: 13 additions & 6 deletions api-go/internal/contributor/contributor.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,22 +19,22 @@ import (
"github.com/api7/contributor-graph/api/internal/utils"
)

func GetContributorList(repoName string) ([]utils.ReturnCon, int, error) {
func GetContributorList(repoName string, token string) ([]utils.ReturnCon, int, error) {
_, _, err := ghapi.SplitRepo(repoName)
if err != nil {
return nil, http.StatusNotFound, fmt.Errorf("Repo format error")
}

fmt.Printf("New request coming with %s\n", repoName)
returnCons, code, err := gcpdb.SingleCon(repoName)
returnCons, code, err := gcpdb.SingleCon(repoName, token)
if err != nil {
return nil, code, err
}

return returnCons, http.StatusOK, nil
}

func GetContributorMonthly(repoInput string) ([]utils.MonthlyConList, int, error) {
func GetContributorMonthly(repoInput string, token string) ([]utils.MonthlyConList, int, error) {
ctx := context.Background()

dbCli, err := datastore.NewClient(ctx, utils.ProjectID)
Expand Down Expand Up @@ -71,15 +71,22 @@ func GetContributorMonthly(repoInput string) ([]utils.MonthlyConList, int, error
})

if len(monthlyConLists) == 0 || monthlyConLists[len(monthlyConLists)-1].Month.AddDate(0, 2, 0).Before(time.Now()) {
ghCli := ghapi.GetGithubClient(ctx, utils.UpdateToken[i%len(utils.UpdateToken)])
var ghToken string
if repoInput == "" {
ghToken = utils.UpdateToken[i%len(utils.UpdateToken)]
} else {
ghToken = token
}

ghCli := ghapi.GetGithubClient(ctx, ghToken)

var firstDay time.Time
if len(monthlyConLists) > 0 {
firstDay = monthlyConLists[len(monthlyConLists)-1].Month.AddDate(0, 1, 0)
} else {
// get first commit of the repo and use it as the start
var code int
firstDay, code, err = ghapi.GetFirstCommit(ctx, ghCli, repoName)
firstDay, code, err = ghapi.GetFirstCommit(ctx, ghCli, repoName, isSearch)
if err != nil {
return nil, code, err
}
Expand Down Expand Up @@ -118,7 +125,7 @@ func GetContributorMonthly(repoInput string) ([]utils.MonthlyConList, int, error
untilDay := sinceDay.AddDate(0, 1, 0)
listCommitOpts := &github.CommitsListOptions{Since: sinceDay, Until: untilDay, ListOptions: ghapi.ListOpts}
for {
commits, resp, statusCode, err := ghapi.GetCommits(ctx, ghCli, repoName, listCommitOpts)
commits, resp, statusCode, err := ghapi.GetCommits(ctx, ghCli, repoName, listCommitOpts, isSearch)
if err != nil {
errCh <- utils.ErrorWithCode{err, statusCode}
return
Expand Down
18 changes: 7 additions & 11 deletions api-go/internal/gcpdb/gcpdb.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ import (

// if repoInput is not empty, fetch single repo and store it in db
// else, use repo list to do daily update for all repos
func UpdateDB(repoInput string) ([]*utils.ConList, int, error) {
func UpdateDB(repoInput string, token string) ([]*utils.ConList, int, error) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()

Expand Down Expand Up @@ -53,7 +53,7 @@ func UpdateDB(repoInput string) ([]*utils.ConList, int, error) {
if repoInput == "" {
ghToken = utils.UpdateToken[i%len(utils.UpdateToken)]
} else {
ghToken = utils.Token
ghToken = token
}
ghCli := ghapi.GetGithubClient(ctx, ghToken)

Expand Down Expand Up @@ -84,7 +84,7 @@ func UpdateDB(repoInput string) ([]*utils.ConList, int, error) {
// get last page
lastPage := 0
listCommitOpts := &github.CommitsListOptions{Since: lastModifiedTimeDB, ListOptions: ghapi.ListOpts}
_, resp, statusCode, err := ghapi.GetCommits(ctx, ghCli, repoName, listCommitOpts)
_, resp, statusCode, err := ghapi.GetCommits(ctx, ghCli, repoName, listCommitOpts, isSearch)
if err != nil {
return nil, statusCode, err
}
Expand Down Expand Up @@ -114,8 +114,8 @@ func UpdateDB(repoInput string) ([]*utils.ConList, int, error) {
return conLists, http.StatusOK, nil
}

func SingleCon(repoInput string) ([]utils.ReturnCon, int, error) {
conLists, code, err := UpdateDB(repoInput)
func SingleCon(repoInput string, token string) ([]utils.ReturnCon, int, error) {
conLists, code, err := UpdateDB(repoInput, token)
if err != nil {
return nil, code, err
}
Expand All @@ -132,7 +132,7 @@ func MultiCon(repoInput string) ([]utils.ReturnCon, int, error) {
conMap := make(map[string]time.Time)

for _, r := range repos {
conLists, code, err := UpdateDB(r)
conLists, code, err := UpdateDB(r, "")
if err != nil {
return nil, code, err
}
Expand Down Expand Up @@ -204,7 +204,7 @@ func updateContributorList(
// could not directly pass listCommitOpts since `Page` would be changed in different goroutine
optsGoroutine := *listCommitOpts
optsGoroutine.Page = i
commits, _, statusCode, err := ghapi.GetCommits(ctx, ghCli, repoName, &optsGoroutine)
commits, _, statusCode, err := ghapi.GetCommits(ctx, ghCli, repoName, &optsGoroutine, isSearch)
if err != nil {
errCh <- utils.ErrorWithCode{err, statusCode}
return
Expand Down Expand Up @@ -245,10 +245,6 @@ func updateContributorList(
return commitLists[i].Date.Before(commitLists[j].Date)
})

for _, c := range commitLists {
fmt.Printf("%s %s\n", c.Date.String(), c.Author)
}

// at most write 500 entities in a single call
rangeMax := 500
rangeNeeded := int(math.Ceil(float64(len(commitLists)) / float64(rangeMax)))
Expand Down
18 changes: 12 additions & 6 deletions api-go/internal/ghapi/ghapi.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,10 @@ func SplitRepo(repo string) (string, string, error) {
}

func GetGithubClient(ctx context.Context, token string) *github.Client {
tc := getToken(ctx, token)
var tc *http.Client
if token != "" {
tc = getToken(ctx, token)
}

return github.NewClient(tc)
}
Expand Down Expand Up @@ -57,7 +60,7 @@ func FormatCommits(ctx context.Context, comLists []*utils.ConList) ([]utils.Retu
return returnCons, http.StatusOK, nil
}

func GetCommits(ctx context.Context, ghCli *github.Client, repoName string, listCommitOpts *github.CommitsListOptions) ([]*github.RepositoryCommit, *github.Response, int, error) {
func GetCommits(ctx context.Context, ghCli *github.Client, repoName string, listCommitOpts *github.CommitsListOptions, isSearch bool) ([]*github.RepositoryCommit, *github.Response, int, error) {
owner, repo, err := SplitRepo(repoName)
if err != nil {
return nil, nil, http.StatusBadRequest, err
Expand All @@ -69,6 +72,9 @@ func GetCommits(ctx context.Context, ghCli *github.Client, repoName string, list
return nil, nil, http.StatusNotFound, fmt.Errorf("Repo not found")
}
if _, ok := err.(*github.RateLimitError); ok || strings.Contains(err.Error(), "403 API rate limit exceeded") {
if isSearch {
return nil, nil, http.StatusForbidden, fmt.Errorf("Hit rate limit! Need Github Token to increase API quota!")
}
// give it another random chance to see if magic happens
*ghCli = *GetGithubClient(ctx, utils.UpdateToken[rand.Intn(len(utils.UpdateToken))])
commits, resp, err = ghCli.Repositories.ListCommits(ctx, owner, repo, listCommitOpts)
Expand All @@ -83,16 +89,16 @@ func GetCommits(ctx context.Context, ghCli *github.Client, repoName string, list
return commits, resp, http.StatusOK, nil
}

func GetFirstCommit(ctx context.Context, ghCli *github.Client, repoName string) (time.Time, int, error) {
func GetFirstCommit(ctx context.Context, ghCli *github.Client, repoName string, isSearch bool) (time.Time, int, error) {
listCommitOpts := &github.CommitsListOptions{}
var firstCommitTime *time.Time
commits, resp, statusCode, err := GetCommits(ctx, ghCli, repoName, listCommitOpts)
commits, resp, statusCode, err := GetCommits(ctx, ghCli, repoName, listCommitOpts, isSearch)
if err != nil {
return time.Time{}, statusCode, err
}
if resp.NextPage != 0 {
listCommitOpts.Page = resp.LastPage
commits, resp, statusCode, err = GetCommits(ctx, ghCli, repoName, listCommitOpts)
commits, resp, statusCode, err = GetCommits(ctx, ghCli, repoName, listCommitOpts, isSearch)
if err != nil {
return time.Time{}, statusCode, err
}
Expand All @@ -116,7 +122,7 @@ func GetFirstCommit(ctx context.Context, ghCli *github.Client, repoName string)

for firstCommitTime == nil {
listCommitOpts.Page = resp.PrevPage
commits, resp, statusCode, err = GetCommits(ctx, ghCli, repoName, listCommitOpts)
commits, resp, statusCode, err = GetCommits(ctx, ghCli, repoName, listCommitOpts, isSearch)
if err != nil {
return time.Time{}, statusCode, err
}
Expand Down
21 changes: 12 additions & 9 deletions api-go/internal/graph/graph.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"io"
"io/ioutil"
"net/http"
"strings"

"cloud.google.com/go/storage"
"github.com/api7/contributor-graph/api/internal/utils"
Expand All @@ -32,16 +33,18 @@ func GenerateAndSaveSVG(ctx context.Context, repo string) error {
return err
}

wc := client.Bucket(bucket).Object(object).NewWriter(ctx)
wc.ACL = []storage.ACLRule{{Entity: storage.AllUsers, Role: storage.RoleReader}}
wc.CacheControl = "public, max-age=86400"
wc.ContentType = "image/svg+xml;charset=utf-8"
if !strings.Contains(repo, ",") {
wc := client.Bucket(bucket).Object(object).NewWriter(ctx)
wc.ACL = []storage.ACLRule{{Entity: storage.AllUsers, Role: storage.RoleReader}}
wc.CacheControl = "public, max-age=86400"
wc.ContentType = "image/svg+xml;charset=utf-8"

if _, err = io.Copy(wc, bytes.NewReader(svg)); err != nil {
return fmt.Errorf("upload svg failed: io.Copy: %v", err)
}
if err := wc.Close(); err != nil {
return fmt.Errorf("upload svg failed: Writer.Close: %v", err)
if _, err = io.Copy(wc, bytes.NewReader(svg)); err != nil {
return fmt.Errorf("upload svg failed: io.Copy: %v", err)
}
if err := wc.Close(); err != nil {
return fmt.Errorf("upload svg failed: Writer.Close: %v", err)
}
}
fmt.Printf("New SVG generated with %s\n", repo)
return nil
Expand Down