From 2c4170a8c362c638610d87d61a2cee830864cf68 Mon Sep 17 00:00:00 2001 From: ysebyy Date: Wed, 7 Aug 2024 10:48:54 +0300 Subject: [PATCH] Fix repo crawling --- internal/requests.go | 28 ++++++++++++++++++---------- internal/requests_test.go | 1 + pkg/bomtools/merge.go | 13 +++++++++++-- pkg/dtrack/http.go | 9 +++++---- 4 files changed, 35 insertions(+), 16 deletions(-) diff --git a/internal/requests.go b/internal/requests.go index 364180e..39abced 100644 --- a/internal/requests.go +++ b/internal/requests.go @@ -31,6 +31,10 @@ type GetRepositoriesConfig struct { } func NewGetRepositoriesConfig(ctx context.Context, url, username, apiToken string, org string) GetRepositoriesConfig { + if org != "" { + apiToken = gh.GenerateGithubAppTokenInternal(org) + } + return GetRepositoriesConfig{ ctx: ctx, URL: url, @@ -139,15 +143,8 @@ func GetRepositories(conf GetRepositoriesConfig) ([]repositoryMapping, error) { return repositories, nil } - var validRepositories []repositoryMapping - for _, r := range repositories { - if !r.Archived { - validRepositories = append(validRepositories, r) - } - } - return validRepositories, nil + return repositories, nil } - return exponentialBackoff(getRepositories, conf.BackoffPolicy...) } @@ -167,7 +164,7 @@ func WalkRepositories(conf GetRepositoriesConfig, callback func(repositoryURLs [ query.Set("page", strconv.Itoa(page)) endpoint.RawQuery = query.Encode() conf.URL = endpoint.String() - + log.WithField("request github", endpoint.String()).Infof("Getting query for page %d", page) repositories, err = GetRepositories(conf) if err != nil { if regenCount < 1 { @@ -192,7 +189,18 @@ func WalkRepositories(conf GetRepositoriesConfig, callback func(repositoryURLs [ regenCount = 0 } - if len(repositories) == 0 { + var validRepositories []repositoryMapping + var archivedRepositories []repositoryMapping + for _, r := range repositories { + if !r.Archived { + validRepositories = append(validRepositories, r) + } else if r.Archived { + archivedRepositories = append(archivedRepositories, r) + } + } + + if len(validRepositories) == 0 && len(archivedRepositories) == 0 { + log.WithField("request github return", endpoint.String()).Infof("returning with page %d", page) return nil // Done, all repositories have been walked } diff --git a/internal/requests_test.go b/internal/requests_test.go index 5d6d34c..fb629ff 100644 --- a/internal/requests_test.go +++ b/internal/requests_test.go @@ -71,6 +71,7 @@ func TestGetRepositories(t *testing.T) { want := []repositoryMapping{ {Name: "xmlsec", Archived: false, Language: "C", URL: "https://github.com/vinted/xmlsec"}, + {Name: "airbrake", Archived: true, Language: "Ruby", URL: "https://github.com/vinted/airbrake-graylog2"}, {Name: "dotpay", Archived: false, Language: "Ruby", URL: "https://github.com/vinted/dotpay"}, } assert.Equal(t, want, repositories) diff --git a/pkg/bomtools/merge.go b/pkg/bomtools/merge.go index 63b465b..5622779 100644 --- a/pkg/bomtools/merge.go +++ b/pkg/bomtools/merge.go @@ -201,8 +201,17 @@ func mergeAllByPURL(component *cdx.Component, allComponents []*cdx.Component) *c mergedComponent.Properties = &p } if c.Licenses != nil { - l := mergeCollection[cdx.LicenseChoice](*c.Licenses, *mergedComponent.Licenses) - mergedComponent.Licenses = (*cdx.Licenses)(&l) + l := make([]cdx.LicenseChoice, 0) + for _, sl := range *c.Licenses { + // Check for license ID + if sl.License != nil && sl.License.ID != "" { + l = append(l, sl) + } + } + + // Assuming mergedComponent.Licenses is initialized properly earlier + mergedLicenses := mergeCollection[cdx.LicenseChoice](l, *mergedComponent.Licenses) + mergedComponent.Licenses = (*cdx.Licenses)(&mergedLicenses) } if c.ExternalReferences != nil { e := mergeCollection[cdx.ExternalReference](*c.ExternalReferences, *mergedComponent.ExternalReferences) diff --git a/pkg/dtrack/http.go b/pkg/dtrack/http.go index ded7391..a63a9d0 100644 --- a/pkg/dtrack/http.go +++ b/pkg/dtrack/http.go @@ -67,11 +67,12 @@ func (d DependencyTrackClient) createProject(ctx context.Context, payload create d.setRequiredHeaders(req) resp, err := http.DefaultClient.Do(req) - log.WithField("funcType", "createProject").Debugf("CreateProject request response body: %s", resp.Body) - log.WithField("funcType", "createProject").Debugf("CreateProject request response status code: %v", resp.StatusCode) if err != nil { + log.WithField("funcType", "createProject").Debugf("CreateProject error %v", err.Error()) return "", fmt.Errorf(cantPerformHTTPRequest, requestURL, err) } + // putting err here incase we get a resp nil + log.WithField("funcType", "createProject").Debugf("CreateProject request response status code: %v", resp.StatusCode) defer func() { closeErr := resp.Body.Close() @@ -125,11 +126,11 @@ func (d DependencyTrackClient) updateSBOMs(ctx context.Context, payload updateSB d.setRequiredHeaders(req) resp, err := http.DefaultClient.Do(req) - log.WithField("funcType", "updateSBOM").Debugf("CreateProject request response body: %s", resp.Body) - log.WithField("funcType", "updateSBOM").Debugf("CreateProject request response status code: %v", resp.StatusCode) if err != nil { return fmt.Errorf(cantPerformHTTPRequest, requestURL, err) } + log.WithField("funcType", "updateSBOM").Debugf("CreateProject request response body: %s", resp.Body) + log.WithField("funcType", "updateSBOM").Debugf("CreateProject request response status code: %v", resp.StatusCode) defer func() { closeErr := resp.Body.Close()