Merge pull request #23 from stacklok/parallel

Support parallel lookups

go.mod

@@ -5,13 +5,18 @@ go 1.22.1
 require (
 	github.com/BurntSushi/toml v1.4.0
 	github.com/google/go-github/v61 v61.0.0
+	github.com/package-url/packageurl-go v0.1.3
 	github.com/stretchr/testify v1.9.0
 	golang.org/x/oauth2 v0.21.0
+	sigs.k8s.io/release-utils v0.8.4
 )
 
 require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
+	github.com/nozzle/throttler v0.0.0-20180817012639-2ea982251481 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/sirupsen/logrus v1.9.3 // indirect
+	golang.org/x/sys v0.20.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -1,5 +1,6 @@
 github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0=
 github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
@@ -9,14 +10,28 @@ github.com/google/go-github/v61 v61.0.0 h1:VwQCBwhyE9JclCI+22/7mLB1PuU9eowCXKY5p
 github.com/google/go-github/v61 v61.0.0/go.mod h1:0WR+KmsWX75G2EbpyGsGmradjo3IiciuI4BmdVCobQY=
 github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
 github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
+github.com/nozzle/throttler v0.0.0-20180817012639-2ea982251481 h1:Up6+btDp321ZG5/zdSLo48H9Iaq0UQGthrhWC6pCxzE=
+github.com/nozzle/throttler v0.0.0-20180817012639-2ea982251481/go.mod h1:yKZQO8QE2bHlgozqWDiRVqTFlLQSj30K/6SAK8EeYFw=
+github.com/package-url/packageurl-go v0.1.3 h1:4juMED3hHiz0set3Vq3KeQ75KD1avthoXLtmE3I0PLs=
+github.com/package-url/packageurl-go v0.1.3/go.mod h1:nKAWB8E6uk1MHqiS/lQb9pYBGH2+mdJ2PJc2s50dQY0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
+github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs=
 golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
+golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
+golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+sigs.k8s.io/release-utils v0.8.4 h1:4QVr3UgbyY/d9p74LBhg0njSVQofUsAZqYOzVZBhdBw=
+sigs.k8s.io/release-utils v0.8.4/go.mod h1:m1bHfscTemQp+z+pLCZnkXih9n0+WukIUU70n6nFnU0=

pkg/client/client.go

@@ -25,34 +25,43 @@ import (
 	"os"
 	"strings"
 
+	packageurl "github.com/package-url/packageurl-go"
+	khttp "sigs.k8s.io/release-utils/http"
+
 	"github.com/stacklok/trusty-sdk-go/pkg/types"
 )
 
 const (
-	defaultEndpoint = "https://api.trustypkg.dev"
+	defaultEndpoint = "https://gh.trustypkg.dev"
 	endpointEnvVar  = "TRUSTY_ENDPOINT"
 	reportPath      = "v1/report"
 )
 
 // Options configures the Trusty API client
 type Options struct {
 	HttpClient netClient
-	BaseURL    string
+	// Workers is the number of parallel request the client makes to the API
+	Workers int
+
+	// BaseURL of the Trusty API
+	BaseURL string
 }
 
 // DefaultOptions is the default Trusty client options set
 var DefaultOptions = Options{
-	HttpClient: &http.Client{},
-	BaseURL:    defaultEndpoint,
+	Workers: 2,
+	BaseURL: defaultEndpoint,
 }
 
 type netClient interface {
-	Do(req *http.Request) (*http.Response, error)
+	GetRequestGroup([]string) ([]*http.Response, []error)
+	GetRequest(string) (*http.Response, error)
 }
 
 // New returns a new Trusty REST client
 func New() *Trusty {
 	opts := DefaultOptions
+	opts.HttpClient = khttp.NewAgent().WithMaxParallel(opts.Workers).WithFailOnHTTPError(true)
 	if ep := os.Getenv(endpointEnvVar); ep != "" {
 		opts.BaseURL = ep
 	}
@@ -90,25 +99,52 @@ type Trusty struct {
 	Options Options
 }
 
-// newRequest buids a new http GET request using the preconfigured trusty API uri
-func (t *Trusty) newRequest(ctx context.Context, path string, params map[string]string) (*http.Request, error) {
-	u, err := urlFromEndpointAndPaths(t.Options.BaseURL, path, params)
-	if err != nil {
-		return nil, fmt.Errorf("could not parse endpoint: %w", err)
+// GroupReport queries the Trusty API in parallel for a group of dependencies.
+func (t *Trusty) GroupReport(_ context.Context, deps []*types.Dependency) ([]*types.Reply, error) {
+	urls := []string{}
+	for _, dep := range deps {
+		u, err := t.PackageEndpoint(dep)
+		if err != nil {
+			return nil, fmt.Errorf("unable to get endpoint for: %q: %w", dep.Name, err)
+		}
+		urls = append(urls, u)
+	}
+
+	responses, errs := t.Options.HttpClient.GetRequestGroup(urls)
+	if err := errors.Join(errs...); err != nil {
+		return nil, fmt.Errorf("fetching data from Trusty: %w", err)
 	}
 
-	req, err := http.NewRequest("GET", u.String(), nil)
+	// Parse the replies
+	resps := make([]*types.Reply, len(responses))
+	for i := range responses {
+		defer responses[i].Body.Close()
+		dec := json.NewDecoder(responses[i].Body)
+		resps[i] = &types.Reply{}
+		if err := dec.Decode(resps[i]); err != nil {
+			return nil, fmt.Errorf("could not unmarshal response #%d: %w", i, err)
+		}
+	}
+	return resps, nil
+}
+
+// PurlEndpoint returns the API endpoint url to query for data about a purl
+func (t *Trusty) PurlEndpoint(purl string) (string, error) {
+	dep, err := t.PurlToDependency(purl)
+	if err != nil {
+		return "", fmt.Errorf("getting dependency from %q", purl)
+	}
+	ep, err := t.PackageEndpoint(dep)
 	if err != nil {
-		return nil, fmt.Errorf("could not create request: %w", err)
+		return "", fmt.Errorf("getting package endpoint: %w", err)
 	}
-	req = req.WithContext(ctx)
-	return req, nil
+	return ep, nil
 }
 
-// Report returns a dependency report with all the data that Trust has
-// available for a package
-func (t *Trusty) Report(ctx context.Context, dep *types.Dependency) (*types.Reply, error) {
-	// Check dependency:
+// PackageEndpoint takes a dependency and returns the Trusty endpoint to
+// query data about it.
+func (t *Trusty) PackageEndpoint(dep *types.Dependency) (string, error) {
+	// Check dependency data:
 	errs := []error{}
 	if dep.Name == "" {
 		errs = append(errs, fmt.Errorf("dependency has no name defined"))
@@ -117,21 +153,76 @@ func (t *Trusty) Report(ctx context.Context, dep *types.Dependency) (*types.Repl
 		errs = append(errs, fmt.Errorf("dependency has no ecosystem set"))
 	}
 
-	preErr := errors.Join(errs...)
-	if preErr != nil {
-		return nil, preErr
+	if err := errors.Join(errs...); err != nil {
+		return "", err
+	}
+
+	u, err := url.Parse(t.Options.BaseURL + "/" + reportPath)
+	if err != nil {
+		return "", fmt.Errorf("failed to parse endpoint: %w", err)
 	}
 
 	params := map[string]string{
 		"package_name": dep.Name,
 		"package_type": strings.ToLower(dep.Ecosystem.AsString()),
 	}
-	req, err := t.newRequest(ctx, reportPath, params)
+
+	// Add query parameters for package_name and package_type
+	q := u.Query()
+	for k, v := range params {
+		q.Set(k, v)
+	}
+	u.RawQuery = q.Encode()
+
+	return u.String(), nil
+}
+
+// PurlToEcosystem returns a trusty ecosystem constant from a Package URL's type
+func (_ *Trusty) PurlToEcosystem(purl string) types.Ecosystem {
+	switch {
+	case strings.HasPrefix(purl, "pkg:golang"):
+		return types.ECOSYSTEM_GO
+	case strings.HasPrefix(purl, "pkg:npm"):
+		return types.ECOSYSTEM_NPM
+	case strings.HasPrefix(purl, "pkg:pypi"):
+		return types.ECOSYSTEM_PYPI
+	default:
+		return types.Ecosystem(0)
+	}
+}
+
+// PurlToDependency takes a string with a package url
+func (t *Trusty) PurlToDependency(purlString string) (*types.Dependency, error) {
+	e := t.PurlToEcosystem(purlString)
+	if e == 0 {
+		// Ecosystem nil or not supported
+		return nil, fmt.Errorf("ecosystem not supported")
+	}
+
+	purl, err := packageurl.FromString(purlString)
+	if err != nil {
+		return nil, fmt.Errorf("unable to parse package url: %w", err)
+	}
+	name := purl.Name
+	if purl.Namespace != "" {
+		name = purl.Namespace + "/" + purl.Name
+	}
+	return &types.Dependency{
+		Ecosystem: e,
+		Name:      name,
+		Version:   purl.Version,
+	}, nil
+}
+
+// Report returns a dependency report with all the data that Trust has
+// available for a package
+func (t *Trusty) Report(_ context.Context, dep *types.Dependency) (*types.Reply, error) {
+	u, err := t.PackageEndpoint(dep)
 	if err != nil {
-		return nil, fmt.Errorf("could not create request: %w", err)
+		return nil, fmt.Errorf("computing package endpoint: %w", err)
 	}
 
-	resp, err := t.Options.HttpClient.Do(req)
+	resp, err := t.Options.HttpClient.GetRequest(u)
 	if err != nil {
 		return nil, fmt.Errorf("could not send request: %w", err)
 	}