Add internal routes for proposals API to get all proposals

cmd/main.go

@@ -8,14 +8,14 @@ import (
 	"net/http"
 	"strings"
 
+	"github.com/gin-contrib/pprof"
 	"github.com/gin-gonic/gin"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/rs/zerolog/log"
 	swaggerFiles "github.com/swaggo/files"
 	ginSwagger "github.com/swaggo/gin-swagger"
 	_ "go.uber.org/automaxprocs"
 
-	"github.com/gin-contrib/pprof"
 	"github.com/mysteriumnetwork/discovery/config"
 	_ "github.com/mysteriumnetwork/discovery/docs"
 	"github.com/mysteriumnetwork/discovery/health"
@@ -78,6 +78,11 @@ func main() {
 	v4 := r.Group("/api/v4")
 	v4.Use(limitMW)
 
+	internal := r.Group("/internal/v4", gin.BasicAuth(gin.Accounts{
+		"internal": cfg.InternalPass,
+	}))
+	internal.Use(limitMW)
+
 	proposalsAPI := proposal.NewAPI(
 		proposalService,
 		locationProvider,
@@ -87,9 +92,9 @@ func main() {
 	)
 	proposalsAPI.RegisterRoutes(v3)
 	proposalsAPI.RegisterRoutes(v4)
+	proposalsAPI.RegisterInternalRoutes(internal)
 
-	health.NewAPI().RegisterRoutes(v3)
-	health.NewAPI().RegisterRoutes(v4)
+	health.NewAPI().RegisterRoutes(v3, v4, internal)
 
 	brokerListener := listener.New(cfg.BrokerURL.String(), proposalRepo)
 

config/options.go

@@ -34,7 +34,8 @@ type Options struct {
 	UniverseJWTSecret string
 	SentinelURL       string
 
-	DevPass string
+	DevPass      string
+	InternalPass string
 
 	MaxRequestsLimit int
 
@@ -81,6 +82,7 @@ func ReadDiscovery() (*Options, error) {
 	}
 
 	devPass := OptionalEnv("DEV_PASS", "")
+	internalPass := OptionalEnv("INTERNAL_PASS", "")
 	logLevel := OptionalEnv("LOG_LEVEL", "debug")
 
 	maxRequestsLimit := OptionalEnv("MAX_REQUESTS_LIMIT", "1000")
@@ -99,6 +101,7 @@ func ReadDiscovery() (*Options, error) {
 		LocationPass:        locationPass,
 		MaxRequestsLimit:    limit,
 		DevPass:             devPass,
+		InternalPass:        internalPass,
 		ProposalsCacheTTL:   *proposalsCacheTTL,
 		ProposalsCacheLimit: proposalsCacheLimit,
 		CountriesCacheLimit: countriesCacheLimit,

health/api.go

@@ -52,7 +52,9 @@ func NewAPI() *API {
 	return &API{}
 }
 
-func (a *API) RegisterRoutes(r gin.IRoutes) {
-	r.GET("/ping", a.Ping)
-	r.GET("/status", a.Status)
+func (a *API) RegisterRoutes(routers ...gin.IRoutes) {
+	for _, r := range routers {
+		r.GET("/ping", a.Ping)
+		r.GET("/status", a.Status)
+	}
 }

proposal/api.go

@@ -75,7 +75,34 @@ func (a *API) Proposals(c *gin.Context) {
 		opts.from = from
 	}
 
-	c.JSON(http.StatusOK, a.service.List(opts))
+	c.JSON(http.StatusOK, a.service.List(opts, true))
+}
+
+// AllProposals list all proposals for internal use.
+// @Summary List all proposals for internal use
+// @Description List all proposals for internal use
+// @Param from query string false "Consumer country"
+// @Param provider_id query string false "Provider ID"
+// @Param service_type query string false "Service type"
+// @Param location_country query string false "Provider country"
+// @Param ip_type query string false "IP type (residential, datacenter, etc.)"
+// @Param access_policy query string false "Access policy. When empty, returns only public proposals (default). Use 'all' to return all."
+// @Param access_policy_source query string false "Access policy source"
+// @Param compatibility_min query number false "Minimum compatibility. When empty, will not filter by it."
+// @Param compatibility_max query number false "Maximum compatibility. When empty, will not filter by it."
+// @Param quality_min query number false "Minimal quality threshold. When empty will be defaulted to 0. Quality ranges from [0.0; 3.0]"
+// @Accept json
+// @Product json
+// @Success 200 {array} v3.Proposal
+// @Router /proposals [get]
+// @Tags proposals
+func (a *API) AllProposals(c *gin.Context) {
+	opts := a.proposalArgs(c)
+	if from, ok := c.Request.Context().Value(ctxCountryKey{}).(string); ok {
+		opts.from = from
+	}
+
+	c.JSON(http.StatusOK, a.service.List(opts, false))
 }
 
 // CountriesNumbers list number of providers in each country.
@@ -101,7 +128,7 @@ func (a *API) CountriesNumbers(c *gin.Context) {
 		opts.from = from
 	}
 
-	c.JSON(http.StatusOK, a.service.ListCountriesNumbers(opts))
+	c.JSON(http.StatusOK, a.service.ListCountriesNumbers(opts, false))
 }
 
 func (a *API) RegisterRoutes(r gin.IRoutes) {
@@ -132,6 +159,26 @@ func (a *API) RegisterRoutes(r gin.IRoutes) {
 		r.GET("/countries", countryMW, a.CountriesNumbers)
 		r.GET("/proposals", countryMW, a.Proposals)
 	}
+	r.GET("/proposals-metadata", a.ProposalsMetadata) // TODO move this into internal routes only once we migrate existing services to use it.
+}
+
+func (a *API) RegisterInternalRoutes(r gin.IRoutes) {
+	countryMW := a.populateCountryMiddleware()
+	cacheStrategy := a.newCacheStrategy()
+	if a.proposalsCacheTTL > 0 {
+		r.GET(
+			"/proposals",
+			countryMW,
+			cache.Cache(
+				a.proposalsCache,
+				a.proposalsCacheTTL,
+				cache.WithCacheStrategyByRequest(cacheStrategy),
+			),
+			a.AllProposals,
+		)
+	} else {
+		r.GET("/proposals", countryMW, a.AllProposals)
+	}
 	r.GET("/proposals-metadata", a.ProposalsMetadata)
 }
 

proposal/inmemory.go

@@ -61,7 +61,7 @@ func NewRepository(enhancers []Enhancer) *Repository {
 	}
 }
 
-func (r *Repository) List(opts repoListOpts) (res []v3.Proposal) {
+func (r *Repository) List(opts repoListOpts, limited bool) (res []v3.Proposal) {
 	r.mu.RLock()
 	defer r.mu.RUnlock()
 
@@ -91,8 +91,8 @@ func (r *Repository) List(opts repoListOpts) (res []v3.Proposal) {
 
 		countryLimit[p.proposal.Location.Country]++
 
-		if countryLimit[p.proposal.Location.Country] <= countryHardLimit {
-			if countryLimit[p.proposal.Location.Country] <= countrySoftLimit || countryLimit[p.proposal.Location.Country]%10 == 0 {
+		if !limited || countryLimit[p.proposal.Location.Country] <= countryHardLimit {
+			if !limited || countryLimit[p.proposal.Location.Country] <= countrySoftLimit || countryLimit[p.proposal.Location.Country]%10 == 0 {
 				res = append(res, p.proposal)
 			}
 		}

proposal/service.go

@@ -45,7 +45,7 @@ type ListOpts struct {
 	presetID                int
 }
 
-func (s *Service) List(opts ListOpts) []v3.Proposal {
+func (s *Service) List(opts ListOpts, limited bool) []v3.Proposal {
 	proposals := s.Repository.List(repoListOpts{
 		providerIDS:        opts.providerIDS,
 		serviceType:        opts.serviceType,
@@ -56,7 +56,7 @@ func (s *Service) List(opts ListOpts) []v3.Proposal {
 		compatibilityMin:   opts.compatibilityMin,
 		compatibilityMax:   opts.compatibilityMax,
 		tags:               opts.tags,
-	})
+	}, limited)
 
 	or := &metrics.OracleResponses{}
 	or.Load(s.qualityService, opts.from)
@@ -77,7 +77,7 @@ func (s *Service) Metadata(opts repoMetadataOpts) []v3.Metadata {
 	return s.Repository.Metadata(opts, or.QualityResponse)
 }
 
-func (s *Service) ListCountriesNumbers(opts ListOpts) map[string]int {
+func (s *Service) ListCountriesNumbers(opts ListOpts, limited bool) map[string]int {
 	if opts.presetID == 0 {
 		return s.Repository.ListCountriesNumbers(repoListOpts{
 			providerIDS:        opts.providerIDS,
@@ -102,7 +102,7 @@ func (s *Service) ListCountriesNumbers(opts ListOpts) map[string]int {
 		compatibilityMin:   opts.compatibilityMin,
 		compatibilityMax:   opts.compatibilityMax,
 		tags:               opts.tags,
-	})
+	}, limited)
 
 	or := &metrics.OracleResponses{}
 	or.Load(s.qualityService, opts.from)