From a975452f504e0d5a5c2eda9a6ad359530bb042e9 Mon Sep 17 00:00:00 2001 From: Dmitry Shihovtsev Date: Tue, 28 Nov 2023 09:20:25 +0600 Subject: [PATCH] Add internal routes for proposals API to get all proposals --- cmd/main.go | 11 +++++++--- config/options.go | 5 ++++- health/api.go | 8 ++++--- proposal/api.go | 51 ++++++++++++++++++++++++++++++++++++++++++-- proposal/inmemory.go | 6 +++--- proposal/service.go | 8 +++---- 6 files changed, 73 insertions(+), 16 deletions(-) diff --git a/cmd/main.go b/cmd/main.go index 1690a3a..8dd4f16 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -8,6 +8,7 @@ import ( "net/http" "strings" + "github.com/gin-contrib/pprof" "github.com/gin-gonic/gin" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/rs/zerolog/log" @@ -15,7 +16,6 @@ import ( ginSwagger "github.com/swaggo/gin-swagger" _ "go.uber.org/automaxprocs" - "github.com/gin-contrib/pprof" "github.com/mysteriumnetwork/discovery/config" _ "github.com/mysteriumnetwork/discovery/docs" "github.com/mysteriumnetwork/discovery/health" @@ -78,6 +78,11 @@ func main() { v4 := r.Group("/api/v4") v4.Use(limitMW) + internal := r.Group("/internal/v4", gin.BasicAuth(gin.Accounts{ + "internal": cfg.InternalPass, + })) + internal.Use(limitMW) + proposalsAPI := proposal.NewAPI( proposalService, locationProvider, @@ -87,9 +92,9 @@ func main() { ) proposalsAPI.RegisterRoutes(v3) proposalsAPI.RegisterRoutes(v4) + proposalsAPI.RegisterInternalRoutes(internal) - health.NewAPI().RegisterRoutes(v3) - health.NewAPI().RegisterRoutes(v4) + health.NewAPI().RegisterRoutes(v3, v4, internal) brokerListener := listener.New(cfg.BrokerURL.String(), proposalRepo) diff --git a/config/options.go b/config/options.go index 9aaa25d..0411009 100644 --- a/config/options.go +++ b/config/options.go @@ -34,7 +34,8 @@ type Options struct { UniverseJWTSecret string SentinelURL string - DevPass string + DevPass string + InternalPass string MaxRequestsLimit int @@ -81,6 +82,7 @@ func ReadDiscovery() (*Options, error) { } devPass := OptionalEnv("DEV_PASS", "") + internalPass := OptionalEnv("INTERNAL_PASS", "") logLevel := OptionalEnv("LOG_LEVEL", "debug") maxRequestsLimit := OptionalEnv("MAX_REQUESTS_LIMIT", "1000") @@ -99,6 +101,7 @@ func ReadDiscovery() (*Options, error) { LocationPass: locationPass, MaxRequestsLimit: limit, DevPass: devPass, + InternalPass: internalPass, ProposalsCacheTTL: *proposalsCacheTTL, ProposalsCacheLimit: proposalsCacheLimit, CountriesCacheLimit: countriesCacheLimit, diff --git a/health/api.go b/health/api.go index c8210c4..143f875 100644 --- a/health/api.go +++ b/health/api.go @@ -52,7 +52,9 @@ func NewAPI() *API { return &API{} } -func (a *API) RegisterRoutes(r gin.IRoutes) { - r.GET("/ping", a.Ping) - r.GET("/status", a.Status) +func (a *API) RegisterRoutes(routers ...gin.IRoutes) { + for _, r := range routers { + r.GET("/ping", a.Ping) + r.GET("/status", a.Status) + } } diff --git a/proposal/api.go b/proposal/api.go index 11f3470..afbfc63 100644 --- a/proposal/api.go +++ b/proposal/api.go @@ -75,7 +75,34 @@ func (a *API) Proposals(c *gin.Context) { opts.from = from } - c.JSON(http.StatusOK, a.service.List(opts)) + c.JSON(http.StatusOK, a.service.List(opts, true)) +} + +// AllProposals list all proposals for internal use. +// @Summary List all proposals for internal use +// @Description List all proposals for internal use +// @Param from query string false "Consumer country" +// @Param provider_id query string false "Provider ID" +// @Param service_type query string false "Service type" +// @Param location_country query string false "Provider country" +// @Param ip_type query string false "IP type (residential, datacenter, etc.)" +// @Param access_policy query string false "Access policy. When empty, returns only public proposals (default). Use 'all' to return all." +// @Param access_policy_source query string false "Access policy source" +// @Param compatibility_min query number false "Minimum compatibility. When empty, will not filter by it." +// @Param compatibility_max query number false "Maximum compatibility. When empty, will not filter by it." +// @Param quality_min query number false "Minimal quality threshold. When empty will be defaulted to 0. Quality ranges from [0.0; 3.0]" +// @Accept json +// @Product json +// @Success 200 {array} v3.Proposal +// @Router /proposals [get] +// @Tags proposals +func (a *API) AllProposals(c *gin.Context) { + opts := a.proposalArgs(c) + if from, ok := c.Request.Context().Value(ctxCountryKey{}).(string); ok { + opts.from = from + } + + c.JSON(http.StatusOK, a.service.List(opts, false)) } // CountriesNumbers list number of providers in each country. @@ -101,7 +128,7 @@ func (a *API) CountriesNumbers(c *gin.Context) { opts.from = from } - c.JSON(http.StatusOK, a.service.ListCountriesNumbers(opts)) + c.JSON(http.StatusOK, a.service.ListCountriesNumbers(opts, false)) } func (a *API) RegisterRoutes(r gin.IRoutes) { @@ -132,6 +159,26 @@ func (a *API) RegisterRoutes(r gin.IRoutes) { r.GET("/countries", countryMW, a.CountriesNumbers) r.GET("/proposals", countryMW, a.Proposals) } + r.GET("/proposals-metadata", a.ProposalsMetadata) // TODO move this into internal routes only once we migrate existing services to use it. +} + +func (a *API) RegisterInternalRoutes(r gin.IRoutes) { + countryMW := a.populateCountryMiddleware() + cacheStrategy := a.newCacheStrategy() + if a.proposalsCacheTTL > 0 { + r.GET( + "/proposals", + countryMW, + cache.Cache( + a.proposalsCache, + a.proposalsCacheTTL, + cache.WithCacheStrategyByRequest(cacheStrategy), + ), + a.AllProposals, + ) + } else { + r.GET("/proposals", countryMW, a.AllProposals) + } r.GET("/proposals-metadata", a.ProposalsMetadata) } diff --git a/proposal/inmemory.go b/proposal/inmemory.go index d16fae1..b774223 100644 --- a/proposal/inmemory.go +++ b/proposal/inmemory.go @@ -61,7 +61,7 @@ func NewRepository(enhancers []Enhancer) *Repository { } } -func (r *Repository) List(opts repoListOpts) (res []v3.Proposal) { +func (r *Repository) List(opts repoListOpts, limited bool) (res []v3.Proposal) { r.mu.RLock() defer r.mu.RUnlock() @@ -91,8 +91,8 @@ func (r *Repository) List(opts repoListOpts) (res []v3.Proposal) { countryLimit[p.proposal.Location.Country]++ - if countryLimit[p.proposal.Location.Country] <= countryHardLimit { - if countryLimit[p.proposal.Location.Country] <= countrySoftLimit || countryLimit[p.proposal.Location.Country]%10 == 0 { + if !limited || countryLimit[p.proposal.Location.Country] <= countryHardLimit { + if !limited || countryLimit[p.proposal.Location.Country] <= countrySoftLimit || countryLimit[p.proposal.Location.Country]%10 == 0 { res = append(res, p.proposal) } } diff --git a/proposal/service.go b/proposal/service.go index 4e8df1a..255ce85 100644 --- a/proposal/service.go +++ b/proposal/service.go @@ -45,7 +45,7 @@ type ListOpts struct { presetID int } -func (s *Service) List(opts ListOpts) []v3.Proposal { +func (s *Service) List(opts ListOpts, limited bool) []v3.Proposal { proposals := s.Repository.List(repoListOpts{ providerIDS: opts.providerIDS, serviceType: opts.serviceType, @@ -56,7 +56,7 @@ func (s *Service) List(opts ListOpts) []v3.Proposal { compatibilityMin: opts.compatibilityMin, compatibilityMax: opts.compatibilityMax, tags: opts.tags, - }) + }, limited) or := &metrics.OracleResponses{} or.Load(s.qualityService, opts.from) @@ -77,7 +77,7 @@ func (s *Service) Metadata(opts repoMetadataOpts) []v3.Metadata { return s.Repository.Metadata(opts, or.QualityResponse) } -func (s *Service) ListCountriesNumbers(opts ListOpts) map[string]int { +func (s *Service) ListCountriesNumbers(opts ListOpts, limited bool) map[string]int { if opts.presetID == 0 { return s.Repository.ListCountriesNumbers(repoListOpts{ providerIDS: opts.providerIDS, @@ -102,7 +102,7 @@ func (s *Service) ListCountriesNumbers(opts ListOpts) map[string]int { compatibilityMin: opts.compatibilityMin, compatibilityMax: opts.compatibilityMax, tags: opts.tags, - }) + }, limited) or := &metrics.OracleResponses{} or.Load(s.qualityService, opts.from)