From a1cd2fd36d7d2fe13f431d6ca6f62404eba0d0e6 Mon Sep 17 00:00:00 2001 From: LUCCA DUKIC <109136188+LuccaBitfly@users.noreply.github.com> Date: Tue, 20 Aug 2024 15:06:27 +0200 Subject: [PATCH 01/16] (BEDS-295) implement automated api doc generation --- backend/.gitignore | 3 +- backend/Makefile | 5 +- backend/cmd/api_docs/main.go | 138 +++++++++++++++++++ backend/go.mod | 12 +- backend/go.sum | 17 ++- backend/pkg/api/data_access/data_access.go | 1 + backend/pkg/api/data_access/dummy.go | 6 + backend/pkg/api/data_access/ratelimit.go | 22 +++ backend/pkg/api/handlers/public.go | 78 ++++++++++- backend/pkg/api/types/data_access.go | 10 ++ backend/pkg/api/types/validator_dashboard.go | 1 + backend/pkg/commons/ratelimit/ratelimit.go | 6 +- 12 files changed, 289 insertions(+), 10 deletions(-) create mode 100644 backend/cmd/api_docs/main.go create mode 100644 backend/pkg/api/data_access/ratelimit.go diff --git a/backend/.gitignore b/backend/.gitignore index 7006633d8..6d056b68d 100644 --- a/backend/.gitignore +++ b/backend/.gitignore @@ -5,4 +5,5 @@ local_deployment/config.yml local_deployment/elconfig.json local_deployment/.env __gitignore -cmd/playground \ No newline at end of file +cmd/playground +docs diff --git a/backend/Makefile b/backend/Makefile index b08c48a60..299c605a9 100644 --- a/backend/Makefile +++ b/backend/Makefile @@ -8,7 +8,7 @@ LDFLAGS="-X ${PACKAGE}/version.Version=${VERSION} -X ${PACKAGE}/version.BuildDat CGO_CFLAGS="-O -D__BLST_PORTABLE__" CGO_CFLAGS_ALLOW="-O -D__BLST_PORTABLE__" -all: exporter blobindexer misc ethstore-exporter api rewards-exporter eth1indexer stats user-service notification-sender notification-collector node-jobs-processor signatures +all: exporter blobindexer misc ethstore-exporter api api-docs rewards-exporter eth1indexer stats user-service notification-sender notification-collector node-jobs-processor signatures clean: rm -rf bin @@ -63,3 +63,6 @@ frontend-types: addhooks: git config core.hooksPath hooks + +api-docs: + go run cmd/api_docs/main.go \ No newline at end of file diff --git a/backend/cmd/api_docs/main.go b/backend/cmd/api_docs/main.go new file mode 100644 index 000000000..6d2097288 --- /dev/null +++ b/backend/cmd/api_docs/main.go @@ -0,0 +1,138 @@ +package main + +import ( + "context" + "encoding/json" + "flag" + "fmt" + "os" + "strings" + + "github.com/go-openapi/spec" + dataaccess "github.com/gobitfly/beaconchain/pkg/api/data_access" + types "github.com/gobitfly/beaconchain/pkg/api/types" + "github.com/gobitfly/beaconchain/pkg/commons/log" + "github.com/gobitfly/beaconchain/pkg/commons/ratelimit" + commonTypes "github.com/gobitfly/beaconchain/pkg/commons/types" + "github.com/gobitfly/beaconchain/pkg/commons/utils" + "github.com/gobitfly/beaconchain/pkg/commons/version" + "github.com/swaggo/swag/gen" +) + +const ( + searchDir = "./pkg/api" + mainAPI = "handlers/public.go" + outputDir = "./docs" + outputType = "json" // can also be yaml + + apiPrefix = "/api/v2" +) + +func main() { + configPath := flag.String("config", "", "Path to the config file, if empty string defaults will be used") + versionFlag := flag.Bool("version", false, "Show version and exit") + flag.Parse() + + if *versionFlag { + log.Infof(version.Version) + log.Infof(version.GoVersion) + return + } + // generate swagger doc + config := &gen.Config{ + SearchDir: searchDir, + MainAPIFile: mainAPI, + OutputDir: outputDir, + OutputTypes: []string{outputType}, + } + err := gen.New().Build(config) + if err != nil { + log.Fatal(err, "error generating swagger docs", 0) + } + + log.Info("swagger docs generated successfully, now loading endpoint weights from db", 0) + + // load endpoint weights from db + cfg := &commonTypes.Config{} + err = utils.ReadConfig(cfg, *configPath) + if err != nil { + log.Fatal(err, "error reading config file", 0) + } + + da := dataaccess.NewDataAccessService(cfg) + defer da.Close() + apiWeights, err := da.GetApiWeights(context.Background()) + if err != nil { + log.Fatal(err, "error loading endpoint weights from db", 0) + } + + // insert endpoint weights into swagger doc + data, err := os.ReadFile(outputDir + "/swagger." + outputType) + if err != nil { + log.Fatal(err, "error reading swagger docs", 0) + } + newData, err := insertApiWeights(data, apiWeights) + if err != nil { + log.Fatal(err, "error inserting api weights", 0) + } + + // write updated swagger doc + err = os.WriteFile(outputDir+"/swagger."+outputType, newData, os.ModePerm) + if err != nil { + log.Fatal(err, "error writing new swagger docs", 0) + } + + log.Info("api weights inserted successfully", 0) +} + +func insertApiWeights(data []byte, apiWeightItems []types.ApiWeightItem) ([]byte, error) { + // unmarshal swagger file + var swagger *spec.Swagger + if err := json.Unmarshal(data, &swagger); err != nil { + return nil, fmt.Errorf("error unmarshalling swagger docs: %w", err) + } + + // iterate endpoints from swagger file + for pathString, pathItem := range swagger.Paths.Paths { + for methodString, operation := range getPathItemOperationMap(pathItem) { + if operation == nil { + continue + } + // get weight and bucket for each endpoint + weight := 1 + bucket := ratelimit.DefaultBucket + for _, apiWeightItem := range apiWeightItems { + // ignore endpoints that don't belong to v2 + if !strings.HasPrefix(apiWeightItem.Endpoint, apiPrefix) { + continue + } + // compare endpoint and method + if pathString == strings.TrimPrefix(apiWeightItem.Endpoint, apiPrefix) && methodString == apiWeightItem.Method { + weight = apiWeightItem.Weight + bucket = apiWeightItem.Bucket + break + } + } + // insert weight and bucket into endpoint summary + plural := "" + if weight > 1 { + plural = "s" + } + operation.Summary = fmt.Sprintf("(%d %s credit%s) %s", weight, bucket, plural, operation.Summary) + } + } + + return json.MarshalIndent(swagger, "", " ") +} + +func getPathItemOperationMap(item spec.PathItem) map[string]*spec.Operation { + return map[string]*spec.Operation{ + "get": item.Get, + "put": item.Put, + "post": item.Post, + "delete": item.Delete, + "options": item.Options, + "head": item.Head, + "patch": item.Patch, + } +} diff --git a/backend/go.mod b/backend/go.mod index f12feed71..6b3e6548f 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -21,6 +21,7 @@ require ( github.com/doug-martin/goqu/v9 v9.19.0 github.com/ethereum/go-ethereum v1.13.12 github.com/go-faker/faker/v4 v4.3.0 + github.com/go-openapi/spec v0.20.14 github.com/go-redis/redis/v8 v8.11.5 github.com/gobitfly/eth-rewards v0.1.2-0.20230403064929-411ddc40a5f7 github.com/gobitfly/eth.store v0.0.0-20240312111708-b43f13990280 @@ -60,6 +61,7 @@ require ( github.com/rocket-pool/smartnode v1.13.6 github.com/shopspring/decimal v1.3.1 github.com/sirupsen/logrus v1.9.3 + github.com/swaggo/swag v1.16.3 github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d github.com/wealdtech/go-ens/v3 v3.6.0 github.com/wealdtech/go-eth2-types/v2 v2.8.2 @@ -69,9 +71,9 @@ require ( golang.org/x/exp v0.0.0-20240213143201-ec583247a57a golang.org/x/sync v0.6.0 golang.org/x/text v0.14.0 + golang.org/x/time v0.5.0 golang.org/x/tools v0.18.0 google.golang.org/api v0.164.0 - google.golang.org/appengine v1.6.8 google.golang.org/protobuf v1.32.0 gopkg.in/yaml.v2 v2.4.0 ) @@ -85,6 +87,7 @@ require ( cloud.google.com/go/longrunning v0.5.4 // indirect cloud.google.com/go/storage v1.36.0 // indirect github.com/ClickHouse/ch-go v0.58.2 // indirect + github.com/KyleBanks/depth v1.2.1 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect github.com/alessio/shellescape v1.4.1 // indirect @@ -132,6 +135,9 @@ require ( github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.3.0 // indirect + github.com/go-openapi/jsonpointer v0.20.2 // indirect + github.com/go-openapi/jsonreference v0.20.4 // indirect + github.com/go-openapi/swag v0.22.9 // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/goccy/go-yaml v1.9.5 // indirect github.com/gogo/protobuf v1.3.2 // indirect @@ -168,6 +174,7 @@ require ( github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect github.com/jackc/puddle/v2 v2.2.1 // indirect github.com/jbenet/goprocess v0.1.4 // indirect + github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/cpuid/v2 v2.2.7 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect @@ -233,8 +240,8 @@ require ( golang.org/x/net v0.21.0 // indirect golang.org/x/oauth2 v0.17.0 // indirect golang.org/x/sys v0.17.0 // indirect - golang.org/x/time v0.5.0 // indirect golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect + google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto v0.0.0-20240125205218-1f4bbc51befe // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240125205218-1f4bbc51befe // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240205150955-31a09d347014 // indirect @@ -243,6 +250,7 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.2.1 // indirect rsc.io/tmplfunc v0.0.3 // indirect + sigs.k8s.io/yaml v1.3.0 // indirect ) replace github.com/wealdtech/go-merkletree v1.0.1-0.20190605192610-2bb163c2ea2a => github.com/rocket-pool/go-merkletree v1.0.1-0.20220406020931-c262d9b976dd diff --git a/backend/go.sum b/backend/go.sum index 97c31bc89..81aa3de99 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -36,6 +36,8 @@ github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/Gurpartap/storekit-go v0.0.0-20201205024111-36b6cd5c6a21 h1:HcdvlzaQ4CJfH7xbfJZ3ZHN//BTEpId46iKEMuP3wHE= github.com/Gurpartap/storekit-go v0.0.0-20201205024111-36b6cd5c6a21/go.mod h1:7PODFS++oNZ6khojmPBvkrDeFO/hrc3jmvWvQAOXorw= +github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= +github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= @@ -271,6 +273,14 @@ github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= +github.com/go-openapi/jsonpointer v0.20.2 h1:mQc3nmndL8ZBzStEo3JYF8wzmeWffDH4VbXz58sAx6Q= +github.com/go-openapi/jsonpointer v0.20.2/go.mod h1:bHen+N0u1KEO3YlmqOjTT9Adn1RfD91Ar825/PuiRVs= +github.com/go-openapi/jsonreference v0.20.4 h1:bKlDxQxQJgwpUSgOENiMPzCTBVuc7vTdXSSgNeAhojU= +github.com/go-openapi/jsonreference v0.20.4/go.mod h1:5pZJyJP2MnYCpoeoMAql78cCHauHj0V9Lhc506VOpw4= +github.com/go-openapi/spec v0.20.14 h1:7CBlRnw+mtjFGlPDRZmAMnq35cRzI91xj03HVyUi/Do= +github.com/go-openapi/spec v0.20.14/go.mod h1:8EOhTpBoFiask8rrgwbLC3zmJfz4zsCUueRuPM6GNkw= +github.com/go-openapi/swag v0.22.9 h1:XX2DssF+mQKM2DHsbgZK74y/zj4mo9I99+89xUmuZCE= +github.com/go-openapi/swag v0.22.9/go.mod h1:3/OXnFfnMAwBD099SwYRk7GD3xOrr1iL7d/XNLXVVwE= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU= @@ -540,6 +550,7 @@ github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8 github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak= github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4= github.com/jonboulle/clockwork v0.4.0/go.mod h1:xgRqUGwRcjKCO1vbZUEtSLrqKoPSsUpK7fnezOII0kc= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -854,6 +865,8 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/supranational/blst v0.3.11 h1:LyU6FolezeWAhvQk0k6O/d49jqgO52MSDDfYgbeoEm4= github.com/supranational/blst v0.3.11/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= +github.com/swaggo/swag v1.16.3 h1:PnCYjPCah8FK4I26l2F/KQ4yz3sILcVUN3cTlBFA9Pg= +github.com/swaggo/swag v1.16.3/go.mod h1:DImHIuOFXKpMFAQjcC7FG4m3Dg4+QuUgUzJmKjI/gRk= github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d h1:vfofYNRScrDdvS342BElfbETmL1Aiz3i2t0zfRj16Hs= github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d/go.mod h1:RRCYJbIwD5jmqPI9XoAFR0OcDxqUctll6zUj/+B4S48= github.com/thomaso-mirodin/intmath v0.0.0-20160323211736-5dc6d854e46e h1:cR8/SYRgyQCt5cNCMniB/ZScMkhI9nk8U5C7SbISXjo= @@ -1227,5 +1240,5 @@ rsc.io/tmplfunc v0.0.3 h1:53XFQh69AfOa8Tw0Jm7t+GV7KZhOi6jzsCzTtKbMvzU= rsc.io/tmplfunc v0.0.3/go.mod h1:AG3sTPzElb1Io3Yg4voV9AGZJuleGAwaVRxL9M49PhA= sigs.k8s.io/structured-merge-diff/v3 v3.0.0 h1:dOmIZBMfhcHS09XZkMyUgkq5trg3/jRyJYFZUiaOp8E= sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw= -sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= -sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/backend/pkg/api/data_access/data_access.go b/backend/pkg/api/data_access/data_access.go index f88fbe63d..b20bb88c8 100644 --- a/backend/pkg/api/data_access/data_access.go +++ b/backend/pkg/api/data_access/data_access.go @@ -28,6 +28,7 @@ type DataAccessor interface { AdminRepository BlockRepository ProtocolRepository + RatelimitRepository Close() diff --git a/backend/pkg/api/data_access/dummy.go b/backend/pkg/api/data_access/dummy.go index 0a0dfe334..f955aa9a8 100644 --- a/backend/pkg/api/data_access/dummy.go +++ b/backend/pkg/api/data_access/dummy.go @@ -847,3 +847,9 @@ func (d *DummyService) GetRocketPoolOverview(ctx context.Context) (*t.RocketPool err := commonFakeData(&r) return &r, err } + +func (d *DummyService) GetApiWeights(ctx context.Context) ([]t.ApiWeightItem, error) { + r := []t.ApiWeightItem{} + err := commonFakeData(&r) + return r, err +} diff --git a/backend/pkg/api/data_access/ratelimit.go b/backend/pkg/api/data_access/ratelimit.go new file mode 100644 index 000000000..973dbed1e --- /dev/null +++ b/backend/pkg/api/data_access/ratelimit.go @@ -0,0 +1,22 @@ +package dataaccess + +import ( + "context" + + "github.com/gobitfly/beaconchain/pkg/api/types" +) + +type RatelimitRepository interface { + GetApiWeights(ctx context.Context) ([]types.ApiWeightItem, error) + // TODO @patrick: move queries from commons/ratelimit/ratelimit.go to here +} + +func (d *DataAccessService) GetApiWeights(ctx context.Context) ([]types.ApiWeightItem, error) { + var result []types.ApiWeightItem + err := d.readerDb.SelectContext(ctx, &result, ` + SELECT bucket, endpoint, method, weight + FROM api_weights + WHERE valid_from <= NOW() + `) + return result, err +} diff --git a/backend/pkg/api/handlers/public.go b/backend/pkg/api/handlers/public.go index 38d4a6612..e8ae9bda2 100644 --- a/backend/pkg/api/handlers/public.go +++ b/backend/pkg/api/handlers/public.go @@ -14,6 +14,25 @@ import ( // Public handlers may only be authenticated by an API key // Public handlers must never call internal handlers +// @title beaconcha.in API +// @version 2.0 +// @description To authenticate your API request beaconcha.in uses API Keys. Set your API Key either by: +// @description - Setting the `Authorization` header in the following format: `Authorization: Bearer `. (recommended) +// @description - Setting the URL query parameter in the following format: `api_key={your_api_key}`.\ +// @description Example: `https://beaconcha.in/api/v2/example?field=value&api_key={your_api_key}` + +// @host beaconcha.in +// @BasePath /api/v2 + +// @securitydefinitions.apikey ApiKeyInHeader +// @in header +// @name Authorization +// @description Use your API key as a Bearer token, e.g. `Bearer ` + +// @securitydefinitions.apikey ApiKeyInQuery +// @in query +// @name api_key + func (h *HandlerService) PublicGetHealthz(w http.ResponseWriter, r *http.Request) { returnOk(w, nil) } @@ -95,8 +114,65 @@ func (h *HandlerService) PublicPutAccountDashboardTransactionsSettings(w http.Re returnOk(w, nil) } +// PublicPostValidatorDashboards godoc +// +// @Description Create a new validator dashboard. **Note**: New dashboards will automatically have a default group created. +// @Security ApiKeyInHeader || ApiKeyInQuery +// @Tags Validator Dashboards +// @Accept json +// @Produce json +// @Param request body handlers.PublicPostValidatorDashboards.request true "`name`: Specify the name of the dashboard.
`network`: Specify the network for the dashboard. Possible options are:" +// @Failure 400 {object} types.ApiErrorResponse +// @Failure 409 {object} types.ApiErrorResponse "Conflict. The request could not be performed by the server because the authenticated user has already reached their dashboard limit." +// @Router /validator-dashboards [post] func (h *HandlerService) PublicPostValidatorDashboards(w http.ResponseWriter, r *http.Request) { - returnCreated(w, nil) + var v validationError + userId, err := GetUserIdByContext(r) + if err != nil { + handleErr(w, err) + return + } + + type request struct { + Name string `json:"name"` + Network intOrString `json:"network" swaggertype:"string" enums:"ethereum,gnosis"` + } + req := request{} + if err := v.checkBody(&req, r); err != nil { + handleErr(w, err) + return + } + name := v.checkNameNotEmpty(req.Name) + chainId := v.checkNetwork(req.Network) + if v.hasErrors() { + handleErr(w, v) + return + } + + userInfo, err := h.dai.GetUserInfo(r.Context(), userId) + if err != nil { + handleErr(w, err) + return + } + dashboardCount, err := h.dai.GetUserValidatorDashboardCount(r.Context(), userId, true) + if err != nil { + handleErr(w, err) + return + } + if dashboardCount >= userInfo.PremiumPerks.ValidatorDasboards && !isUserAdmin(userInfo) { + returnConflict(w, errors.New("maximum number of validator dashboards reached")) + return + } + + data, err := h.dai.CreateValidatorDashboard(r.Context(), userId, name, chainId) + if err != nil { + handleErr(w, err) + return + } + response := types.PostValidatorDashboardsResponse{ + Data: *data, + } + returnCreated(w, response) } func (h *HandlerService) PublicGetValidatorDashboard(w http.ResponseWriter, r *http.Request) { diff --git a/backend/pkg/api/types/data_access.go b/backend/pkg/api/types/data_access.go index 9f4aea5d2..abb838f05 100644 --- a/backend/pkg/api/types/data_access.go +++ b/backend/pkg/api/types/data_access.go @@ -210,3 +210,13 @@ type VDBValidatorSummaryChartRow struct { SyncExecuted float64 `db:"sync_executed"` SyncScheduled float64 `db:"sync_scheduled"` } + +// ------------------------- +// ratelimiting + +type ApiWeightItem struct { + Bucket string `db:"bucket"` + Endpoint string `db:"endpoint"` + Method string `db:"method"` + Weight int `db:"weight"` +} diff --git a/backend/pkg/api/types/validator_dashboard.go b/backend/pkg/api/types/validator_dashboard.go index 5e50cf50a..5b906abbe 100644 --- a/backend/pkg/api/types/validator_dashboard.go +++ b/backend/pkg/api/types/validator_dashboard.go @@ -367,6 +367,7 @@ type VDBPostReturnData struct { Network uint64 `db:"network" json:"network"` CreatedAt int64 `db:"created_at" json:"created_at"` } +type PostValidatorDashboardsResponse ApiDataResponse[VDBPostReturnData] type VDBPostCreateGroupData struct { Id uint64 `db:"id" json:"id"` diff --git a/backend/pkg/commons/ratelimit/ratelimit.go b/backend/pkg/commons/ratelimit/ratelimit.go index b627b2601..4dc6b9818 100644 --- a/backend/pkg/commons/ratelimit/ratelimit.go +++ b/backend/pkg/commons/ratelimit/ratelimit.go @@ -54,7 +54,7 @@ const ( FallbackRateLimitSecond = 20 // RateLimit per second for when redis is offline FallbackRateLimitBurst = 20 // RateLimit burst for when redis is offline - defaultBucket = "default" // if no bucket is set for a route, use this one + DefaultBucket = "default" // if no bucket is set for a route, use this one statsTruncateDuration = time.Hour * 1 // ratelimit-stats are truncated to this duration ) @@ -364,7 +364,7 @@ func updateWeights(firstRun bool) error { } buckets[w.Endpoint] = strings.ReplaceAll(w.Bucket, ":", "_") if buckets[w.Endpoint] == "" { - buckets[w.Endpoint] = defaultBucket + buckets[w.Endpoint] = DefaultBucket } if !firstRun && oldBuckets[w.Endpoint] != buckets[w.Endpoint] { log.InfoWithFields(log.Fields{"endpoint": w.Endpoint, "bucket": w.Weight, "oldBucket": oldBuckets[w.Endpoint]}, "bucket changed") @@ -934,7 +934,7 @@ func getWeight(r *http.Request) (cost int64, identifier, bucket string) { weight = 1 } if !bucketOk { - bucket = defaultBucket + bucket = DefaultBucket } return weight, route, bucket } From e3be3c2de13c00a68ad618ffb31dd05987f21e39 Mon Sep 17 00:00:00 2001 From: LUCCA DUKIC <109136188+LuccaBitfly@users.noreply.github.com> Date: Wed, 21 Aug 2024 09:40:13 +0200 Subject: [PATCH 02/16] (BEDS-295) add option to not insert weights --- backend/cmd/api_docs/main.go | 16 ++++++++++------ backend/pkg/api/data_access/data_access.go | 11 +++++------ backend/pkg/api/handlers/public.go | 1 + backend/pkg/commons/utils/config.go | 1 + 4 files changed, 17 insertions(+), 12 deletions(-) diff --git a/backend/cmd/api_docs/main.go b/backend/cmd/api_docs/main.go index 6d2097288..d77ad6fbf 100644 --- a/backend/cmd/api_docs/main.go +++ b/backend/cmd/api_docs/main.go @@ -12,7 +12,6 @@ import ( dataaccess "github.com/gobitfly/beaconchain/pkg/api/data_access" types "github.com/gobitfly/beaconchain/pkg/api/types" "github.com/gobitfly/beaconchain/pkg/commons/log" - "github.com/gobitfly/beaconchain/pkg/commons/ratelimit" commonTypes "github.com/gobitfly/beaconchain/pkg/commons/types" "github.com/gobitfly/beaconchain/pkg/commons/utils" "github.com/gobitfly/beaconchain/pkg/commons/version" @@ -50,7 +49,12 @@ func main() { log.Fatal(err, "error generating swagger docs", 0) } - log.Info("swagger docs generated successfully, now loading endpoint weights from db", 0) + log.Info("\n-------------\nswagger docs generated successfully, now loading endpoint weights from db\n-------------") + + if *configPath == "" { + log.Warn("no config file provided, weights will not be inserted into swagger docs", 0) + os.Exit(0) + } // load endpoint weights from db cfg := &commonTypes.Config{} @@ -82,7 +86,7 @@ func main() { log.Fatal(err, "error writing new swagger docs", 0) } - log.Info("api weights inserted successfully", 0) + log.Info("\n-------------\napi weights inserted successfully\n-------------") } func insertApiWeights(data []byte, apiWeightItems []types.ApiWeightItem) ([]byte, error) { @@ -100,7 +104,7 @@ func insertApiWeights(data []byte, apiWeightItems []types.ApiWeightItem) ([]byte } // get weight and bucket for each endpoint weight := 1 - bucket := ratelimit.DefaultBucket + bucket := "" for _, apiWeightItem := range apiWeightItems { // ignore endpoints that don't belong to v2 if !strings.HasPrefix(apiWeightItem.Endpoint, apiPrefix) { @@ -109,7 +113,7 @@ func insertApiWeights(data []byte, apiWeightItems []types.ApiWeightItem) ([]byte // compare endpoint and method if pathString == strings.TrimPrefix(apiWeightItem.Endpoint, apiPrefix) && methodString == apiWeightItem.Method { weight = apiWeightItem.Weight - bucket = apiWeightItem.Bucket + bucket = apiWeightItem.Bucket + " " break } } @@ -118,7 +122,7 @@ func insertApiWeights(data []byte, apiWeightItems []types.ApiWeightItem) ([]byte if weight > 1 { plural = "s" } - operation.Summary = fmt.Sprintf("(%d %s credit%s) %s", weight, bucket, plural, operation.Summary) + operation.Summary = fmt.Sprintf("(%d %scredit%s) %s", weight, bucket, plural, operation.Summary) } } diff --git a/backend/pkg/api/data_access/data_access.go b/backend/pkg/api/data_access/data_access.go index b20bb88c8..229a2ddd2 100644 --- a/backend/pkg/api/data_access/data_access.go +++ b/backend/pkg/api/data_access/data_access.go @@ -13,7 +13,6 @@ import ( "github.com/gobitfly/beaconchain/pkg/commons/db" "github.com/gobitfly/beaconchain/pkg/commons/log" "github.com/gobitfly/beaconchain/pkg/commons/types" - "github.com/gobitfly/beaconchain/pkg/commons/utils" "github.com/jmoiron/sqlx" "github.com/pkg/errors" ) @@ -207,7 +206,7 @@ func createDataAccessService(cfg *types.Config) *DataAccessService { wg.Add(1) go func() { defer wg.Done() - bt, err := db.InitBigtable(utils.Config.Bigtable.Project, utils.Config.Bigtable.Instance, fmt.Sprintf("%d", utils.Config.Chain.ClConfig.DepositChainID), utils.Config.RedisCacheEndpoint) + bt, err := db.InitBigtable(cfg.Bigtable.Project, cfg.Bigtable.Instance, fmt.Sprintf("%d", cfg.Chain.ClConfig.DepositChainID), cfg.RedisCacheEndpoint) if err != nil { log.Fatal(err, "error connecting to bigtable", 0) } @@ -215,11 +214,11 @@ func createDataAccessService(cfg *types.Config) *DataAccessService { }() // Initialize the tiered cache (redis) - if utils.Config.TieredCacheProvider == "redis" || len(utils.Config.RedisCacheEndpoint) != 0 { + if cfg.TieredCacheProvider == "redis" || len(cfg.RedisCacheEndpoint) != 0 { wg.Add(1) go func() { defer wg.Done() - cache.MustInitTieredCache(utils.Config.RedisCacheEndpoint) + cache.MustInitTieredCache(cfg.RedisCacheEndpoint) log.Infof("tiered Cache initialized, latest finalized epoch: %v", cache.LatestFinalizedEpoch.Get()) }() } @@ -229,7 +228,7 @@ func createDataAccessService(cfg *types.Config) *DataAccessService { go func() { defer wg.Done() rdc := redis.NewClient(&redis.Options{ - Addr: utils.Config.RedisSessionStoreEndpoint, + Addr: cfg.RedisSessionStoreEndpoint, ReadTimeout: time.Second * 60, }) @@ -241,7 +240,7 @@ func createDataAccessService(cfg *types.Config) *DataAccessService { wg.Wait() - if utils.Config.TieredCacheProvider != "redis" { + if cfg.TieredCacheProvider != "redis" { log.Fatal(fmt.Errorf("no cache provider set, please set TierdCacheProvider (example redis)"), "", 0) } diff --git a/backend/pkg/api/handlers/public.go b/backend/pkg/api/handlers/public.go index e8ae9bda2..51b2dc631 100644 --- a/backend/pkg/api/handlers/public.go +++ b/backend/pkg/api/handlers/public.go @@ -122,6 +122,7 @@ func (h *HandlerService) PublicPutAccountDashboardTransactionsSettings(w http.Re // @Accept json // @Produce json // @Param request body handlers.PublicPostValidatorDashboards.request true "`name`: Specify the name of the dashboard.
`network`: Specify the network for the dashboard. Possible options are:" +// @Success 201 {object} types.PostValidatorDashboardsResponse // @Failure 400 {object} types.ApiErrorResponse // @Failure 409 {object} types.ApiErrorResponse "Conflict. The request could not be performed by the server because the authenticated user has already reached their dashboard limit." // @Router /validator-dashboards [post] diff --git a/backend/pkg/commons/utils/config.go b/backend/pkg/commons/utils/config.go index 97f7a0167..8e3484354 100644 --- a/backend/pkg/commons/utils/config.go +++ b/backend/pkg/commons/utils/config.go @@ -251,6 +251,7 @@ func ReadConfig(cfg *types.Config, path string) error { "mainCurrency": cfg.Frontend.MainCurrency, }, "did init config") + Config = cfg return nil } From 86b330bbd25c6e1b3757b7cee4e270c283e29dc0 Mon Sep 17 00:00:00 2001 From: LUCCA DUKIC <109136188+LuccaBitfly@users.noreply.github.com> Date: Wed, 21 Aug 2024 09:46:53 +0200 Subject: [PATCH 03/16] (BEDS-295) add comments --- backend/cmd/api_docs/main.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/backend/cmd/api_docs/main.go b/backend/cmd/api_docs/main.go index d77ad6fbf..7d9c5d095 100644 --- a/backend/cmd/api_docs/main.go +++ b/backend/cmd/api_docs/main.go @@ -27,6 +27,11 @@ const ( apiPrefix = "/api/v2" ) +// Expects the following flags: +// --config: (optional) Path to the config file to add endpoint weights to the swagger docs + +// Standard usage (execute in backend folder): go run cmd/api_docs/main.go --config + func main() { configPath := flag.String("config", "", "Path to the config file, if empty string defaults will be used") versionFlag := flag.Bool("version", false, "Show version and exit") From 16c916c0a69c3f2391d3f2a02d554ae98a5917cf Mon Sep 17 00:00:00 2001 From: LUCCA DUKIC <109136188+LuccaBitfly@users.noreply.github.com> Date: Wed, 28 Aug 2024 09:27:42 +0200 Subject: [PATCH 04/16] (BEDS-295) lint, small adjustments --- backend/cmd/api_docs/main.go | 46 +++++++++++----------- backend/cmd/typescript_converter/main.go | 2 +- backend/pkg/commons/ratelimit/ratelimit.go | 3 +- 3 files changed, 26 insertions(+), 25 deletions(-) diff --git a/backend/cmd/api_docs/main.go b/backend/cmd/api_docs/main.go index 72078f7b1..425f1ae3d 100644 --- a/backend/cmd/api_docs/main.go +++ b/backend/cmd/api_docs/main.go @@ -6,12 +6,14 @@ import ( "flag" "fmt" "os" - "strings" + "slices" + "time" "github.com/go-openapi/spec" dataaccess "github.com/gobitfly/beaconchain/pkg/api/data_access" types "github.com/gobitfly/beaconchain/pkg/api/types" "github.com/gobitfly/beaconchain/pkg/commons/log" + "github.com/gobitfly/beaconchain/pkg/commons/ratelimit" commonTypes "github.com/gobitfly/beaconchain/pkg/commons/types" "github.com/gobitfly/beaconchain/pkg/commons/utils" "github.com/gobitfly/beaconchain/pkg/commons/version" @@ -22,15 +24,15 @@ const ( searchDir = "./pkg/api" mainAPI = "handlers/public.go" outputDir = "./docs" - outputType = "json" // can also be yaml + outputType = "json" // can also be "yaml" - apiPrefix = "/api/v2" + apiPrefix = "/api/v2/" ) // Expects the following flags: // --config: (optional) Path to the config file to add endpoint weights to the swagger docs -// Standard usage (execute in backend folder): go run cmd/api_docs/main.go --config +// Standard usage (execute in backend folder): go run cmd/main.go api-docs --config func Run() { fs := flag.NewFlagSet("fs", flag.ExitOnError) @@ -40,8 +42,8 @@ func Run() { _ = fs.Parse(os.Args[2:]) if *versionFlag { - log.Infof(version.Version) - log.Infof(version.GoVersion) + log.Info(version.Version) + log.Info(version.GoVersion) return } // generate swagger doc @@ -72,7 +74,9 @@ func Run() { da := dataaccess.NewDataAccessService(cfg) defer da.Close() - apiWeights, err := da.GetApiWeights(context.Background()) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + apiWeights, err := da.GetApiWeights(ctx) if err != nil { log.Fatal(err, "error loading endpoint weights from db", 0) } @@ -82,7 +86,7 @@ func Run() { if err != nil { log.Fatal(err, "error reading swagger docs", 0) } - newData, err := insertApiWeights(data, apiWeights) + newData, err := getDataWithWeights(data, apiWeights) if err != nil { log.Fatal(err, "error inserting api weights", 0) } @@ -96,7 +100,7 @@ func Run() { log.Info("\n-------------\napi weights inserted successfully\n-------------") } -func insertApiWeights(data []byte, apiWeightItems []types.ApiWeightItem) ([]byte, error) { +func getDataWithWeights(data []byte, apiWeightItems []types.ApiWeightItem) ([]byte, error) { // unmarshal swagger file var swagger *spec.Swagger if err := json.Unmarshal(data, &swagger); err != nil { @@ -105,31 +109,27 @@ func insertApiWeights(data []byte, apiWeightItems []types.ApiWeightItem) ([]byte // iterate endpoints from swagger file for pathString, pathItem := range swagger.Paths.Paths { + pathString = apiPrefix + pathString for methodString, operation := range getPathItemOperationMap(pathItem) { if operation == nil { continue } // get weight and bucket for each endpoint - weight := 1 - bucket := "" - for _, apiWeightItem := range apiWeightItems { - // ignore endpoints that don't belong to v2 - if !strings.HasPrefix(apiWeightItem.Endpoint, apiPrefix) { - continue - } - // compare endpoint and method - if pathString == strings.TrimPrefix(apiWeightItem.Endpoint, apiPrefix) && methodString == apiWeightItem.Method { - weight = apiWeightItem.Weight - bucket = apiWeightItem.Bucket + " " - break - } + weight := ratelimit.DefaultWeight + bucket := ratelimit.DefaultBucket + index := slices.IndexFunc(apiWeightItems, func(item types.ApiWeightItem) bool { + return pathString == item.Endpoint && methodString == item.Method + }) + if index != -1 { + weight = apiWeightItems[index].Weight + bucket = apiWeightItems[index].Bucket } // insert weight and bucket into endpoint summary plural := "" if weight > 1 { plural = "s" } - operation.Summary = fmt.Sprintf("(%d %scredit%s) %s", weight, bucket, plural, operation.Summary) + operation.Summary = fmt.Sprintf("(%d %s credit%s) %s", weight, bucket, plural, operation.Summary) } } diff --git a/backend/cmd/typescript_converter/main.go b/backend/cmd/typescript_converter/main.go index 2a3ce86f5..6d5f9afee 100644 --- a/backend/cmd/typescript_converter/main.go +++ b/backend/cmd/typescript_converter/main.go @@ -31,7 +31,7 @@ var typeMappings = map[string]string{ // Expects the following flags: // -out: Output folder for the generated TypeScript file -// Standard usage (execute in backend folder): go run cmd/typescript_converter/main.go -out ../frontend/types/api +// Standard usage (execute in backend folder): go run cmd/main.go typescript-converter -out ../frontend/types/api func Run() { var out string diff --git a/backend/pkg/commons/ratelimit/ratelimit.go b/backend/pkg/commons/ratelimit/ratelimit.go index 749abda50..3762a9b56 100644 --- a/backend/pkg/commons/ratelimit/ratelimit.go +++ b/backend/pkg/commons/ratelimit/ratelimit.go @@ -56,6 +56,7 @@ const ( FallbackRateLimitSecond = 20 // RateLimit per second for when redis is offline FallbackRateLimitBurst = 20 // RateLimit burst for when redis is offline + DefaultWeight = 1 // if no bucket is set for a route, use this one DefaultBucket = "default" // if no bucket is set for a route, use this one statsTruncateDuration = time.Hour * 1 // ratelimit-stats are truncated to this duration @@ -943,7 +944,7 @@ func getWeight(r *http.Request) (cost int64, identifier, bucket string) { bucket, bucketOk := buckets[route] weightsMu.RUnlock() if !weightOk { - weight = 1 + weight = DefaultWeight } if !bucketOk { bucket = DefaultBucket From 26cae11da777d900686c80a7060c268a13421359 Mon Sep 17 00:00:00 2001 From: LUCCA DUKIC <109136188+LuccaBitfly@users.noreply.github.com> Date: Wed, 28 Aug 2024 09:29:48 +0200 Subject: [PATCH 05/16] (BEDS-295) fix comment --- backend/pkg/commons/ratelimit/ratelimit.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/pkg/commons/ratelimit/ratelimit.go b/backend/pkg/commons/ratelimit/ratelimit.go index 3762a9b56..610fc5342 100644 --- a/backend/pkg/commons/ratelimit/ratelimit.go +++ b/backend/pkg/commons/ratelimit/ratelimit.go @@ -56,7 +56,7 @@ const ( FallbackRateLimitSecond = 20 // RateLimit per second for when redis is offline FallbackRateLimitBurst = 20 // RateLimit burst for when redis is offline - DefaultWeight = 1 // if no bucket is set for a route, use this one + DefaultWeight = 1 // if no weight is set for a route, use this one DefaultBucket = "default" // if no bucket is set for a route, use this one statsTruncateDuration = time.Hour * 1 // ratelimit-stats are truncated to this duration From 21def46484fb1081a0a430c865ac2f41ec263690 Mon Sep 17 00:00:00 2001 From: LUCCA DUKIC <109136188+LuccaBitfly@users.noreply.github.com> Date: Wed, 28 Aug 2024 09:44:12 +0200 Subject: [PATCH 06/16] (BEDS-295) fix bug --- backend/pkg/api/handlers/public.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/pkg/api/handlers/public.go b/backend/pkg/api/handlers/public.go index 2e938dd7e..af5980c46 100644 --- a/backend/pkg/api/handlers/public.go +++ b/backend/pkg/api/handlers/public.go @@ -177,7 +177,7 @@ func (h *HandlerService) PublicPostValidatorDashboards(w http.ResponseWriter, r return } if dashboardCount >= userInfo.PremiumPerks.ValidatorDasboards && !isUserAdmin(userInfo) { - returnConflict(w, errors.New("maximum number of validator dashboards reached")) + returnConflict(w, r, errors.New("maximum number of validator dashboards reached")) return } From 5ca508a6534afa7a794dcb7a09ad513270527b09 Mon Sep 17 00:00:00 2001 From: LUCCA DUKIC <109136188+LuccaBitfly@users.noreply.github.com> Date: Wed, 28 Aug 2024 10:16:28 +0200 Subject: [PATCH 07/16] (BEDS-295) defeat linter --- backend/cmd/api_docs/main.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/backend/cmd/api_docs/main.go b/backend/cmd/api_docs/main.go index 425f1ae3d..55f84faa9 100644 --- a/backend/cmd/api_docs/main.go +++ b/backend/cmd/api_docs/main.go @@ -25,8 +25,7 @@ const ( mainAPI = "handlers/public.go" outputDir = "./docs" outputType = "json" // can also be "yaml" - - apiPrefix = "/api/v2/" + apiPrefix = "/api/v2/" ) // Expects the following flags: @@ -92,7 +91,7 @@ func Run() { } // write updated swagger doc - err = os.WriteFile(outputDir+"/swagger."+outputType, newData, os.ModePerm) + err = os.WriteFile(outputDir+"/swagger."+outputType, newData, 0600) if err != nil { log.Fatal(err, "error writing new swagger docs", 0) } From e77f030bfb8c6499415193887201c2bc935596d1 Mon Sep 17 00:00:00 2001 From: LUCCA DUKIC <109136188+LuccaBitfly@users.noreply.github.com> Date: Mon, 9 Sep 2024 13:17:33 +0200 Subject: [PATCH 08/16] (BEDS-295) remove api doc main file --- backend/cmd/api_docs/main.go | 148 ----------------------------------- backend/go.mod | 3 - backend/go.sum | 13 --- 3 files changed, 164 deletions(-) delete mode 100644 backend/cmd/api_docs/main.go diff --git a/backend/cmd/api_docs/main.go b/backend/cmd/api_docs/main.go deleted file mode 100644 index 55f84faa9..000000000 --- a/backend/cmd/api_docs/main.go +++ /dev/null @@ -1,148 +0,0 @@ -package api_docs - -import ( - "context" - "encoding/json" - "flag" - "fmt" - "os" - "slices" - "time" - - "github.com/go-openapi/spec" - dataaccess "github.com/gobitfly/beaconchain/pkg/api/data_access" - types "github.com/gobitfly/beaconchain/pkg/api/types" - "github.com/gobitfly/beaconchain/pkg/commons/log" - "github.com/gobitfly/beaconchain/pkg/commons/ratelimit" - commonTypes "github.com/gobitfly/beaconchain/pkg/commons/types" - "github.com/gobitfly/beaconchain/pkg/commons/utils" - "github.com/gobitfly/beaconchain/pkg/commons/version" - "github.com/swaggo/swag/gen" -) - -const ( - searchDir = "./pkg/api" - mainAPI = "handlers/public.go" - outputDir = "./docs" - outputType = "json" // can also be "yaml" - apiPrefix = "/api/v2/" -) - -// Expects the following flags: -// --config: (optional) Path to the config file to add endpoint weights to the swagger docs - -// Standard usage (execute in backend folder): go run cmd/main.go api-docs --config - -func Run() { - fs := flag.NewFlagSet("fs", flag.ExitOnError) - - configPath := fs.String("config", "", "Path to the config file, if empty string defaults will be used") - versionFlag := fs.Bool("version", false, "Show version and exit") - _ = fs.Parse(os.Args[2:]) - - if *versionFlag { - log.Info(version.Version) - log.Info(version.GoVersion) - return - } - // generate swagger doc - config := &gen.Config{ - SearchDir: searchDir, - MainAPIFile: mainAPI, - OutputDir: outputDir, - OutputTypes: []string{outputType}, - } - err := gen.New().Build(config) - if err != nil { - log.Fatal(err, "error generating swagger docs", 0) - } - - log.Info("\n-------------\nswagger docs generated successfully, now loading endpoint weights from db\n-------------") - - if *configPath == "" { - log.Warn("no config file provided, weights will not be inserted into swagger docs", 0) - os.Exit(0) - } - - // load endpoint weights from db - cfg := &commonTypes.Config{} - err = utils.ReadConfig(cfg, *configPath) - if err != nil { - log.Fatal(err, "error reading config file", 0) - } - - da := dataaccess.NewDataAccessService(cfg) - defer da.Close() - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - apiWeights, err := da.GetApiWeights(ctx) - if err != nil { - log.Fatal(err, "error loading endpoint weights from db", 0) - } - - // insert endpoint weights into swagger doc - data, err := os.ReadFile(outputDir + "/swagger." + outputType) - if err != nil { - log.Fatal(err, "error reading swagger docs", 0) - } - newData, err := getDataWithWeights(data, apiWeights) - if err != nil { - log.Fatal(err, "error inserting api weights", 0) - } - - // write updated swagger doc - err = os.WriteFile(outputDir+"/swagger."+outputType, newData, 0600) - if err != nil { - log.Fatal(err, "error writing new swagger docs", 0) - } - - log.Info("\n-------------\napi weights inserted successfully\n-------------") -} - -func getDataWithWeights(data []byte, apiWeightItems []types.ApiWeightItem) ([]byte, error) { - // unmarshal swagger file - var swagger *spec.Swagger - if err := json.Unmarshal(data, &swagger); err != nil { - return nil, fmt.Errorf("error unmarshalling swagger docs: %w", err) - } - - // iterate endpoints from swagger file - for pathString, pathItem := range swagger.Paths.Paths { - pathString = apiPrefix + pathString - for methodString, operation := range getPathItemOperationMap(pathItem) { - if operation == nil { - continue - } - // get weight and bucket for each endpoint - weight := ratelimit.DefaultWeight - bucket := ratelimit.DefaultBucket - index := slices.IndexFunc(apiWeightItems, func(item types.ApiWeightItem) bool { - return pathString == item.Endpoint && methodString == item.Method - }) - if index != -1 { - weight = apiWeightItems[index].Weight - bucket = apiWeightItems[index].Bucket - } - // insert weight and bucket into endpoint summary - plural := "" - if weight > 1 { - plural = "s" - } - operation.Summary = fmt.Sprintf("(%d %s credit%s) %s", weight, bucket, plural, operation.Summary) - } - } - - return json.MarshalIndent(swagger, "", " ") -} - -func getPathItemOperationMap(item spec.PathItem) map[string]*spec.Operation { - return map[string]*spec.Operation{ - "get": item.Get, - "put": item.Put, - "post": item.Post, - "delete": item.Delete, - "options": item.Options, - "head": item.Head, - "patch": item.Patch, - } -} diff --git a/backend/go.mod b/backend/go.mod index 864de8794..58a24bf21 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -24,7 +24,6 @@ require ( github.com/fergusstrange/embedded-postgres v1.29.0 github.com/gavv/httpexpect/v2 v2.16.0 github.com/go-faker/faker/v4 v4.3.0 - github.com/go-openapi/spec v0.20.14 github.com/go-redis/redis/v8 v8.11.5 github.com/gobitfly/eth-rewards v0.1.2-0.20230403064929-411ddc40a5f7 github.com/gobitfly/eth.store v0.0.0-20240312111708-b43f13990280 @@ -90,7 +89,6 @@ require ( cloud.google.com/go/longrunning v0.5.4 // indirect cloud.google.com/go/storage v1.36.0 // indirect github.com/ClickHouse/ch-go v0.58.2 // indirect - github.com/KyleBanks/depth v1.2.1 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/TylerBrock/colorjson v0.0.0-20200706003622-8a50f05110d2 // indirect github.com/ajg/form v1.5.1 // indirect @@ -180,7 +178,6 @@ require ( github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect github.com/jackc/puddle/v2 v2.2.1 // indirect github.com/jbenet/goprocess v0.1.4 // indirect - github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/cpuid/v2 v2.2.7 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect diff --git a/backend/go.sum b/backend/go.sum index 1c3112ecc..fa1e4f202 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -36,8 +36,6 @@ github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ= github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/Gurpartap/storekit-go v0.0.0-20201205024111-36b6cd5c6a21 h1:HcdvlzaQ4CJfH7xbfJZ3ZHN//BTEpId46iKEMuP3wHE= github.com/Gurpartap/storekit-go v0.0.0-20201205024111-36b6cd5c6a21/go.mod h1:7PODFS++oNZ6khojmPBvkrDeFO/hrc3jmvWvQAOXorw= -github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= -github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= @@ -287,14 +285,6 @@ github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= -github.com/go-openapi/jsonpointer v0.20.2 h1:mQc3nmndL8ZBzStEo3JYF8wzmeWffDH4VbXz58sAx6Q= -github.com/go-openapi/jsonpointer v0.20.2/go.mod h1:bHen+N0u1KEO3YlmqOjTT9Adn1RfD91Ar825/PuiRVs= -github.com/go-openapi/jsonreference v0.20.4 h1:bKlDxQxQJgwpUSgOENiMPzCTBVuc7vTdXSSgNeAhojU= -github.com/go-openapi/jsonreference v0.20.4/go.mod h1:5pZJyJP2MnYCpoeoMAql78cCHauHj0V9Lhc506VOpw4= -github.com/go-openapi/spec v0.20.14 h1:7CBlRnw+mtjFGlPDRZmAMnq35cRzI91xj03HVyUi/Do= -github.com/go-openapi/spec v0.20.14/go.mod h1:8EOhTpBoFiask8rrgwbLC3zmJfz4zsCUueRuPM6GNkw= -github.com/go-openapi/swag v0.22.9 h1:XX2DssF+mQKM2DHsbgZK74y/zj4mo9I99+89xUmuZCE= -github.com/go-openapi/swag v0.22.9/go.mod h1:3/OXnFfnMAwBD099SwYRk7GD3xOrr1iL7d/XNLXVVwE= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU= @@ -572,7 +562,6 @@ github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8 github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak= github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4= github.com/jonboulle/clockwork v0.4.0/go.mod h1:xgRqUGwRcjKCO1vbZUEtSLrqKoPSsUpK7fnezOII0kc= -github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -899,8 +888,6 @@ github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsT github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/supranational/blst v0.3.11 h1:LyU6FolezeWAhvQk0k6O/d49jqgO52MSDDfYgbeoEm4= github.com/supranational/blst v0.3.11/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= -github.com/swaggo/swag v1.16.3 h1:PnCYjPCah8FK4I26l2F/KQ4yz3sILcVUN3cTlBFA9Pg= -github.com/swaggo/swag v1.16.3/go.mod h1:DImHIuOFXKpMFAQjcC7FG4m3Dg4+QuUgUzJmKjI/gRk= github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d h1:vfofYNRScrDdvS342BElfbETmL1Aiz3i2t0zfRj16Hs= github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d/go.mod h1:RRCYJbIwD5jmqPI9XoAFR0OcDxqUctll6zUj/+B4S48= github.com/tailscale/depaware v0.0.0-20210622194025-720c4b409502/go.mod h1:p9lPsd+cx33L3H9nNoecRRxPssFKUwwI50I3pZ0yT+8= From d7ac7c2de029cf5a3fd7b19e8d6a82ec19692bbe Mon Sep 17 00:00:00 2001 From: LUCCA DUKIC <109136188+LuccaBitfly@users.noreply.github.com> Date: Tue, 10 Sep 2024 12:45:17 +0200 Subject: [PATCH 09/16] (BEDS-295) provide ratelimit weights with endpoint --- backend/Makefile | 5 +--- backend/pkg/api/handlers/common.go | 3 +-- backend/pkg/api/handlers/internal.go | 23 +++++++++++++--- backend/pkg/api/handlers/public.go | 40 ++++++++++++++-------------- backend/pkg/api/router.go | 2 ++ backend/pkg/api/types/data_access.go | 10 ------- backend/pkg/api/types/ratelimit.go | 10 +++++++ 7 files changed, 53 insertions(+), 40 deletions(-) create mode 100644 backend/pkg/api/types/ratelimit.go diff --git a/backend/Makefile b/backend/Makefile index 8597c64cb..c384ba55b 100644 --- a/backend/Makefile +++ b/backend/Makefile @@ -11,7 +11,7 @@ CGO_CFLAGS_ALLOW="-O -D__BLST_PORTABLE__" all: mkdir -p bin CGO_CFLAGS=${CGO_CFLAGS} CGO_CFLAGS_ALLOW=${CGO_CFLAGS_ALLOW} go build --ldflags=${LDFLAGS} -o ./bin/bc ./cmd/main.go - + go install github.com/swaggo/swag/cmd/swag@latest && swag init --ot json -o docs -d ./pkg/api/ -g ./handlers/public.go clean: rm -rf bin @@ -23,6 +23,3 @@ frontend-types: addhooks: git config core.hooksPath hooks - -api-docs: - go run cmd/main.go api_docs \ No newline at end of file diff --git a/backend/pkg/api/handlers/common.go b/backend/pkg/api/handlers/common.go index ecc9794db..3a895c481 100644 --- a/backend/pkg/api/handlers/common.go +++ b/backend/pkg/api/handlers/common.go @@ -528,8 +528,7 @@ func checkEnum[T enums.EnumFactory[T]](v *validationError, enumString string, na } // checkEnumIsAllowed checks if the given enum is in the list of allowed enums. -// precondition: the enum is the same type as the allowed enums. -func (v *validationError) checkEnumIsAllowed(enum enums.Enum, allowed []enums.Enum, name string) { +func checkEnumIsAllowed[T enums.EnumFactory[T]](v *validationError, enum T, allowed []T, name string) { if enums.IsInvalidEnum(enum) { v.add(name, "parameter is missing or invalid, please check the API documentation") return diff --git a/backend/pkg/api/handlers/internal.go b/backend/pkg/api/handlers/internal.go index 1c4157290..1554aaee2 100644 --- a/backend/pkg/api/handlers/internal.go +++ b/backend/pkg/api/handlers/internal.go @@ -26,6 +26,21 @@ func (h *HandlerService) InternalGetProductSummary(w http.ResponseWriter, r *htt returnOk(w, r, response) } +// -------------------------------------- +// API Ratelimit Weights + +func (h *HandlerService) InternalGetRatelimitWeights(w http.ResponseWriter, r *http.Request) { + data, err := h.dai.GetApiWeights(r.Context()) + if err != nil { + handleErr(w, r, err) + return + } + response := types.InternalGetRatelimitWeightsResponse{ + Data: data, + } + returnOk(w, r, response) +} + // -------------------------------------- // Latest State @@ -85,7 +100,7 @@ func (h *HandlerService) InternalPostAdConfigurations(w http.ResponseWriter, r * handleErr(w, r, err) return } - if user.UserGroup != "ADMIN" { + if user.UserGroup != types.UserGroupAdmin { returnForbidden(w, r, errors.New("user is not an admin")) return } @@ -131,7 +146,7 @@ func (h *HandlerService) InternalGetAdConfigurations(w http.ResponseWriter, r *h handleErr(w, r, err) return } - if user.UserGroup != "ADMIN" { + if user.UserGroup != types.UserGroupAdmin { returnForbidden(w, r, errors.New("user is not an admin")) return } @@ -161,7 +176,7 @@ func (h *HandlerService) InternalPutAdConfiguration(w http.ResponseWriter, r *ht handleErr(w, r, err) return } - if user.UserGroup != "ADMIN" { + if user.UserGroup != types.UserGroupAdmin { returnForbidden(w, r, errors.New("user is not an admin")) return } @@ -207,7 +222,7 @@ func (h *HandlerService) InternalDeleteAdConfiguration(w http.ResponseWriter, r handleErr(w, r, err) return } - if user.UserGroup != "ADMIN" { + if user.UserGroup != types.UserGroupAdmin { returnForbidden(w, r, errors.New("user is not an admin")) return } diff --git a/backend/pkg/api/handlers/public.go b/backend/pkg/api/handlers/public.go index 1e56c8b46..c0c127ce5 100644 --- a/backend/pkg/api/handlers/public.go +++ b/backend/pkg/api/handlers/public.go @@ -24,17 +24,17 @@ import ( // @description - Setting the URL query parameter in the following format: `api_key={your_api_key}`.\ // @description Example: `https://beaconcha.in/api/v2/example?field=value&api_key={your_api_key}` -// @host beaconcha.in -// @BasePath /api/v2 +// @host beaconcha.in +// @BasePath /api/v2 -// @securitydefinitions.apikey ApiKeyInHeader -// @in header -// @name Authorization -// @description Use your API key as a Bearer token, e.g. `Bearer ` +// @securitydefinitions.apikey ApiKeyInHeader +// @in header +// @name Authorization +// @description Use your API key as a Bearer token, e.g. `Bearer ` -// @securitydefinitions.apikey ApiKeyInQuery -// @in query -// @name api_key +// @securitydefinitions.apikey ApiKeyInQuery +// @in query +// @name api_key func (h *HandlerService) PublicGetHealthz(w http.ResponseWriter, r *http.Request) { var v validationError @@ -134,14 +134,14 @@ func (h *HandlerService) PublicPutAccountDashboardTransactionsSettings(w http.Re // PublicPostValidatorDashboards godoc // // @Description Create a new validator dashboard. **Note**: New dashboards will automatically have a default group created. -// @Security ApiKeyInHeader || ApiKeyInQuery +// @Security ApiKeyInHeader || ApiKeyInQuery // @Tags Validator Dashboards // @Accept json // @Produce json -// @Param request body handlers.PublicPostValidatorDashboards.request true "`name`: Specify the name of the dashboard.
`network`: Specify the network for the dashboard. Possible options are:
  • `ethereum`
  • `gnosis`
" -// @Success 201 {object} types.PostValidatorDashboardsResponse -// @Failure 400 {object} types.ApiErrorResponse -// @Failure 409 {object} types.ApiErrorResponse "Conflict. The request could not be performed by the server because the authenticated user has already reached their dashboard limit." +// @Param request body handlers.PublicPostValidatorDashboards.request true "`name`: Specify the name of the dashboard.
`network`: Specify the network for the dashboard. Possible options are:
  • `ethereum`
  • `gnosis`
" +// @Success 201 {object} types.PostValidatorDashboardsResponse +// @Failure 400 {object} types.ApiErrorResponse +// @Failure 409 {object} types.ApiErrorResponse "Conflict. The request could not be performed by the server because the authenticated user has already reached their dashboard limit." // @Router /validator-dashboards [post] func (h *HandlerService) PublicPostValidatorDashboards(w http.ResponseWriter, r *http.Request) { var v validationError @@ -823,8 +823,8 @@ func (h *HandlerService) PublicGetValidatorDashboardSummary(w http.ResponseWrite period := checkEnum[enums.TimePeriod](&v, q.Get("period"), "period") // allowed periods are: all_time, last_30d, last_7d, last_24h, last_1h - allowedPeriods := []enums.Enum{enums.TimePeriods.AllTime, enums.TimePeriods.Last30d, enums.TimePeriods.Last7d, enums.TimePeriods.Last24h, enums.TimePeriods.Last1h} - v.checkEnumIsAllowed(period, allowedPeriods, "period") + allowedPeriods := []enums.TimePeriod{enums.TimePeriods.AllTime, enums.TimePeriods.Last30d, enums.TimePeriods.Last7d, enums.TimePeriods.Last24h, enums.TimePeriods.Last1h} + checkEnumIsAllowed(&v, period, allowedPeriods, "period") if v.hasErrors() { handleErr(w, r, v) return @@ -859,8 +859,8 @@ func (h *HandlerService) PublicGetValidatorDashboardGroupSummary(w http.Response groupId := v.checkGroupId(vars["group_id"], forbidEmpty) period := checkEnum[enums.TimePeriod](&v, r.URL.Query().Get("period"), "period") // allowed periods are: all_time, last_30d, last_7d, last_24h, last_1h - allowedPeriods := []enums.Enum{enums.TimePeriods.AllTime, enums.TimePeriods.Last30d, enums.TimePeriods.Last7d, enums.TimePeriods.Last24h, enums.TimePeriods.Last1h} - v.checkEnumIsAllowed(period, allowedPeriods, "period") + allowedPeriods := []enums.TimePeriod{enums.TimePeriods.AllTime, enums.TimePeriods.Last30d, enums.TimePeriods.Last7d, enums.TimePeriods.Last24h, enums.TimePeriods.Last1h} + checkEnumIsAllowed(&v, period, allowedPeriods, "period") if v.hasErrors() { handleErr(w, r, v) return @@ -928,8 +928,8 @@ func (h *HandlerService) PublicGetValidatorDashboardSummaryValidators(w http.Res duty := checkEnum[enums.ValidatorDuty](&v, q.Get("duty"), "duty") period := checkEnum[enums.TimePeriod](&v, q.Get("period"), "period") // allowed periods are: all_time, last_30d, last_7d, last_24h, last_1h - allowedPeriods := []enums.Enum{enums.TimePeriods.AllTime, enums.TimePeriods.Last30d, enums.TimePeriods.Last7d, enums.TimePeriods.Last24h, enums.TimePeriods.Last1h} - v.checkEnumIsAllowed(period, allowedPeriods, "period") + allowedPeriods := []enums.TimePeriod{enums.TimePeriods.AllTime, enums.TimePeriods.Last30d, enums.TimePeriods.Last7d, enums.TimePeriods.Last24h, enums.TimePeriods.Last1h} + checkEnumIsAllowed(&v, period, allowedPeriods, "period") if v.hasErrors() { handleErr(w, r, v) return diff --git a/backend/pkg/api/router.go b/backend/pkg/api/router.go index 5603e4856..b98b3cb0c 100644 --- a/backend/pkg/api/router.go +++ b/backend/pkg/api/router.go @@ -88,6 +88,8 @@ func addRoutes(hs *handlers.HandlerService, publicRouter, internalRouter *mux.Ro {http.MethodGet, "/healthz", hs.PublicGetHealthz, nil}, {http.MethodGet, "/healthz-loadbalancer", hs.PublicGetHealthzLoadbalancer, nil}, + {http.MethodGet, "/ratelimit-weights", nil, hs.InternalGetRatelimitWeights}, + {http.MethodPost, "/login", nil, hs.InternalPostLogin}, {http.MethodGet, "/mobile/authorize", nil, hs.InternalPostMobileAuthorize}, diff --git a/backend/pkg/api/types/data_access.go b/backend/pkg/api/types/data_access.go index ff840e24b..8cb1b3312 100644 --- a/backend/pkg/api/types/data_access.go +++ b/backend/pkg/api/types/data_access.go @@ -212,16 +212,6 @@ type VDBValidatorSummaryChartRow struct { SyncScheduled float64 `db:"sync_scheduled"` } -// ------------------------- -// ratelimiting - -type ApiWeightItem struct { - Bucket string `db:"bucket"` - Endpoint string `db:"endpoint"` - Method string `db:"method"` - Weight int `db:"weight"` -} - // healthz structs type HealthzResult struct { diff --git a/backend/pkg/api/types/ratelimit.go b/backend/pkg/api/types/ratelimit.go new file mode 100644 index 000000000..6a1155096 --- /dev/null +++ b/backend/pkg/api/types/ratelimit.go @@ -0,0 +1,10 @@ +package types + +type ApiWeightItem struct { + Bucket string `db:"bucket"` + Endpoint string `db:"endpoint"` + Method string `db:"method"` + Weight int `db:"weight"` +} + +type InternalGetRatelimitWeightsResponse ApiDataResponse[[]ApiWeightItem] From 813af365970b6cb36328d4d7cc720d3f1aedfb43 Mon Sep 17 00:00:00 2001 From: LUCCA DUKIC <109136188+LuccaBitfly@users.noreply.github.com> Date: Tue, 10 Sep 2024 13:00:18 +0200 Subject: [PATCH 10/16] (BEDS-295) unexpose ratelimit consts --- backend/pkg/commons/ratelimit/ratelimit.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/backend/pkg/commons/ratelimit/ratelimit.go b/backend/pkg/commons/ratelimit/ratelimit.go index 82523865c..a57900bba 100644 --- a/backend/pkg/commons/ratelimit/ratelimit.go +++ b/backend/pkg/commons/ratelimit/ratelimit.go @@ -56,8 +56,8 @@ const ( FallbackRateLimitSecond = 20 // RateLimit per second for when redis is offline FallbackRateLimitBurst = 20 // RateLimit burst for when redis is offline - DefaultWeight = 1 // if no weight is set for a route, use this one - DefaultBucket = "default" // if no bucket is set for a route, use this one + defaultWeight = 1 // if no weight is set for a route, use this one + defaultBucket = "default" // if no bucket is set for a route, use this one statsTruncateDuration = time.Hour * 1 // ratelimit-stats are truncated to this duration ) @@ -385,7 +385,7 @@ func updateWeights(firstRun bool) error { } buckets[w.Endpoint] = strings.ReplaceAll(w.Bucket, ":", "_") if buckets[w.Endpoint] == "" { - buckets[w.Endpoint] = DefaultBucket + buckets[w.Endpoint] = defaultBucket } if !firstRun && oldBuckets[w.Endpoint] != buckets[w.Endpoint] { log.InfoWithFields(log.Fields{"endpoint": w.Endpoint, "bucket": w.Weight, "oldBucket": oldBuckets[w.Endpoint]}, "bucket changed") @@ -952,10 +952,10 @@ func getWeight(r *http.Request) (cost int64, identifier, bucket string) { bucket, bucketOk := buckets[route] weightsMu.RUnlock() if !weightOk { - weight = DefaultWeight + weight = defaultWeight } if !bucketOk { - bucket = DefaultBucket + bucket = defaultBucket } return weight, route, bucket } From 59df5237b18de7ef7a54a1d6bd58506930ec34b7 Mon Sep 17 00:00:00 2001 From: LUCCA DUKIC <109136188+LuccaBitfly@users.noreply.github.com> Date: Tue, 10 Sep 2024 13:01:21 +0200 Subject: [PATCH 11/16] (BEDS-295) remove unnecessary type --- backend/pkg/api/handlers/public.go | 4 ++-- backend/pkg/api/types/validator_dashboard.go | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/backend/pkg/api/handlers/public.go b/backend/pkg/api/handlers/public.go index c0c127ce5..a06e798c3 100644 --- a/backend/pkg/api/handlers/public.go +++ b/backend/pkg/api/handlers/public.go @@ -139,7 +139,7 @@ func (h *HandlerService) PublicPutAccountDashboardTransactionsSettings(w http.Re // @Accept json // @Produce json // @Param request body handlers.PublicPostValidatorDashboards.request true "`name`: Specify the name of the dashboard.
`network`: Specify the network for the dashboard. Possible options are:
  • `ethereum`
  • `gnosis`
" -// @Success 201 {object} types.PostValidatorDashboardsResponse +// @Success 201 {object} types.ApiDataResponse[types.VDBPostReturnData] // @Failure 400 {object} types.ApiErrorResponse // @Failure 409 {object} types.ApiErrorResponse "Conflict. The request could not be performed by the server because the authenticated user has already reached their dashboard limit." // @Router /validator-dashboards [post] @@ -187,7 +187,7 @@ func (h *HandlerService) PublicPostValidatorDashboards(w http.ResponseWriter, r handleErr(w, r, err) return } - response := types.PostValidatorDashboardsResponse{ + response := types.ApiDataResponse[types.VDBPostReturnData]{ Data: *data, } returnCreated(w, r, response) diff --git a/backend/pkg/api/types/validator_dashboard.go b/backend/pkg/api/types/validator_dashboard.go index d7fec23c6..39d5e5d0d 100644 --- a/backend/pkg/api/types/validator_dashboard.go +++ b/backend/pkg/api/types/validator_dashboard.go @@ -368,7 +368,6 @@ type VDBPostReturnData struct { Network uint64 `db:"network" json:"network"` CreatedAt int64 `db:"created_at" json:"created_at"` } -type PostValidatorDashboardsResponse ApiDataResponse[VDBPostReturnData] type VDBPostCreateGroupData struct { Id uint64 `db:"id" json:"id"` From d90611e37b8b72e6c83cc67daef465d77265b2b4 Mon Sep 17 00:00:00 2001 From: LUCCA DUKIC <109136188+LuccaBitfly@users.noreply.github.com> Date: Tue, 10 Sep 2024 13:03:48 +0200 Subject: [PATCH 12/16] (BEDS-295) consistent makefile command --- backend/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/Makefile b/backend/Makefile index c384ba55b..62ac053d8 100644 --- a/backend/Makefile +++ b/backend/Makefile @@ -11,7 +11,7 @@ CGO_CFLAGS_ALLOW="-O -D__BLST_PORTABLE__" all: mkdir -p bin CGO_CFLAGS=${CGO_CFLAGS} CGO_CFLAGS_ALLOW=${CGO_CFLAGS_ALLOW} go build --ldflags=${LDFLAGS} -o ./bin/bc ./cmd/main.go - go install github.com/swaggo/swag/cmd/swag@latest && swag init --ot json -o docs -d ./pkg/api/ -g ./handlers/public.go + go install github.com/swaggo/swag/cmd/swag@latest && swag init --ot json -o ./docs -d ./pkg/api/ -g ./handlers/public.go clean: rm -rf bin From c339f46b1d5afa3c7ad3f52e78a4e721d561d308 Mon Sep 17 00:00:00 2001 From: LUCCA DUKIC <109136188+LuccaBitfly@users.noreply.github.com> Date: Tue, 10 Sep 2024 14:34:06 +0200 Subject: [PATCH 13/16] (BEDS-295) embed doc file in binary --- backend/Makefile | 2 +- backend/pkg/api/router.go | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/backend/Makefile b/backend/Makefile index 62ac053d8..dd099490d 100644 --- a/backend/Makefile +++ b/backend/Makefile @@ -10,8 +10,8 @@ CGO_CFLAGS_ALLOW="-O -D__BLST_PORTABLE__" all: mkdir -p bin + go install github.com/swaggo/swag/cmd/swag@latest && swag init --ot json -o ./pkg/api/docs -d ./pkg/api/ -g ./handlers/public.go CGO_CFLAGS=${CGO_CFLAGS} CGO_CFLAGS_ALLOW=${CGO_CFLAGS_ALLOW} go build --ldflags=${LDFLAGS} -o ./bin/bc ./cmd/main.go - go install github.com/swaggo/swag/cmd/swag@latest && swag init --ot json -o ./docs -d ./pkg/api/ -g ./handlers/public.go clean: rm -rf bin diff --git a/backend/pkg/api/router.go b/backend/pkg/api/router.go index b98b3cb0c..6244c68ea 100644 --- a/backend/pkg/api/router.go +++ b/backend/pkg/api/router.go @@ -1,6 +1,7 @@ package api import ( + "embed" "net/http" "regexp" @@ -20,6 +21,9 @@ type endpoint struct { InternalHander func(w http.ResponseWriter, r *http.Request) } +//go:embed docs/* +var docFiles embed.FS + func NewApiRouter(dataAccessor dataaccess.DataAccessor, cfg *types.Config) *mux.Router { router := mux.NewRouter() apiRouter := router.PathPrefix("/api").Subrouter() @@ -39,6 +43,8 @@ func NewApiRouter(dataAccessor dataaccess.DataAccessor, cfg *types.Config) *mux. addRoutes(handlerService, publicRouter, internalRouter, cfg) + // serve static files + publicRouter.PathPrefix("/docs/").Handler(http.StripPrefix("/api/v2/", http.FileServer(http.FS(docFiles)))) router.Use(metrics.HttpMiddleware) return router From 571e027a1f27c49803639736d8e5f855d58ff6e3 Mon Sep 17 00:00:00 2001 From: LUCCA DUKIC <109136188+LuccaBitfly@users.noreply.github.com> Date: Tue, 10 Sep 2024 14:41:45 +0200 Subject: [PATCH 14/16] (BEDS-295) move docs embed var --- backend/.gitignore | 2 +- backend/pkg/api/docs/static.go | 6 ++++++ backend/pkg/api/router.go | 7 ++----- 3 files changed, 9 insertions(+), 6 deletions(-) create mode 100644 backend/pkg/api/docs/static.go diff --git a/backend/.gitignore b/backend/.gitignore index 6d056b68d..b5f10c4da 100644 --- a/backend/.gitignore +++ b/backend/.gitignore @@ -6,4 +6,4 @@ local_deployment/elconfig.json local_deployment/.env __gitignore cmd/playground -docs +pkg/api/docs/swagger.json diff --git a/backend/pkg/api/docs/static.go b/backend/pkg/api/docs/static.go new file mode 100644 index 000000000..93087d82f --- /dev/null +++ b/backend/pkg/api/docs/static.go @@ -0,0 +1,6 @@ +package docs + +import "embed" + +//go:embed * +var Files embed.FS diff --git a/backend/pkg/api/router.go b/backend/pkg/api/router.go index 6244c68ea..c52eb2e30 100644 --- a/backend/pkg/api/router.go +++ b/backend/pkg/api/router.go @@ -1,11 +1,11 @@ package api import ( - "embed" "net/http" "regexp" dataaccess "github.com/gobitfly/beaconchain/pkg/api/data_access" + "github.com/gobitfly/beaconchain/pkg/api/docs" handlers "github.com/gobitfly/beaconchain/pkg/api/handlers" "github.com/gobitfly/beaconchain/pkg/commons/log" "github.com/gobitfly/beaconchain/pkg/commons/metrics" @@ -21,9 +21,6 @@ type endpoint struct { InternalHander func(w http.ResponseWriter, r *http.Request) } -//go:embed docs/* -var docFiles embed.FS - func NewApiRouter(dataAccessor dataaccess.DataAccessor, cfg *types.Config) *mux.Router { router := mux.NewRouter() apiRouter := router.PathPrefix("/api").Subrouter() @@ -44,7 +41,7 @@ func NewApiRouter(dataAccessor dataaccess.DataAccessor, cfg *types.Config) *mux. addRoutes(handlerService, publicRouter, internalRouter, cfg) // serve static files - publicRouter.PathPrefix("/docs/").Handler(http.StripPrefix("/api/v2/", http.FileServer(http.FS(docFiles)))) + publicRouter.PathPrefix("/docs/").Handler(http.StripPrefix("/api/v2/docs/", http.FileServer(http.FS(docs.Files)))) router.Use(metrics.HttpMiddleware) return router From 149a9ab37409760b7f4a57292049a62874d34f43 Mon Sep 17 00:00:00 2001 From: LUCCA DUKIC <109136188+LuccaBitfly@users.noreply.github.com> Date: Wed, 11 Sep 2024 12:45:57 +0200 Subject: [PATCH 15/16] (BEDS-295) add test for api doc --- .../workflows/backend-integration-test.yml | 4 +- backend/go.mod | 5 +++ backend/go.sum | 9 ++++ backend/pkg/api/api_test.go | 43 +++++++++++++++++++ backend/pkg/api/data_access/ratelimit.go | 2 +- backend/pkg/api/handlers/internal.go | 4 ++ frontend/types/api/ratelimit.ts | 14 ++++++ 7 files changed, 79 insertions(+), 2 deletions(-) create mode 100644 frontend/types/api/ratelimit.ts diff --git a/.github/workflows/backend-integration-test.yml b/.github/workflows/backend-integration-test.yml index 48243480e..c95d43c70 100644 --- a/.github/workflows/backend-integration-test.yml +++ b/.github/workflows/backend-integration-test.yml @@ -33,7 +33,9 @@ jobs: cache-dependency-path: 'backend/go.sum' - name: Test with the Go CLI working-directory: backend - run: go test -failfast ./pkg/api/... -config "${{ secrets.CI_CONFIG_PATH }}" + run: + go install github.com/swaggo/swag/cmd/swag@latest && swag init --ot json -o ./pkg/api/docs -d ./pkg/api/ -g ./handlers/public.go + go test -failfast ./pkg/api/... -config "${{ secrets.CI_CONFIG_PATH }}" diff --git a/backend/go.mod b/backend/go.mod index 58a24bf21..59fc22b83 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -24,6 +24,7 @@ require ( github.com/fergusstrange/embedded-postgres v1.29.0 github.com/gavv/httpexpect/v2 v2.16.0 github.com/go-faker/faker/v4 v4.3.0 + github.com/go-openapi/spec v0.20.14 github.com/go-redis/redis/v8 v8.11.5 github.com/gobitfly/eth-rewards v0.1.2-0.20230403064929-411ddc40a5f7 github.com/gobitfly/eth.store v0.0.0-20240312111708-b43f13990280 @@ -139,6 +140,9 @@ require ( github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.3.0 // indirect + github.com/go-openapi/jsonpointer v0.20.2 // indirect + github.com/go-openapi/jsonreference v0.20.4 // indirect + github.com/go-openapi/swag v0.22.9 // indirect github.com/gobwas/glob v0.2.3 // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/goccy/go-yaml v1.9.5 // indirect @@ -178,6 +182,7 @@ require ( github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect github.com/jackc/puddle/v2 v2.2.1 // indirect github.com/jbenet/goprocess v0.1.4 // indirect + github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/cpuid/v2 v2.2.7 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect diff --git a/backend/go.sum b/backend/go.sum index fa1e4f202..34c1e36af 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -285,6 +285,14 @@ github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= +github.com/go-openapi/jsonpointer v0.20.2 h1:mQc3nmndL8ZBzStEo3JYF8wzmeWffDH4VbXz58sAx6Q= +github.com/go-openapi/jsonpointer v0.20.2/go.mod h1:bHen+N0u1KEO3YlmqOjTT9Adn1RfD91Ar825/PuiRVs= +github.com/go-openapi/jsonreference v0.20.4 h1:bKlDxQxQJgwpUSgOENiMPzCTBVuc7vTdXSSgNeAhojU= +github.com/go-openapi/jsonreference v0.20.4/go.mod h1:5pZJyJP2MnYCpoeoMAql78cCHauHj0V9Lhc506VOpw4= +github.com/go-openapi/spec v0.20.14 h1:7CBlRnw+mtjFGlPDRZmAMnq35cRzI91xj03HVyUi/Do= +github.com/go-openapi/spec v0.20.14/go.mod h1:8EOhTpBoFiask8rrgwbLC3zmJfz4zsCUueRuPM6GNkw= +github.com/go-openapi/swag v0.22.9 h1:XX2DssF+mQKM2DHsbgZK74y/zj4mo9I99+89xUmuZCE= +github.com/go-openapi/swag v0.22.9/go.mod h1:3/OXnFfnMAwBD099SwYRk7GD3xOrr1iL7d/XNLXVVwE= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU= @@ -562,6 +570,7 @@ github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8 github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak= github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4= github.com/jonboulle/clockwork v0.4.0/go.mod h1:xgRqUGwRcjKCO1vbZUEtSLrqKoPSsUpK7fnezOII0kc= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= diff --git a/backend/pkg/api/api_test.go b/backend/pkg/api/api_test.go index c422bf17f..a259f9196 100644 --- a/backend/pkg/api/api_test.go +++ b/backend/pkg/api/api_test.go @@ -9,12 +9,14 @@ import ( "net/http/httptest" "os" "os/exec" + "slices" "sort" "testing" "time" embeddedpostgres "github.com/fergusstrange/embedded-postgres" "github.com/gavv/httpexpect/v2" + "github.com/go-openapi/spec" "github.com/gobitfly/beaconchain/pkg/api" dataaccess "github.com/gobitfly/beaconchain/pkg/api/data_access" api_types "github.com/gobitfly/beaconchain/pkg/api/types" @@ -25,6 +27,7 @@ import ( "github.com/jmoiron/sqlx" "github.com/pressly/goose/v3" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "golang.org/x/crypto/bcrypt" ) @@ -111,6 +114,16 @@ func setup() error { return fmt.Errorf("error inserting user 2: %w", err) } + // insert dummy api weight for testing + _, err = tempDb.Exec(` + INSERT INTO api_weights (bucket, endpoint, method, params, weight, valid_from) + VALUES ($1, $2, $3, $4, $5, TO_TIMESTAMP($6))`, + "default", "/api/v2/test-ratelimit", "GET", "", 2, time.Now().Unix(), + ) + if err != nil { + return fmt.Errorf("error inserting user 2: %w", err) + } + cfg := &types.Config{} err = utils.ReadConfig(cfg, *configPath) if err != nil { @@ -469,3 +482,33 @@ func TestPublicAndSharedDashboards(t *testing.T) { }) } } + +func TestApiDoc(t *testing.T) { + e := httpexpect.WithConfig(getExpectConfig(t, ts)) + + t.Run("test api doc json", func(t *testing.T) { + resp := spec.Swagger{} + e.GET("/api/v2/docs/swagger.json"). + Expect(). + Status(http.StatusOK).JSON().Decode(&resp) + + assert.Equal(t, "/api/v2", resp.BasePath, "swagger base path should be '/api/v2'") + require.NotNil(t, 0, resp.Paths, "swagger paths should not nil") + assert.NotEqual(t, 0, len(resp.Paths.Paths), "swagger paths should not be empty") + assert.NotEqual(t, 0, len(resp.Definitions), "swagger definitions should not be empty") + assert.NotEqual(t, 0, len(resp.Host), "swagger host should not be empty") + }) + + t.Run("test api ratelimit weights endpoint", func(t *testing.T) { + resp := api_types.InternalGetRatelimitWeightsResponse{} + e.GET("/api/i/ratelimit-weights"). + Expect(). + Status(http.StatusOK).JSON().Decode(&resp) + + assert.GreaterOrEqual(t, len(resp.Data), 1, "ratelimit weights should contain at least one entry") + testEndpointIndex := slices.IndexFunc(resp.Data, func(item api_types.ApiWeightItem) bool { + return item.Endpoint == "/api/v2/test-ratelimit" + }) + assert.GreaterOrEqual(t, testEndpointIndex, 0, "ratelimit weights should contain an entry for /api/v2/test-ratelimit") + }) +} diff --git a/backend/pkg/api/data_access/ratelimit.go b/backend/pkg/api/data_access/ratelimit.go index 973dbed1e..8c17c5d0f 100644 --- a/backend/pkg/api/data_access/ratelimit.go +++ b/backend/pkg/api/data_access/ratelimit.go @@ -13,7 +13,7 @@ type RatelimitRepository interface { func (d *DataAccessService) GetApiWeights(ctx context.Context) ([]types.ApiWeightItem, error) { var result []types.ApiWeightItem - err := d.readerDb.SelectContext(ctx, &result, ` + err := d.userReader.SelectContext(ctx, &result, ` SELECT bucket, endpoint, method, weight FROM api_weights WHERE valid_from <= NOW() diff --git a/backend/pkg/api/handlers/internal.go b/backend/pkg/api/handlers/internal.go index 1554aaee2..461ec54a7 100644 --- a/backend/pkg/api/handlers/internal.go +++ b/backend/pkg/api/handlers/internal.go @@ -1280,3 +1280,7 @@ func (h *HandlerService) InternalGetSlotBlobs(w http.ResponseWriter, r *http.Req } returnOk(w, r, response) } + +func (h *HandlerService) ReturnOk(w http.ResponseWriter, r *http.Request) { + returnOk(w, r, nil) +} diff --git a/frontend/types/api/ratelimit.ts b/frontend/types/api/ratelimit.ts new file mode 100644 index 000000000..aec92ffb8 --- /dev/null +++ b/frontend/types/api/ratelimit.ts @@ -0,0 +1,14 @@ +// Code generated by tygo. DO NOT EDIT. +/* eslint-disable */ +import type { ApiDataResponse } from './common' + +////////// +// source: ratelimit.go + +export interface ApiWeightItem { + Bucket: string; + Endpoint: string; + Method: string; + Weight: number /* int */; +} +export type InternalGetRatelimitWeightsResponse = ApiDataResponse; From dbe61f07030f471eeaa7c9030596d9ca95c138f6 Mon Sep 17 00:00:00 2001 From: LUCCA DUKIC <109136188+LuccaBitfly@users.noreply.github.com> Date: Wed, 11 Sep 2024 12:47:27 +0200 Subject: [PATCH 16/16] (BEDS-295) fix error msg when inserting api weight --- backend/pkg/api/api_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/pkg/api/api_test.go b/backend/pkg/api/api_test.go index a259f9196..a38763240 100644 --- a/backend/pkg/api/api_test.go +++ b/backend/pkg/api/api_test.go @@ -121,7 +121,7 @@ func setup() error { "default", "/api/v2/test-ratelimit", "GET", "", 2, time.Now().Unix(), ) if err != nil { - return fmt.Errorf("error inserting user 2: %w", err) + return fmt.Errorf("error inserting api weight: %w", err) } cfg := &types.Config{}