(BEDS-295) implement automated api doc generation (#748)

.github/workflows/backend-integration-test.yml

@@ -33,7 +33,9 @@ jobs:
           cache-dependency-path: 'backend/go.sum'
       - name: Test with the Go CLI
         working-directory: backend
-        run: go test -failfast ./pkg/api/... -config "${{ secrets.CI_CONFIG_PATH }}"
+        run:
+          go install github.com/swaggo/swag/cmd/swag@latest && swag init --ot json -o ./pkg/api/docs -d ./pkg/api/ -g ./handlers/public.go
+          go test -failfast ./pkg/api/... -config "${{ secrets.CI_CONFIG_PATH }}"
 
 
 

backend/.gitignore

@@ -5,4 +5,5 @@ local_deployment/config.yml
 local_deployment/elconfig.json
 local_deployment/.env
 __gitignore
-cmd/playground
+cmd/playground
+pkg/api/docs/swagger.json

backend/Makefile

@@ -10,8 +10,8 @@ CGO_CFLAGS_ALLOW="-O -D__BLST_PORTABLE__"
 
 all: 
 	mkdir -p bin
+	go install github.com/swaggo/swag/cmd/swag@latest && swag init --ot json -o ./pkg/api/docs -d ./pkg/api/ -g ./handlers/public.go
 	CGO_CFLAGS=${CGO_CFLAGS} CGO_CFLAGS_ALLOW=${CGO_CFLAGS_ALLOW} go build --ldflags=${LDFLAGS} -o ./bin/bc ./cmd/main.go
-
 clean:
 	rm -rf bin
 

backend/cmd/typescript_converter/main.go

@@ -31,7 +31,7 @@ var typeMappings = map[string]string{
 // Expects the following flags:
 // -out: Output folder for the generated TypeScript file
 
-// Standard usage (execute in backend folder): go run cmd/typescript_converter/main.go -out ../frontend/types/api
+// Standard usage (execute in backend folder): go run cmd/main.go typescript-converter -out ../frontend/types/api
 
 func Run() {
 	var out string

backend/go.mod

@@ -24,6 +24,7 @@ require (
 	github.com/fergusstrange/embedded-postgres v1.29.0
 	github.com/gavv/httpexpect/v2 v2.16.0
 	github.com/go-faker/faker/v4 v4.3.0
+	github.com/go-openapi/spec v0.20.14
 	github.com/go-redis/redis/v8 v8.11.5
 	github.com/gobitfly/eth-rewards v0.1.2-0.20230403064929-411ddc40a5f7
 	github.com/gobitfly/eth.store v0.0.0-20240312111708-b43f13990280
@@ -139,6 +140,9 @@ require (
 	github.com/go-logr/logr v1.4.1 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-ole/go-ole v1.3.0 // indirect
+	github.com/go-openapi/jsonpointer v0.20.2 // indirect
+	github.com/go-openapi/jsonreference v0.20.4 // indirect
+	github.com/go-openapi/swag v0.22.9 // indirect
 	github.com/gobwas/glob v0.2.3 // indirect
 	github.com/goccy/go-json v0.10.2 // indirect
 	github.com/goccy/go-yaml v1.9.5 // indirect
@@ -178,6 +182,7 @@ require (
 	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
 	github.com/jackc/puddle/v2 v2.2.1 // indirect
 	github.com/jbenet/goprocess v0.1.4 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.7 // indirect
 	github.com/libp2p/go-buffer-pool v0.1.0 // indirect
@@ -264,6 +269,7 @@ require (
 	lukechampine.com/blake3 v1.2.1 // indirect
 	moul.io/http2curl/v2 v2.3.0 // indirect
 	rsc.io/tmplfunc v0.0.3 // indirect
+	sigs.k8s.io/yaml v1.3.0 // indirect
 )
 
 replace github.com/wealdtech/go-merkletree v1.0.1-0.20190605192610-2bb163c2ea2a => github.com/rocket-pool/go-merkletree v1.0.1-0.20220406020931-c262d9b976dd

backend/go.sum

@@ -285,6 +285,14 @@ github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre
 github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
 github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
 github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=
+github.com/go-openapi/jsonpointer v0.20.2 h1:mQc3nmndL8ZBzStEo3JYF8wzmeWffDH4VbXz58sAx6Q=
+github.com/go-openapi/jsonpointer v0.20.2/go.mod h1:bHen+N0u1KEO3YlmqOjTT9Adn1RfD91Ar825/PuiRVs=
+github.com/go-openapi/jsonreference v0.20.4 h1:bKlDxQxQJgwpUSgOENiMPzCTBVuc7vTdXSSgNeAhojU=
+github.com/go-openapi/jsonreference v0.20.4/go.mod h1:5pZJyJP2MnYCpoeoMAql78cCHauHj0V9Lhc506VOpw4=
+github.com/go-openapi/spec v0.20.14 h1:7CBlRnw+mtjFGlPDRZmAMnq35cRzI91xj03HVyUi/Do=
+github.com/go-openapi/spec v0.20.14/go.mod h1:8EOhTpBoFiask8rrgwbLC3zmJfz4zsCUueRuPM6GNkw=
+github.com/go-openapi/swag v0.22.9 h1:XX2DssF+mQKM2DHsbgZK74y/zj4mo9I99+89xUmuZCE=
+github.com/go-openapi/swag v0.22.9/go.mod h1:3/OXnFfnMAwBD099SwYRk7GD3xOrr1iL7d/XNLXVVwE=
 github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
 github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
 github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU=
@@ -562,6 +570,7 @@ github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8
 github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak=
 github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4=
 github.com/jonboulle/clockwork v0.4.0/go.mod h1:xgRqUGwRcjKCO1vbZUEtSLrqKoPSsUpK7fnezOII0kc=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
@@ -1284,5 +1293,5 @@ rsc.io/tmplfunc v0.0.3 h1:53XFQh69AfOa8Tw0Jm7t+GV7KZhOi6jzsCzTtKbMvzU=
 rsc.io/tmplfunc v0.0.3/go.mod h1:AG3sTPzElb1Io3Yg4voV9AGZJuleGAwaVRxL9M49PhA=
 sigs.k8s.io/structured-merge-diff/v3 v3.0.0 h1:dOmIZBMfhcHS09XZkMyUgkq5trg3/jRyJYFZUiaOp8E=
 sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw=
-sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
-sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
+sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
+sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=

backend/pkg/api/api_test.go

@@ -9,12 +9,14 @@ import (
 	"net/http/httptest"
 	"os"
 	"os/exec"
+	"slices"
 	"sort"
 	"testing"
 	"time"
 
 	embeddedpostgres "github.com/fergusstrange/embedded-postgres"
 	"github.com/gavv/httpexpect/v2"
+	"github.com/go-openapi/spec"
 	"github.com/gobitfly/beaconchain/pkg/api"
 	dataaccess "github.com/gobitfly/beaconchain/pkg/api/data_access"
 	api_types "github.com/gobitfly/beaconchain/pkg/api/types"
@@ -25,6 +27,7 @@ import (
 	"github.com/jmoiron/sqlx"
 	"github.com/pressly/goose/v3"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	"golang.org/x/crypto/bcrypt"
 )
 
@@ -111,6 +114,16 @@ func setup() error {
 		return fmt.Errorf("error inserting user 2: %w", err)
 	}
 
+	// insert dummy api weight for testing
+	_, err = tempDb.Exec(`
+      INSERT INTO api_weights (bucket, endpoint, method, params, weight, valid_from)
+      VALUES ($1, $2, $3, $4, $5, TO_TIMESTAMP($6))`,
+		"default", "/api/v2/test-ratelimit", "GET", "", 2, time.Now().Unix(),
+	)
+	if err != nil {
+		return fmt.Errorf("error inserting api weight: %w", err)
+	}
+
 	cfg := &types.Config{}
 	err = utils.ReadConfig(cfg, *configPath)
 	if err != nil {
@@ -469,3 +482,33 @@ func TestPublicAndSharedDashboards(t *testing.T) {
 		})
 	}
 }
+
+func TestApiDoc(t *testing.T) {
+	e := httpexpect.WithConfig(getExpectConfig(t, ts))
+
+	t.Run("test api doc json", func(t *testing.T) {
+		resp := spec.Swagger{}
+		e.GET("/api/v2/docs/swagger.json").
+			Expect().
+			Status(http.StatusOK).JSON().Decode(&resp)
+
+		assert.Equal(t, "/api/v2", resp.BasePath, "swagger base path should be '/api/v2'")
+		require.NotNil(t, 0, resp.Paths, "swagger paths should not nil")
+		assert.NotEqual(t, 0, len(resp.Paths.Paths), "swagger paths should not be empty")
+		assert.NotEqual(t, 0, len(resp.Definitions), "swagger definitions should not be empty")
+		assert.NotEqual(t, 0, len(resp.Host), "swagger host should not be empty")
+	})
+
+	t.Run("test api ratelimit weights endpoint", func(t *testing.T) {
+		resp := api_types.InternalGetRatelimitWeightsResponse{}
+		e.GET("/api/i/ratelimit-weights").
+			Expect().
+			Status(http.StatusOK).JSON().Decode(&resp)
+
+		assert.GreaterOrEqual(t, len(resp.Data), 1, "ratelimit weights should contain at least one entry")
+		testEndpointIndex := slices.IndexFunc(resp.Data, func(item api_types.ApiWeightItem) bool {
+			return item.Endpoint == "/api/v2/test-ratelimit"
+		})
+		assert.GreaterOrEqual(t, testEndpointIndex, 0, "ratelimit weights should contain an entry for /api/v2/test-ratelimit")
+	})
+}

backend/pkg/api/data_access/data_access.go

@@ -13,7 +13,6 @@ import (
 	"github.com/gobitfly/beaconchain/pkg/commons/db"
 	"github.com/gobitfly/beaconchain/pkg/commons/log"
 	"github.com/gobitfly/beaconchain/pkg/commons/types"
-	"github.com/gobitfly/beaconchain/pkg/commons/utils"
 	"github.com/jmoiron/sqlx"
 	"github.com/pkg/errors"
 )
@@ -29,6 +28,7 @@ type DataAccessor interface {
 	BlockRepository
 	ArchiverRepository
 	ProtocolRepository
+	RatelimitRepository
 	HealthzRepository
 
 	StartDataAccessServices()
@@ -203,19 +203,19 @@ func createDataAccessService(cfg *types.Config) *DataAccessService {
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
-		bt, err := db.InitBigtable(utils.Config.Bigtable.Project, utils.Config.Bigtable.Instance, fmt.Sprintf("%d", utils.Config.Chain.ClConfig.DepositChainID), utils.Config.RedisCacheEndpoint)
+		bt, err := db.InitBigtable(cfg.Bigtable.Project, cfg.Bigtable.Instance, fmt.Sprintf("%d", cfg.Chain.ClConfig.DepositChainID), cfg.RedisCacheEndpoint)
 		if err != nil {
 			log.Fatal(err, "error connecting to bigtable", 0)
 		}
 		dataAccessService.bigtable = bt
 	}()
 
 	// Initialize the tiered cache (redis)
-	if utils.Config.TieredCacheProvider == "redis" || len(utils.Config.RedisCacheEndpoint) != 0 {
+	if cfg.TieredCacheProvider == "redis" || len(cfg.RedisCacheEndpoint) != 0 {
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			cache.MustInitTieredCache(utils.Config.RedisCacheEndpoint)
+			cache.MustInitTieredCache(cfg.RedisCacheEndpoint)
 			log.Infof("tiered Cache initialized, latest finalized epoch: %v", cache.LatestFinalizedEpoch.Get())
 		}()
 	}
@@ -225,7 +225,7 @@ func createDataAccessService(cfg *types.Config) *DataAccessService {
 	go func() {
 		defer wg.Done()
 		rdc := redis.NewClient(&redis.Options{
-			Addr:        utils.Config.RedisSessionStoreEndpoint,
+			Addr:        cfg.RedisSessionStoreEndpoint,
 			ReadTimeout: time.Second * 60,
 		})
 
@@ -237,7 +237,7 @@ func createDataAccessService(cfg *types.Config) *DataAccessService {
 
 	wg.Wait()
 
-	if utils.Config.TieredCacheProvider != "redis" {
+	if cfg.TieredCacheProvider != "redis" {
 		log.Fatal(fmt.Errorf("no cache provider set, please set TierdCacheProvider (example redis)"), "", 0)
 	}
 

backend/pkg/api/data_access/dummy.go

@@ -636,6 +636,12 @@ func (d *DummyService) GetRocketPoolOverview(ctx context.Context) (*t.RocketPool
 	return getDummyStruct[t.RocketPoolData]()
 }
 
+func (d *DummyService) GetApiWeights(ctx context.Context) ([]t.ApiWeightItem, error) {
+	r := []t.ApiWeightItem{}
+	err := commonFakeData(&r)
+	return r, err
+}
+
 func (d *DummyService) GetHealthz(ctx context.Context, showAll bool) t.HealthzData {
 	r, _ := getDummyData[t.HealthzData]()
 	return r

backend/pkg/api/data_access/ratelimit.go

@@ -0,0 +1,22 @@
+package dataaccess
+
+import (
+	"context"
+
+	"github.com/gobitfly/beaconchain/pkg/api/types"
+)
+
+type RatelimitRepository interface {
+	GetApiWeights(ctx context.Context) ([]types.ApiWeightItem, error)
+	// TODO @patrick: move queries from commons/ratelimit/ratelimit.go to here
+}
+
+func (d *DataAccessService) GetApiWeights(ctx context.Context) ([]types.ApiWeightItem, error) {
+	var result []types.ApiWeightItem
+	err := d.userReader.SelectContext(ctx, &result, `
+		SELECT bucket, endpoint, method, weight
+		FROM api_weights
+		WHERE valid_from <= NOW()
+	`)
+	return result, err
+}

backend/pkg/api/docs/static.go

@@ -0,0 +1,6 @@
+package docs
+
+import "embed"
+
+//go:embed *
+var Files embed.FS

backend/pkg/api/handlers/common.go

@@ -528,8 +528,7 @@ func checkEnum[T enums.EnumFactory[T]](v *validationError, enumString string, na
 }
 
 // checkEnumIsAllowed checks if the given enum is in the list of allowed enums.
-// precondition: the enum is the same type as the allowed enums.
-func (v *validationError) checkEnumIsAllowed(enum enums.Enum, allowed []enums.Enum, name string) {
+func checkEnumIsAllowed[T enums.EnumFactory[T]](v *validationError, enum T, allowed []T, name string) {
 	if enums.IsInvalidEnum(enum) {
 		v.add(name, "parameter is missing or invalid, please check the API documentation")
 		return

backend/pkg/api/handlers/internal.go

@@ -26,6 +26,21 @@ func (h *HandlerService) InternalGetProductSummary(w http.ResponseWriter, r *htt
 	returnOk(w, r, response)
 }
 
+// --------------------------------------
+// API Ratelimit Weights
+
+func (h *HandlerService) InternalGetRatelimitWeights(w http.ResponseWriter, r *http.Request) {
+	data, err := h.dai.GetApiWeights(r.Context())
+	if err != nil {
+		handleErr(w, r, err)
+		return
+	}
+	response := types.InternalGetRatelimitWeightsResponse{
+		Data: data,
+	}
+	returnOk(w, r, response)
+}
+
 // --------------------------------------
 // Latest State
 
@@ -85,7 +100,7 @@ func (h *HandlerService) InternalPostAdConfigurations(w http.ResponseWriter, r *
 		handleErr(w, r, err)
 		return
 	}
-	if user.UserGroup != "ADMIN" {
+	if user.UserGroup != types.UserGroupAdmin {
 		returnForbidden(w, r, errors.New("user is not an admin"))
 		return
 	}
@@ -131,7 +146,7 @@ func (h *HandlerService) InternalGetAdConfigurations(w http.ResponseWriter, r *h
 		handleErr(w, r, err)
 		return
 	}
-	if user.UserGroup != "ADMIN" {
+	if user.UserGroup != types.UserGroupAdmin {
 		returnForbidden(w, r, errors.New("user is not an admin"))
 		return
 	}
@@ -161,7 +176,7 @@ func (h *HandlerService) InternalPutAdConfiguration(w http.ResponseWriter, r *ht
 		handleErr(w, r, err)
 		return
 	}
-	if user.UserGroup != "ADMIN" {
+	if user.UserGroup != types.UserGroupAdmin {
 		returnForbidden(w, r, errors.New("user is not an admin"))
 		return
 	}
@@ -207,7 +222,7 @@ func (h *HandlerService) InternalDeleteAdConfiguration(w http.ResponseWriter, r
 		handleErr(w, r, err)
 		return
 	}
-	if user.UserGroup != "ADMIN" {
+	if user.UserGroup != types.UserGroupAdmin {
 		returnForbidden(w, r, errors.New("user is not an admin"))
 		return
 	}
@@ -1311,3 +1326,7 @@ func (h *HandlerService) InternalGetSlotBlobs(w http.ResponseWriter, r *http.Req
 	}
 	returnOk(w, r, response)
 }
+
+func (h *HandlerService) ReturnOk(w http.ResponseWriter, r *http.Request) {
+	returnOk(w, r, nil)
+}