Merge pull request #867 from gobitfly/staging

Merge staging->main

.github/workflows/backend-integration-test.yml

@@ -33,7 +33,9 @@ jobs:
           cache-dependency-path: 'backend/go.sum'
       - name: Test with the Go CLI
         working-directory: backend
-        run: go test -failfast ./pkg/api/... -config "${{ secrets.CI_CONFIG_PATH }}"
+        run:
+          go install github.com/swaggo/swag/cmd/swag@latest && swag init --ot json -o ./pkg/api/docs -d ./pkg/api/ -g ./handlers/public.go
+          go test -failfast ./pkg/api/... -config "${{ secrets.CI_CONFIG_PATH }}"
 
 
 

backend/.gitignore

@@ -5,4 +5,5 @@ local_deployment/config.yml
 local_deployment/elconfig.json
 local_deployment/.env
 __gitignore
-cmd/playground
+cmd/playground
+pkg/api/docs/swagger.json

backend/Makefile

@@ -10,8 +10,8 @@ CGO_CFLAGS_ALLOW="-O -D__BLST_PORTABLE__"
 
 all: 
 	mkdir -p bin
+	go install github.com/swaggo/swag/cmd/swag@latest && swag init --ot json -o ./pkg/api/docs -d ./pkg/api/ -g ./handlers/public.go
 	CGO_CFLAGS=${CGO_CFLAGS} CGO_CFLAGS_ALLOW=${CGO_CFLAGS_ALLOW} go build --ldflags=${LDFLAGS} -o ./bin/bc ./cmd/main.go
-
 clean:
 	rm -rf bin
 

backend/cmd/eth1indexer/main.go

@@ -189,6 +189,10 @@ func Run() {
 		}()
 	}
 
+	if *enableEnsUpdater {
+		go ImportEnsUpdatesLoop(bt, client, *ensBatchSize)
+	}
+
 	if *enableFullBalanceUpdater {
 		ProcessMetadataUpdates(bt, client, balanceUpdaterPrefix, *balanceUpdaterBatchSize, -1)
 		return
@@ -375,21 +379,26 @@ func Run() {
 			ProcessMetadataUpdates(bt, client, balanceUpdaterPrefix, *balanceUpdaterBatchSize, 10)
 		}
 
-		if *enableEnsUpdater {
-			err := bt.ImportEnsUpdates(client.GetNativeClient(), *ensBatchSize)
-			if err != nil {
-				log.Error(err, "error importing ens updates", 0, nil)
-				continue
-			}
-		}
-
 		log.Infof("index run completed")
 		services.ReportStatus("eth1indexer", "Running", nil)
 	}
 
 	// utils.WaitForCtrlC()
 }
 
+func ImportEnsUpdatesLoop(bt *db.Bigtable, client *rpc.ErigonClient, batchSize int64) {
+	time.Sleep(time.Second * 5)
+	for {
+		err := bt.ImportEnsUpdates(client.GetNativeClient(), batchSize)
+		if err != nil {
+			log.Error(err, "error importing ens updates", 0, nil)
+		} else {
+			services.ReportStatus("ensIndexer", "Running", nil)
+		}
+		time.Sleep(time.Second * 5)
+	}
+}
+
 func UpdateTokenPrices(bt *db.Bigtable, client *rpc.ErigonClient, tokenListPath string) error {
 	tokenListContent, err := os.ReadFile(tokenListPath)
 	if err != nil {

backend/cmd/typescript_converter/main.go

@@ -21,7 +21,7 @@ const (
 )
 
 // Files that should not be converted to TypeScript
-var ignoredFiles = []string{"data_access", "search_types"}
+var ignoredFiles = []string{"data_access", "search_types", "archiver"}
 
 var typeMappings = map[string]string{
 	"decimal.Decimal": "string /* decimal.Decimal */",
@@ -31,7 +31,7 @@ var typeMappings = map[string]string{
 // Expects the following flags:
 // -out: Output folder for the generated TypeScript file
 
-// Standard usage (execute in backend folder): go run cmd/typescript_converter/main.go -out ../frontend/types/api
+// Standard usage (execute in backend folder): go run cmd/main.go typescript-converter -out ../frontend/types/api
 
 func Run() {
 	var out string

backend/go.mod

@@ -24,6 +24,7 @@ require (
 	github.com/fergusstrange/embedded-postgres v1.29.0
 	github.com/gavv/httpexpect/v2 v2.16.0
 	github.com/go-faker/faker/v4 v4.3.0
+	github.com/go-openapi/spec v0.20.14
 	github.com/go-redis/redis/v8 v8.11.5
 	github.com/gobitfly/eth-rewards v0.1.2-0.20230403064929-411ddc40a5f7
 	github.com/gobitfly/eth.store v0.0.0-20240312111708-b43f13990280
@@ -139,6 +140,9 @@ require (
 	github.com/go-logr/logr v1.4.1 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-ole/go-ole v1.3.0 // indirect
+	github.com/go-openapi/jsonpointer v0.20.2 // indirect
+	github.com/go-openapi/jsonreference v0.20.4 // indirect
+	github.com/go-openapi/swag v0.22.9 // indirect
 	github.com/gobwas/glob v0.2.3 // indirect
 	github.com/goccy/go-json v0.10.2 // indirect
 	github.com/goccy/go-yaml v1.9.5 // indirect
@@ -178,6 +182,7 @@ require (
 	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
 	github.com/jackc/puddle/v2 v2.2.1 // indirect
 	github.com/jbenet/goprocess v0.1.4 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.7 // indirect
 	github.com/libp2p/go-buffer-pool v0.1.0 // indirect
@@ -264,6 +269,7 @@ require (
 	lukechampine.com/blake3 v1.2.1 // indirect
 	moul.io/http2curl/v2 v2.3.0 // indirect
 	rsc.io/tmplfunc v0.0.3 // indirect
+	sigs.k8s.io/yaml v1.3.0 // indirect
 )
 
 replace github.com/wealdtech/go-merkletree v1.0.1-0.20190605192610-2bb163c2ea2a => github.com/rocket-pool/go-merkletree v1.0.1-0.20220406020931-c262d9b976dd

backend/go.sum

@@ -285,6 +285,14 @@ github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre
 github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
 github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
 github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=
+github.com/go-openapi/jsonpointer v0.20.2 h1:mQc3nmndL8ZBzStEo3JYF8wzmeWffDH4VbXz58sAx6Q=
+github.com/go-openapi/jsonpointer v0.20.2/go.mod h1:bHen+N0u1KEO3YlmqOjTT9Adn1RfD91Ar825/PuiRVs=
+github.com/go-openapi/jsonreference v0.20.4 h1:bKlDxQxQJgwpUSgOENiMPzCTBVuc7vTdXSSgNeAhojU=
+github.com/go-openapi/jsonreference v0.20.4/go.mod h1:5pZJyJP2MnYCpoeoMAql78cCHauHj0V9Lhc506VOpw4=
+github.com/go-openapi/spec v0.20.14 h1:7CBlRnw+mtjFGlPDRZmAMnq35cRzI91xj03HVyUi/Do=
+github.com/go-openapi/spec v0.20.14/go.mod h1:8EOhTpBoFiask8rrgwbLC3zmJfz4zsCUueRuPM6GNkw=
+github.com/go-openapi/swag v0.22.9 h1:XX2DssF+mQKM2DHsbgZK74y/zj4mo9I99+89xUmuZCE=
+github.com/go-openapi/swag v0.22.9/go.mod h1:3/OXnFfnMAwBD099SwYRk7GD3xOrr1iL7d/XNLXVVwE=
 github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
 github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
 github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU=
@@ -562,6 +570,7 @@ github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 h1:rp+c0RAYOWj8
 github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901/go.mod h1:Z86h9688Y0wesXCyonoVr47MasHilkuLMqGhRZ4Hpak=
 github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4=
 github.com/jonboulle/clockwork v0.4.0/go.mod h1:xgRqUGwRcjKCO1vbZUEtSLrqKoPSsUpK7fnezOII0kc=
+github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
 github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
 github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
@@ -1284,5 +1293,5 @@ rsc.io/tmplfunc v0.0.3 h1:53XFQh69AfOa8Tw0Jm7t+GV7KZhOi6jzsCzTtKbMvzU=
 rsc.io/tmplfunc v0.0.3/go.mod h1:AG3sTPzElb1Io3Yg4voV9AGZJuleGAwaVRxL9M49PhA=
 sigs.k8s.io/structured-merge-diff/v3 v3.0.0 h1:dOmIZBMfhcHS09XZkMyUgkq5trg3/jRyJYFZUiaOp8E=
 sigs.k8s.io/structured-merge-diff/v3 v3.0.0/go.mod h1:PlARxl6Hbt/+BC80dRLi1qAmnMqwqDg62YvvVkZjemw=
-sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
-sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
+sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
+sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=

backend/pkg/api/api_test.go

@@ -9,12 +9,14 @@ import (
 	"net/http/httptest"
 	"os"
 	"os/exec"
+	"slices"
 	"sort"
 	"testing"
 	"time"
 
 	embeddedpostgres "github.com/fergusstrange/embedded-postgres"
 	"github.com/gavv/httpexpect/v2"
+	"github.com/go-openapi/spec"
 	"github.com/gobitfly/beaconchain/pkg/api"
 	dataaccess "github.com/gobitfly/beaconchain/pkg/api/data_access"
 	api_types "github.com/gobitfly/beaconchain/pkg/api/types"
@@ -25,6 +27,7 @@ import (
 	"github.com/jmoiron/sqlx"
 	"github.com/pressly/goose/v3"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	"golang.org/x/crypto/bcrypt"
 )
 
@@ -111,6 +114,16 @@ func setup() error {
 		return fmt.Errorf("error inserting user 2: %w", err)
 	}
 
+	// insert dummy api weight for testing
+	_, err = tempDb.Exec(`
+      INSERT INTO api_weights (bucket, endpoint, method, params, weight, valid_from)
+      VALUES ($1, $2, $3, $4, $5, TO_TIMESTAMP($6))`,
+		"default", "/api/v2/test-ratelimit", "GET", "", 2, time.Now().Unix(),
+	)
+	if err != nil {
+		return fmt.Errorf("error inserting api weight: %w", err)
+	}
+
 	cfg := &types.Config{}
 	err = utils.ReadConfig(cfg, *configPath)
 	if err != nil {
@@ -469,3 +482,33 @@ func TestPublicAndSharedDashboards(t *testing.T) {
 		})
 	}
 }
+
+func TestApiDoc(t *testing.T) {
+	e := httpexpect.WithConfig(getExpectConfig(t, ts))
+
+	t.Run("test api doc json", func(t *testing.T) {
+		resp := spec.Swagger{}
+		e.GET("/api/v2/docs/swagger.json").
+			Expect().
+			Status(http.StatusOK).JSON().Decode(&resp)
+
+		assert.Equal(t, "/api/v2", resp.BasePath, "swagger base path should be '/api/v2'")
+		require.NotNil(t, 0, resp.Paths, "swagger paths should not nil")
+		assert.NotEqual(t, 0, len(resp.Paths.Paths), "swagger paths should not be empty")
+		assert.NotEqual(t, 0, len(resp.Definitions), "swagger definitions should not be empty")
+		assert.NotEqual(t, 0, len(resp.Host), "swagger host should not be empty")
+	})
+
+	t.Run("test api ratelimit weights endpoint", func(t *testing.T) {
+		resp := api_types.InternalGetRatelimitWeightsResponse{}
+		e.GET("/api/i/ratelimit-weights").
+			Expect().
+			Status(http.StatusOK).JSON().Decode(&resp)
+
+		assert.GreaterOrEqual(t, len(resp.Data), 1, "ratelimit weights should contain at least one entry")
+		testEndpointIndex := slices.IndexFunc(resp.Data, func(item api_types.ApiWeightItem) bool {
+			return item.Endpoint == "/api/v2/test-ratelimit"
+		})
+		assert.GreaterOrEqual(t, testEndpointIndex, 0, "ratelimit weights should contain an entry for /api/v2/test-ratelimit")
+	})
+}

backend/pkg/api/data_access/app.go

@@ -1,6 +1,7 @@
 package dataaccess
 
 import (
+	"context"
 	"database/sql"
 	"fmt"
 	"time"
@@ -19,6 +20,8 @@ type AppRepository interface {
 	AddMobileNotificationToken(userID uint64, deviceID, notifyToken string) error
 	GetAppSubscriptionCount(userID uint64) (uint64, error)
 	AddMobilePurchase(tx *sql.Tx, userID uint64, paymentDetails t.MobileSubscription, verifyResponse *userservice.VerifyResponse, extSubscriptionId string) error
+	GetLatestBundleForNativeVersion(ctx context.Context, nativeVersion uint64) (*t.MobileAppBundleStats, error)
+	IncrementBundleDeliveryCount(ctx context.Context, bundleVerison uint64) error
 }
 
 // GetUserIdByRefreshToken basically used to confirm the claimed user id with the refresh token. Returns the userId if successful
@@ -105,3 +108,13 @@ func (d *DataAccessService) AddMobilePurchase(tx *sql.Tx, userID uint64, payment
 
 	return err
 }
+
+func (d *DataAccessService) GetLatestBundleForNativeVersion(ctx context.Context, nativeVersion uint64) (*t.MobileAppBundleStats, error) {
+	// @TODO data access
+	return d.dummy.GetLatestBundleForNativeVersion(ctx, nativeVersion)
+}
+
+func (d *DataAccessService) IncrementBundleDeliveryCount(ctx context.Context, bundleVerison uint64) error {
+	// @TODO data access
+	return d.dummy.IncrementBundleDeliveryCount(ctx, bundleVerison)
+}

backend/pkg/api/data_access/data_access.go

@@ -13,7 +13,6 @@ import (
 	"github.com/gobitfly/beaconchain/pkg/commons/db"
 	"github.com/gobitfly/beaconchain/pkg/commons/log"
 	"github.com/gobitfly/beaconchain/pkg/commons/types"
-	"github.com/gobitfly/beaconchain/pkg/commons/utils"
 	"github.com/jmoiron/sqlx"
 	"github.com/pkg/errors"
 )
@@ -29,6 +28,7 @@ type DataAccessor interface {
 	BlockRepository
 	ArchiverRepository
 	ProtocolRepository
+	RatelimitRepository
 	HealthzRepository
 
 	StartDataAccessServices()
@@ -203,19 +203,19 @@ func createDataAccessService(cfg *types.Config) *DataAccessService {
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
-		bt, err := db.InitBigtable(utils.Config.Bigtable.Project, utils.Config.Bigtable.Instance, fmt.Sprintf("%d", utils.Config.Chain.ClConfig.DepositChainID), utils.Config.RedisCacheEndpoint)
+		bt, err := db.InitBigtable(cfg.Bigtable.Project, cfg.Bigtable.Instance, fmt.Sprintf("%d", cfg.Chain.ClConfig.DepositChainID), cfg.RedisCacheEndpoint)
 		if err != nil {
 			log.Fatal(err, "error connecting to bigtable", 0)
 		}
 		dataAccessService.bigtable = bt
 	}()
 
 	// Initialize the tiered cache (redis)
-	if utils.Config.TieredCacheProvider == "redis" || len(utils.Config.RedisCacheEndpoint) != 0 {
+	if cfg.TieredCacheProvider == "redis" || len(cfg.RedisCacheEndpoint) != 0 {
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
-			cache.MustInitTieredCache(utils.Config.RedisCacheEndpoint)
+			cache.MustInitTieredCache(cfg.RedisCacheEndpoint)
 			log.Infof("tiered Cache initialized, latest finalized epoch: %v", cache.LatestFinalizedEpoch.Get())
 		}()
 	}
@@ -225,7 +225,7 @@ func createDataAccessService(cfg *types.Config) *DataAccessService {
 	go func() {
 		defer wg.Done()
 		rdc := redis.NewClient(&redis.Options{
-			Addr:        utils.Config.RedisSessionStoreEndpoint,
+			Addr:        cfg.RedisSessionStoreEndpoint,
 			ReadTimeout: time.Second * 60,
 		})
 
@@ -237,7 +237,7 @@ func createDataAccessService(cfg *types.Config) *DataAccessService {
 
 	wg.Wait()
 
-	if utils.Config.TieredCacheProvider != "redis" {
+	if cfg.TieredCacheProvider != "redis" {
 		log.Fatal(fmt.Errorf("no cache provider set, please set TierdCacheProvider (example redis)"), "", 0)
 	}
 

backend/pkg/api/data_access/dummy.go

@@ -11,7 +11,6 @@ import (
 	"github.com/go-faker/faker/v4"
 	"github.com/go-faker/faker/v4/pkg/options"
 	"github.com/gobitfly/beaconchain/pkg/api/enums"
-	"github.com/gobitfly/beaconchain/pkg/api/types"
 	t "github.com/gobitfly/beaconchain/pkg/api/types"
 	"github.com/gobitfly/beaconchain/pkg/userservice"
 	"github.com/shopspring/decimal"
@@ -44,9 +43,9 @@ func NewDummyService() *DummyService {
 	return &DummyService{}
 }
 
-// generate random decimal.Decimal, should result in somewhere around 0.001 ETH (+/- a few decimal places) in Wei
+// generate random decimal.Decimal, result is between 0.001 and 1000 GWei (returned in Wei)
 func randomEthDecimal() decimal.Decimal {
-	decimal, _ := decimal.NewFromString(fmt.Sprintf("%d00000000000", rand.Int64N(10000000))) //nolint:gosec
+	decimal, _ := decimal.NewFromString(fmt.Sprintf("%d000000", rand.Int64N(1000000)+1)) //nolint:gosec
 	return decimal
 }
 
@@ -453,11 +452,11 @@ func (d *DummyService) GetDashboardNotifications(ctx context.Context, userId uin
 	return getDummyWithPaging[t.NotificationDashboardsTableRow]()
 }
 
-func (d *DummyService) GetValidatorDashboardNotificationDetails(ctx context.Context, notificationId string) (*t.NotificationValidatorDashboardDetail, error) {
+func (d *DummyService) GetValidatorDashboardNotificationDetails(ctx context.Context, dashboardId t.VDBIdPrimary, groupId uint64, epoch uint64) (*t.NotificationValidatorDashboardDetail, error) {
 	return getDummyStruct[t.NotificationValidatorDashboardDetail]()
 }
 
-func (d *DummyService) GetAccountDashboardNotificationDetails(ctx context.Context, notificationId string) (*t.NotificationAccountDashboardDetail, error) {
+func (d *DummyService) GetAccountDashboardNotificationDetails(ctx context.Context, dashboardId uint64, groupId uint64, epoch uint64) (*t.NotificationAccountDashboardDetail, error) {
 	return getDummyStruct[t.NotificationAccountDashboardDetail]()
 }
 
@@ -483,10 +482,10 @@ func (d *DummyService) UpdateNotificationSettingsGeneral(ctx context.Context, us
 func (d *DummyService) UpdateNotificationSettingsNetworks(ctx context.Context, userId uint64, chainId uint64, settings t.NotificationSettingsNetwork) error {
 	return nil
 }
-func (d *DummyService) UpdateNotificationSettingsPairedDevice(ctx context.Context, pairedDeviceId string, name string, IsNotificationsEnabled bool) error {
+func (d *DummyService) UpdateNotificationSettingsPairedDevice(ctx context.Context, userId uint64, pairedDeviceId string, name string, IsNotificationsEnabled bool) error {
 	return nil
 }
-func (d *DummyService) DeleteNotificationSettingsPairedDevice(ctx context.Context, pairedDeviceId string) error {
+func (d *DummyService) DeleteNotificationSettingsPairedDevice(ctx context.Context, userId uint64, pairedDeviceId string) error {
 	return nil
 }
 func (d *DummyService) GetNotificationSettingsDashboards(ctx context.Context, userId uint64, cursor string, colSort t.Sort[enums.NotificationSettingsDashboardColumn], search string, limit uint64) ([]t.NotificationSettingsDashboardsTableRow, *t.Paging, error) {
@@ -637,7 +636,21 @@ func (d *DummyService) GetRocketPoolOverview(ctx context.Context) (*t.RocketPool
 	return getDummyStruct[t.RocketPoolData]()
 }
 
-func (d *DummyService) GetHealthz(ctx context.Context, showAll bool) types.HealthzData {
-	r, _ := getDummyData[types.HealthzData]()
+func (d *DummyService) GetApiWeights(ctx context.Context) ([]t.ApiWeightItem, error) {
+	r := []t.ApiWeightItem{}
+	err := commonFakeData(&r)
+	return r, err
+}
+
+func (d *DummyService) GetHealthz(ctx context.Context, showAll bool) t.HealthzData {
+	r, _ := getDummyData[t.HealthzData]()
 	return r
 }
+
+func (d *DummyService) GetLatestBundleForNativeVersion(ctx context.Context, nativeVersion uint64) (*t.MobileAppBundleStats, error) {
+	return getDummyStruct[t.MobileAppBundleStats]()
+}
+
+func (d *DummyService) IncrementBundleDeliveryCount(ctx context.Context, bundleVerison uint64) error {
+	return nil
+}