Merge branch 'staging' into NOBIDS/local-deployment-update

.git-blame-ignore-revs

@@ -4,6 +4,8 @@
 # in order to work execute once:
 # git config blame.ignoreRevsFile .git-blame-ignore-revs
 
+#chore(eslint): add `natural sorting` for `json` files 
+f3da99c5c685eae1914aad8513d3b4b2f1cdcaa2
 # style(eslint): add `typescript delimiter` rule
 d6b42edb3f687cad3082a9044bcb71fc39a46176
 # style(eslint): apply `max-len` rule

.github/workflows/backend-integration-test.yml

@@ -0,0 +1,41 @@
+name: Backend-Integration-Test
+on:
+  push:
+    paths:
+      - 'backend/**'
+    branches:
+      - main
+      - staging
+  pull_request:
+    paths:
+      - 'backend/**'
+    branches:
+      - '*'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+  pull-requests: read
+  checks: write
+
+jobs:
+  build:
+    name: integration-test
+    runs-on: self-hosted
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v4
+        with:
+          go-version-file: 'backend/go.mod'
+          cache-dependency-path: 'backend/go.sum'
+      - name: Test with the Go CLI
+        working-directory: backend
+        run:
+          go install github.com/swaggo/swag/cmd/swag@latest && swag init --ot json -o ./pkg/api/docs -d ./pkg/api/ -g ./handlers/public.go
+          go test -failfast ./pkg/api/... -config "${{ secrets.CI_CONFIG_PATH }}"
+
+
+

.github/workflows/backend-publish-docker.yml

@@ -6,6 +6,7 @@ on:
   push:
     paths:
       - 'backend/**'
+      - '.github/**'
     branches:
       - main
       - staging
@@ -27,6 +28,9 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
+      - name: Set version
+        run: |
+          echo "BEACONCHAIN_VERSION=$(TZ=UTC0 git show --quiet --date='format-local:%Y%m%d%H%M%S' --format="%cd" $GITHUB_SHA)-$(git describe $GITHUB_SHA --always --tags)" >> "$GITHUB_ENV"
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
       # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here.
@@ -42,6 +46,9 @@ jobs:
         uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            ${{ env.BEACONCHAIN_VERSION }}
+            type=ref,event=branch
       # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages.
       # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository.
       # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step.

.github/workflows/frontend-publish-docker.yml

@@ -6,6 +6,7 @@ on:
   push:
     paths:
       - 'frontend/**'
+      - '.github/**'
     branches:
       - main
       - staging
@@ -31,7 +32,7 @@ jobs:
         uses: docker/setup-buildx-action@v3
       - name: Set version
         run: |
-          echo "BEACONCHAIN_VERSION=$(git describe --always --tags)" >> "$GITHUB_ENV"
+          echo "BEACONCHAIN_VERSION=$(TZ=UTC0 git show --quiet --date='format-local:%Y%m%d%H%M%S' --format="%cd" $GITHUB_SHA)-$(git describe $GITHUB_SHA --always --tags)" >> "$GITHUB_ENV"
       # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here.
       - name: Log in to the Container registry
         uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
@@ -45,6 +46,9 @@ jobs:
         uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            ${{ env.BEACONCHAIN_VERSION }}
+            type=ref,event=branch
       # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages.
       # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository.
       # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step.

backend/.gitignore

@@ -5,4 +5,5 @@ local_deployment/config.yml
 local_deployment/elconfig.json
 local_deployment/.env
 __gitignore
-cmd/playground
+cmd/playground
+pkg/api/docs/swagger.json

backend/Makefile

@@ -10,8 +10,8 @@ CGO_CFLAGS_ALLOW="-O -D__BLST_PORTABLE__"
 
 all: 
 	mkdir -p bin
+	go install github.com/swaggo/swag/cmd/swag@latest && swag init --ot json -o ./pkg/api/docs -d ./pkg/api/ -g ./handlers/public.go
 	CGO_CFLAGS=${CGO_CFLAGS} CGO_CFLAGS_ALLOW=${CGO_CFLAGS_ALLOW} go build --ldflags=${LDFLAGS} -o ./bin/bc ./cmd/main.go
-
 clean:
 	rm -rf bin
 

backend/cmd/api/main.go

@@ -52,14 +52,14 @@ func Run() {
 		// enable light-weight db connection monitoring
 		monitoring.Init(false)
 		monitoring.Start()
-		defer monitoring.Stop()
 	}
 
 	var dataAccessor dataaccess.DataAccessor
 	if dummyApi {
 		dataAccessor = dataaccess.NewDummyService()
 	} else {
 		dataAccessor = dataaccess.NewDataAccessService(cfg)
+		dataAccessor.StartDataAccessServices()
 	}
 	defer dataAccessor.Close()
 
@@ -70,7 +70,7 @@ func Run() {
 		router.Use(metrics.HttpMiddleware)
 		go func() {
 			log.Infof("serving metrics on %v", utils.Config.Metrics.Address)
-			if err := metrics.Serve(utils.Config.Metrics.Address, utils.Config.Metrics.Pprof); err != nil {
+			if err := metrics.Serve(utils.Config.Metrics.Address, utils.Config.Metrics.Pprof, utils.Config.Metrics.PprofExtra); err != nil {
 				log.Fatal(err, "error serving metrics", 0)
 			}
 		}()
@@ -97,7 +97,7 @@ func Run() {
 	}()
 
 	utils.WaitForCtrlC()
-
+	monitoring.Stop() // this will emit a clean shutdown event
 	log.Info("shutting down server")
 	if srv != nil {
 		shutDownCtx, cancelShutDownCtx := context.WithTimeout(context.Background(), 10*time.Second)

backend/cmd/archiver/main.go

@@ -0,0 +1,47 @@
+package archiver
+
+import (
+	"flag"
+	"os"
+
+	dataaccess "github.com/gobitfly/beaconchain/pkg/api/data_access"
+
+	"github.com/gobitfly/beaconchain/pkg/commons/log"
+	"github.com/gobitfly/beaconchain/pkg/commons/types"
+	"github.com/gobitfly/beaconchain/pkg/commons/utils"
+	"github.com/gobitfly/beaconchain/pkg/commons/version"
+
+	"github.com/gobitfly/beaconchain/pkg/archiver"
+)
+
+func Run() {
+	fs := flag.NewFlagSet("fs", flag.ExitOnError)
+	configPath := fs.String("config", "", "Path to the config file, if empty string defaults will be used")
+	versionFlag := fs.Bool("version", false, "Show version and exit")
+	_ = fs.Parse(os.Args[2:])
+
+	if *versionFlag {
+		log.Info(version.Version)
+		log.Info(version.GoVersion)
+		return
+	}
+
+	cfg := &types.Config{}
+	err := utils.ReadConfig(cfg, *configPath)
+	if err != nil {
+		log.Fatal(err, "error reading config file", 0)
+	}
+	utils.Config = cfg
+
+	log.InfoWithFields(log.Fields{"config": *configPath, "version": version.Version, "commit": version.GitCommit, "chainName": utils.Config.Chain.ClConfig.ConfigName}, "starting")
+
+	dataAccessor := dataaccess.NewDataAccessService(cfg)
+	defer dataAccessor.Close()
+
+	archiver, err := archiver.NewArchiver(dataAccessor)
+	if err != nil {
+		log.Fatal(err, "error initializing archiving service", 0)
+	}
+	go archiver.Start()
+	utils.WaitForCtrlC()
+}

backend/cmd/blobindexer/main.go

@@ -16,7 +16,7 @@ import (
 func Run() {
 	fs := flag.NewFlagSet("fs", flag.ExitOnError)
 
-	configFlag := fs.String("config", "config.yml", "path to config")
+	configFlag := fs.String("config", "", "path to config")
 	versionFlag := fs.Bool("version", false, "print version and exit")
 	_ = fs.Parse(os.Args[2:])
 	if *versionFlag {
@@ -31,7 +31,7 @@ func Run() {
 	if utils.Config.Metrics.Enabled {
 		go func() {
 			log.Infof("serving metrics on %v", utils.Config.Metrics.Address)
-			if err := metrics.Serve(utils.Config.Metrics.Address, utils.Config.Metrics.Pprof); err != nil {
+			if err := metrics.Serve(utils.Config.Metrics.Address, utils.Config.Metrics.Pprof, utils.Config.Metrics.PprofExtra); err != nil {
 				log.Fatal(err, "error serving metrics", 0)
 			}
 		}()

backend/cmd/eth1indexer/main.go

@@ -96,7 +96,7 @@ func Run() {
 	if utils.Config.Metrics.Enabled {
 		go func() {
 			log.Infof("serving metrics on %v", utils.Config.Metrics.Address)
-			if err := metrics.Serve(utils.Config.Metrics.Address, utils.Config.Metrics.Pprof); err != nil {
+			if err := metrics.Serve(utils.Config.Metrics.Address, utils.Config.Metrics.Pprof, utils.Config.Metrics.PprofExtra); err != nil {
 				log.Fatal(err, "error serving metrics", 0)
 			}
 		}()
@@ -189,6 +189,10 @@ func Run() {
 		}()
 	}
 
+	if *enableEnsUpdater {
+		go ImportEnsUpdatesLoop(bt, client, *ensBatchSize)
+	}
+
 	if *enableFullBalanceUpdater {
 		ProcessMetadataUpdates(bt, client, balanceUpdaterPrefix, *balanceUpdaterBatchSize, -1)
 		return
@@ -375,21 +379,26 @@ func Run() {
 			ProcessMetadataUpdates(bt, client, balanceUpdaterPrefix, *balanceUpdaterBatchSize, 10)
 		}
 
-		if *enableEnsUpdater {
-			err := bt.ImportEnsUpdates(client.GetNativeClient(), *ensBatchSize)
-			if err != nil {
-				log.Error(err, "error importing ens updates", 0, nil)
-				continue
-			}
-		}
-
 		log.Infof("index run completed")
 		services.ReportStatus("eth1indexer", "Running", nil)
 	}
 
 	// utils.WaitForCtrlC()
 }
 
+func ImportEnsUpdatesLoop(bt *db.Bigtable, client *rpc.ErigonClient, batchSize int64) {
+	time.Sleep(time.Second * 5)
+	for {
+		err := bt.ImportEnsUpdates(client.GetNativeClient(), batchSize)
+		if err != nil {
+			log.Error(err, "error importing ens updates", 0, nil)
+		} else {
+			services.ReportStatus("ensIndexer", "Running", nil)
+		}
+		time.Sleep(time.Second * 5)
+	}
+}
+
 func UpdateTokenPrices(bt *db.Bigtable, client *rpc.ErigonClient, tokenListPath string) error {
 	tokenListContent, err := os.ReadFile(tokenListPath)
 	if err != nil {

backend/cmd/ethstore_exporter/main.go

@@ -55,7 +55,7 @@ func Run() {
 	if utils.Config.Metrics.Enabled {
 		go func() {
 			log.Infof("serving metrics on %v", utils.Config.Metrics.Address)
-			if err := metrics.Serve(utils.Config.Metrics.Address, utils.Config.Metrics.Pprof); err != nil {
+			if err := metrics.Serve(utils.Config.Metrics.Address, utils.Config.Metrics.Pprof, utils.Config.Metrics.PprofExtra); err != nil {
 				log.Fatal(err, "error serving metrics", 0)
 			}
 		}()

backend/cmd/exporter/main.go

@@ -46,7 +46,7 @@ func Run() {
 	if utils.Config.Metrics.Enabled {
 		go func() {
 			log.Infof("serving metrics on %v", utils.Config.Metrics.Address)
-			if err := metrics.Serve(utils.Config.Metrics.Address, utils.Config.Metrics.Pprof); err != nil {
+			if err := metrics.Serve(utils.Config.Metrics.Address, utils.Config.Metrics.Pprof, utils.Config.Metrics.PprofExtra); err != nil {
 				log.Fatal(err, "error serving metrics", 0)
 			}
 		}()

backend/cmd/main.go

@@ -5,6 +5,7 @@ import (
 	"os"
 
 	"github.com/gobitfly/beaconchain/cmd/api"
+	"github.com/gobitfly/beaconchain/cmd/archiver"
 	"github.com/gobitfly/beaconchain/cmd/blobindexer"
 	"github.com/gobitfly/beaconchain/cmd/eth1indexer"
 	"github.com/gobitfly/beaconchain/cmd/ethstore_exporter"
@@ -32,6 +33,8 @@ func main() {
 	switch target {
 	case "api":
 		api.Run()
+	case "archiver":
+		archiver.Run()
 	case "blobindexer":
 		blobindexer.Run()
 	case "eth1indexer":