Merge branch 'staging' into BIDS-3208/dashboard_notifications_backend

.github/workflows/backend-publish-docker.yml

@@ -46,7 +46,9 @@ jobs:
         uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
-          tags: ${{ env.BEACONCHAIN_VERSION }}
+          tags: |
+            ${{ env.BEACONCHAIN_VERSION }}
+            type=ref,event=branch
       # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages.
       # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository.
       # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step.

.github/workflows/frontend-publish-docker.yml

@@ -32,7 +32,7 @@ jobs:
         uses: docker/setup-buildx-action@v3
       - name: Set version
         run: |
-          echo "BEACONCHAIN_VERSION=$(git describe --always --tags)" >> "$GITHUB_ENV"
+          echo "BEACONCHAIN_VERSION=$(TZ=UTC0 git show --quiet --date='format-local:%Y%m%d%H%M%S' --format="%cd" $GITHUB_SHA)-$(git describe $GITHUB_SHA --always --tags)" >> "$GITHUB_ENV"
       # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here.
       - name: Log in to the Container registry
         uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1
@@ -46,7 +46,9 @@ jobs:
         uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
-          tags: ${{ env.BEACONCHAIN_VERSION }}
+          tags: |
+            ${{ env.BEACONCHAIN_VERSION }}
+            type=ref,event=branch
       # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages.
       # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository.
       # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step.

backend/cmd/api/main.go

@@ -52,7 +52,6 @@ func Run() {
 		// enable light-weight db connection monitoring
 		monitoring.Init(false)
 		monitoring.Start()
-		defer monitoring.Stop()
 	}
 
 	var dataAccessor dataaccess.DataAccessor
@@ -98,7 +97,7 @@ func Run() {
 	}()
 
 	utils.WaitForCtrlC()
-
+	monitoring.Stop() // this will emit a clean shutdown event
 	log.Info("shutting down server")
 	if srv != nil {
 		shutDownCtx, cancelShutDownCtx := context.WithTimeout(context.Background(), 10*time.Second)

backend/cmd/eth1indexer/main.go

@@ -189,6 +189,10 @@ func Run() {
 		}()
 	}
 
+	if *enableEnsUpdater {
+		go ImportEnsUpdatesLoop(bt, client, *ensBatchSize)
+	}
+
 	if *enableFullBalanceUpdater {
 		ProcessMetadataUpdates(bt, client, balanceUpdaterPrefix, *balanceUpdaterBatchSize, -1)
 		return
@@ -375,21 +379,26 @@ func Run() {
 			ProcessMetadataUpdates(bt, client, balanceUpdaterPrefix, *balanceUpdaterBatchSize, 10)
 		}
 
-		if *enableEnsUpdater {
-			err := bt.ImportEnsUpdates(client.GetNativeClient(), *ensBatchSize)
-			if err != nil {
-				log.Error(err, "error importing ens updates", 0, nil)
-				continue
-			}
-		}
-
 		log.Infof("index run completed")
 		services.ReportStatus("eth1indexer", "Running", nil)
 	}
 
 	// utils.WaitForCtrlC()
 }
 
+func ImportEnsUpdatesLoop(bt *db.Bigtable, client *rpc.ErigonClient, batchSize int64) {
+	time.Sleep(time.Second * 5)
+	for {
+		err := bt.ImportEnsUpdates(client.GetNativeClient(), batchSize)
+		if err != nil {
+			log.Error(err, "error importing ens updates", 0, nil)
+		} else {
+			services.ReportStatus("ensIndexer", "Running", nil)
+		}
+		time.Sleep(time.Second * 5)
+	}
+}
+
 func UpdateTokenPrices(bt *db.Bigtable, client *rpc.ErigonClient, tokenListPath string) error {
 	tokenListContent, err := os.ReadFile(tokenListPath)
 	if err != nil {

backend/cmd/monitoring/main.go

@@ -55,8 +55,8 @@ func Run() {
 
 	monitoring.Init(true)
 	monitoring.Start()
-	defer monitoring.Stop()
 
 	// gotta wait forever
 	utils.WaitForCtrlC()
+	monitoring.Stop()
 }

backend/cmd/typescript_converter/main.go

@@ -21,7 +21,7 @@ const (
 )
 
 // Files that should not be converted to TypeScript
-var ignoredFiles = []string{"data_access", "search_types"}
+var ignoredFiles = []string{"data_access", "search_types", "archiver"}
 
 var typeMappings = map[string]string{
 	"decimal.Decimal": "string /* decimal.Decimal */",

backend/pkg/api/data_access/app.go

@@ -1,6 +1,7 @@
 package dataaccess
 
 import (
+	"context"
 	"database/sql"
 	"fmt"
 	"time"
@@ -19,6 +20,8 @@ type AppRepository interface {
 	AddMobileNotificationToken(userID uint64, deviceID, notifyToken string) error
 	GetAppSubscriptionCount(userID uint64) (uint64, error)
 	AddMobilePurchase(tx *sql.Tx, userID uint64, paymentDetails t.MobileSubscription, verifyResponse *userservice.VerifyResponse, extSubscriptionId string) error
+	GetLatestBundleForNativeVersion(ctx context.Context, nativeVersion uint64) (*t.MobileAppBundleStats, error)
+	IncrementBundleDeliveryCount(ctx context.Context, bundleVerison uint64) error
 }
 
 // GetUserIdByRefreshToken basically used to confirm the claimed user id with the refresh token. Returns the userId if successful
@@ -105,3 +108,13 @@ func (d *DataAccessService) AddMobilePurchase(tx *sql.Tx, userID uint64, payment
 
 	return err
 }
+
+func (d *DataAccessService) GetLatestBundleForNativeVersion(ctx context.Context, nativeVersion uint64) (*t.MobileAppBundleStats, error) {
+	// @TODO data access
+	return d.dummy.GetLatestBundleForNativeVersion(ctx, nativeVersion)
+}
+
+func (d *DataAccessService) IncrementBundleDeliveryCount(ctx context.Context, bundleVerison uint64) error {
+	// @TODO data access
+	return d.dummy.IncrementBundleDeliveryCount(ctx, bundleVerison)
+}

backend/pkg/api/data_access/dummy.go

@@ -11,7 +11,6 @@ import (
 	"github.com/go-faker/faker/v4"
 	"github.com/go-faker/faker/v4/pkg/options"
 	"github.com/gobitfly/beaconchain/pkg/api/enums"
-	"github.com/gobitfly/beaconchain/pkg/api/types"
 	t "github.com/gobitfly/beaconchain/pkg/api/types"
 	"github.com/gobitfly/beaconchain/pkg/userservice"
 	"github.com/shopspring/decimal"
@@ -637,7 +636,15 @@ func (d *DummyService) GetRocketPoolOverview(ctx context.Context) (*t.RocketPool
 	return getDummyStruct[t.RocketPoolData]()
 }
 
-func (d *DummyService) GetHealthz(ctx context.Context, showAll bool) types.HealthzData {
-	r, _ := getDummyData[types.HealthzData]()
+func (d *DummyService) GetHealthz(ctx context.Context, showAll bool) t.HealthzData {
+	r, _ := getDummyData[t.HealthzData]()
 	return r
 }
+
+func (d *DummyService) GetLatestBundleForNativeVersion(ctx context.Context, nativeVersion uint64) (*t.MobileAppBundleStats, error) {
+	return getDummyStruct[t.MobileAppBundleStats]()
+}
+
+func (d *DummyService) IncrementBundleDeliveryCount(ctx context.Context, bundleVerison uint64) error {
+	return nil
+}

backend/pkg/api/data_access/healthz.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"slices"
 
+	ch "github.com/ClickHouse/clickhouse-go/v2"
 	"github.com/gobitfly/beaconchain/pkg/api/types"
 	"github.com/gobitfly/beaconchain/pkg/commons/db"
 	"github.com/gobitfly/beaconchain/pkg/commons/log"
@@ -19,7 +20,17 @@ func (d *DataAccessService) GetHealthz(ctx context.Context, showAll bool) types.
 	var results []types.HealthzResult
 	var response types.HealthzData
 	query := `
-		with active_reports as (
+		with clean_shutdown_events as (
+			SELECT
+				emitter,
+				toNullable(inserted_at) as inserted_at
+			FROM
+				status_reports
+			WHERE
+				deployment_type = {deployment_type:String}
+				AND inserted_at >= now() - interval 1 days
+				AND event_id = {clean_shutdown_event_id:String}
+		), active_reports as (
 			SELECT
 				event_id,
 				emitter,
@@ -31,7 +42,8 @@ func (d *DataAccessService) GetHealthz(ctx context.Context, showAll bool) types.
 				status,
 				metadata
 			FROM status_reports
-			WHERE expires_at > now() and deployment_type = ?
+			LEFT JOIN clean_shutdown_events cse ON status_reports.emitter = clean_shutdown_events.emitter
+			WHERE expires_at > now() and deployment_type = {deployment_type:String} and (status_reports.inserted_at < cse.inserted_at or cse.inserted_at is null)
 			ORDER BY
 				event_id ASC,
 				emitter ASC,
@@ -99,7 +111,7 @@ func (d *DataAccessService) GetHealthz(ctx context.Context, showAll bool) types.
 	response.Reports = make(map[string][]types.HealthzResult)
 	response.ReportingUUID = utils.GetUUID()
 	response.DeploymentType = utils.Config.DeploymentType
-	err := db.ClickHouseReader.SelectContext(ctx, &results, query, utils.Config.DeploymentType)
+	err := db.ClickHouseReader.SelectContext(ctx, &results, query, ch.Named("deployment_type", utils.Config.DeploymentType), ch.Named("clean_shutdown_event_id", constants.CleanShutdownEvent))
 	if err != nil {
 		response.Reports["response_error"] = []types.HealthzResult{
 			{

backend/pkg/api/data_access/user.go

@@ -682,6 +682,7 @@ func (d *DataAccessService) GetUserDashboards(ctx context.Context, userId uint64
 		dbReturn := []struct {
 			Id           uint64         `db:"id"`
 			Name         string         `db:"name"`
+			Network      uint64         `db:"network"`
 			IsArchived   sql.NullString `db:"is_archived"`
 			PublicId     sql.NullString `db:"public_id"`
 			PublicName   sql.NullString `db:"public_name"`
@@ -692,6 +693,7 @@ func (d *DataAccessService) GetUserDashboards(ctx context.Context, userId uint64
 		SELECT
 			uvd.id,
 			uvd.name,
+			uvd.network,
 			uvd.is_archived,
 			uvds.public_id,
 			uvds.name AS public_name,
@@ -709,6 +711,7 @@ func (d *DataAccessService) GetUserDashboards(ctx context.Context, userId uint64
 				validatorDashboardMap[row.Id] = &t.ValidatorDashboard{
 					Id:             row.Id,
 					Name:           row.Name,
+					Network:        row.Network,
 					PublicIds:      []t.VDBPublicId{},
 					IsArchived:     row.IsArchived.Valid,
 					ArchivedReason: row.IsArchived.String,

backend/pkg/api/data_access/vdb_management.go

@@ -67,6 +67,7 @@ func (d *DataAccessService) GetValidatorDashboardInfo(ctx context.Context, dashb
 	wg.Go(func() error {
 		dbReturn := []struct {
 			Name         string         `db:"name"`
+			Network      uint64         `db:"network"`
 			IsArchived   sql.NullString `db:"is_archived"`
 			PublicId     sql.NullString `db:"public_id"`
 			PublicName   sql.NullString `db:"public_name"`
@@ -76,6 +77,7 @@ func (d *DataAccessService) GetValidatorDashboardInfo(ctx context.Context, dashb
 		err := d.alloyReader.SelectContext(ctx, &dbReturn, `
 		SELECT
 			uvd.name,
+			uvd.network,
 			uvd.is_archived,
 			uvds.public_id,
 			uvds.name AS public_name,
@@ -95,6 +97,7 @@ func (d *DataAccessService) GetValidatorDashboardInfo(ctx context.Context, dashb
 		mutex.Lock()
 		result.Id = uint64(dashboardId)
 		result.Name = dbReturn[0].Name
+		result.Network = dbReturn[0].Network
 		result.IsArchived = dbReturn[0].IsArchived.Valid
 		result.ArchivedReason = dbReturn[0].IsArchived.String
 
@@ -273,6 +276,20 @@ func (d *DataAccessService) GetValidatorDashboardOverview(ctx context.Context, d
 	data := t.VDBOverviewData{}
 	eg := errgroup.Group{}
 	var err error
+
+	// Network
+	if dashboardId.Validators == nil {
+		eg.Go(func() error {
+			query := `SELECT network
+			FROM
+				users_val_dashboards
+			WHERE
+				id = $1`
+			return d.alloyReader.GetContext(ctx, &data.Network, query, dashboardId.Id)
+		})
+	}
+	// TODO handle network of validator set dashboards
+
 	// Groups
 	if dashboardId.Validators == nil && !dashboardId.AggregateGroups {
 		// should have valid primary id

backend/pkg/api/handlers/internal.go

@@ -461,6 +461,52 @@ func (h *HandlerService) InternalGetValidatorDashboardRocketPoolMinipools(w http
 	h.PublicGetValidatorDashboardRocketPoolMinipools(w, r)
 }
 
+// --------------------------------------
+// Mobile
+
+func (h *HandlerService) InternalGetMobileLatestBundle(w http.ResponseWriter, r *http.Request) {
+	var v validationError
+	q := r.URL.Query()
+	force := v.checkBool(q.Get("force"), "force")
+	bundleVersion := v.checkUint(q.Get("bundle_version"), "bundle_version")
+	nativeVersion := v.checkUint(q.Get("native_version"), "native_version")
+	if v.hasErrors() {
+		handleErr(w, r, v)
+		return
+	}
+	stats, err := h.dai.GetLatestBundleForNativeVersion(r.Context(), nativeVersion)
+	if err != nil {
+		handleErr(w, r, err)
+		return
+	}
+	var data types.MobileBundleData
+	data.HasNativeUpdateAvailable = stats.MaxNativeVersion > nativeVersion
+	// if given bundle version is smaller than the latest and delivery count is less than target count, return the latest bundle
+	if force || (bundleVersion < stats.LatestBundleVersion && (stats.TargetCount == 0 || stats.DeliveryCount < stats.TargetCount)) {
+		data.BundleUrl = stats.BundleUrl
+	}
+	response := types.GetMobileLatestBundleResponse{
+		Data: data,
+	}
+	returnOk(w, r, response)
+}
+
+func (h *HandlerService) InternalPostMobileBundleDeliveries(w http.ResponseWriter, r *http.Request) {
+	var v validationError
+	vars := mux.Vars(r)
+	bundleVersion := v.checkUint(vars["bundle_version"], "bundle_version")
+	if v.hasErrors() {
+		handleErr(w, r, v)
+		return
+	}
+	err := h.dai.IncrementBundleDeliveryCount(r.Context(), bundleVersion)
+	if err != nil {
+		handleErr(w, r, err)
+		return
+	}
+	returnNoContent(w, r)
+}
+
 // --------------------------------------
 // Notifications
 

backend/pkg/api/router.go

@@ -93,6 +93,8 @@ func addRoutes(hs *handlers.HandlerService, publicRouter, internalRouter *mux.Ro
 		{http.MethodGet, "/mobile/authorize", nil, hs.InternalPostMobileAuthorize},
 		{http.MethodPost, "/mobile/equivalent-exchange", nil, hs.InternalPostMobileEquivalentExchange},
 		{http.MethodPost, "/mobile/purchase", nil, hs.InternalHandleMobilePurchase},
+		{http.MethodGet, "/mobile/latest-bundle", nil, hs.InternalGetMobileLatestBundle},
+		{http.MethodPost, "/mobile/bundles/{bundle_version}/deliveries", nil, hs.InternalPostMobileBundleDeliveries},
 
 		{http.MethodPost, "/logout", nil, hs.InternalPostLogout},
 

backend/pkg/api/types/dashboard.go

@@ -7,6 +7,7 @@ type AccountDashboard struct {
 type ValidatorDashboard struct {
 	Id             uint64        `json:"id"`
 	Name           string        `json:"name"`
+	Network        uint64        `json:"network"`
 	PublicIds      []VDBPublicId `json:"public_ids,omitempty"`
 	IsArchived     bool          `json:"is_archived"`
 	ArchivedReason string        `json:"archived_reason,omitempty" tstype:"'user' | 'dashboard_limit' | 'validator_limit' | 'group_limit'"`

backend/pkg/api/types/data_access.go

@@ -227,3 +227,14 @@ type HealthzData struct {
 	DeploymentType    string                     `json:"deployment_type"`
 	Reports           map[string][]HealthzResult `json:"status_reports"`
 }
+
+// -------------------------
+// Mobile structs
+
+type MobileAppBundleStats struct {
+	LatestBundleVersion uint64
+	BundleUrl           string
+	TargetCount         uint64 // coalesce to 0 if column is null
+	DeliveryCount       uint64
+	MaxNativeVersion    uint64 // the max native version of the whole table for the given environment
+}

backend/pkg/api/types/mobile.go

@@ -0,0 +1,8 @@
+package types
+
+type MobileBundleData struct {
+	BundleUrl                string `json:"bundle_url,omitempty"`
+	HasNativeUpdateAvailable bool   `json:"has_native_update_available"`
+}
+
+type GetMobileLatestBundleResponse ApiDataResponse[MobileBundleData]

backend/pkg/api/types/validator_dashboard.go

@@ -28,6 +28,7 @@ type VDBOverviewBalances struct {
 
 type VDBOverviewData struct {
 	Name                string                                     `json:"name,omitempty"`
+	Network             uint64                                     `json:"network"`
 	Groups              []VDBOverviewGroup                         `json:"groups"`
 	Validators          VDBOverviewValidators                      `json:"validators"`
 	Efficiency          PeriodicValues[float64]                    `json:"efficiency"`