diff --git a/.github/workflows/backend-publish-docker.yml b/.github/workflows/backend-publish-docker.yml index 393585908..adfb81822 100644 --- a/.github/workflows/backend-publish-docker.yml +++ b/.github/workflows/backend-publish-docker.yml @@ -46,7 +46,9 @@ jobs: uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - tags: ${{ env.BEACONCHAIN_VERSION }} + tags: | + ${{ env.BEACONCHAIN_VERSION }} + type=ref,event=branch # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages. # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository. # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. diff --git a/.github/workflows/frontend-publish-docker.yml b/.github/workflows/frontend-publish-docker.yml index 1ee6c4391..8c85468f9 100644 --- a/.github/workflows/frontend-publish-docker.yml +++ b/.github/workflows/frontend-publish-docker.yml @@ -32,7 +32,7 @@ jobs: uses: docker/setup-buildx-action@v3 - name: Set version run: | - echo "BEACONCHAIN_VERSION=$(git describe --always --tags)" >> "$GITHUB_ENV" + echo "BEACONCHAIN_VERSION=$(TZ=UTC0 git show --quiet --date='format-local:%Y%m%d%H%M%S' --format="%cd" $GITHUB_SHA)-$(git describe $GITHUB_SHA --always --tags)" >> "$GITHUB_ENV" # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. - name: Log in to the Container registry uses: docker/login-action@65b78e6e13532edd9afa3aa52ac7964289d1a9c1 @@ -46,7 +46,9 @@ jobs: uses: docker/metadata-action@9ec57ed1fcdbf14dcef7dfbe97b2010124a938b7 with: images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} - tags: ${{ env.BEACONCHAIN_VERSION }} + tags: | + ${{ env.BEACONCHAIN_VERSION }} + type=ref,event=branch # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages. # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository. # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. diff --git a/backend/cmd/api/main.go b/backend/cmd/api/main.go index 54f6ccf1c..90ce205f3 100644 --- a/backend/cmd/api/main.go +++ b/backend/cmd/api/main.go @@ -52,7 +52,6 @@ func Run() { // enable light-weight db connection monitoring monitoring.Init(false) monitoring.Start() - defer monitoring.Stop() } var dataAccessor dataaccess.DataAccessor @@ -98,7 +97,7 @@ func Run() { }() utils.WaitForCtrlC() - + monitoring.Stop() // this will emit a clean shutdown event log.Info("shutting down server") if srv != nil { shutDownCtx, cancelShutDownCtx := context.WithTimeout(context.Background(), 10*time.Second) diff --git a/backend/cmd/eth1indexer/main.go b/backend/cmd/eth1indexer/main.go index badcc7745..d6198b22a 100644 --- a/backend/cmd/eth1indexer/main.go +++ b/backend/cmd/eth1indexer/main.go @@ -189,6 +189,10 @@ func Run() { }() } + if *enableEnsUpdater { + go ImportEnsUpdatesLoop(bt, client, *ensBatchSize) + } + if *enableFullBalanceUpdater { ProcessMetadataUpdates(bt, client, balanceUpdaterPrefix, *balanceUpdaterBatchSize, -1) return @@ -375,14 +379,6 @@ func Run() { ProcessMetadataUpdates(bt, client, balanceUpdaterPrefix, *balanceUpdaterBatchSize, 10) } - if *enableEnsUpdater { - err := bt.ImportEnsUpdates(client.GetNativeClient(), *ensBatchSize) - if err != nil { - log.Error(err, "error importing ens updates", 0, nil) - continue - } - } - log.Infof("index run completed") services.ReportStatus("eth1indexer", "Running", nil) } @@ -390,6 +386,19 @@ func Run() { // utils.WaitForCtrlC() } +func ImportEnsUpdatesLoop(bt *db.Bigtable, client *rpc.ErigonClient, batchSize int64) { + time.Sleep(time.Second * 5) + for { + err := bt.ImportEnsUpdates(client.GetNativeClient(), batchSize) + if err != nil { + log.Error(err, "error importing ens updates", 0, nil) + } else { + services.ReportStatus("ensIndexer", "Running", nil) + } + time.Sleep(time.Second * 5) + } +} + func UpdateTokenPrices(bt *db.Bigtable, client *rpc.ErigonClient, tokenListPath string) error { tokenListContent, err := os.ReadFile(tokenListPath) if err != nil { diff --git a/backend/cmd/monitoring/main.go b/backend/cmd/monitoring/main.go index 57df9f824..a4bef3a60 100644 --- a/backend/cmd/monitoring/main.go +++ b/backend/cmd/monitoring/main.go @@ -55,8 +55,8 @@ func Run() { monitoring.Init(true) monitoring.Start() - defer monitoring.Stop() // gotta wait forever utils.WaitForCtrlC() + monitoring.Stop() } diff --git a/backend/cmd/typescript_converter/main.go b/backend/cmd/typescript_converter/main.go index 2a3ce86f5..360e57188 100644 --- a/backend/cmd/typescript_converter/main.go +++ b/backend/cmd/typescript_converter/main.go @@ -21,7 +21,7 @@ const ( ) // Files that should not be converted to TypeScript -var ignoredFiles = []string{"data_access", "search_types"} +var ignoredFiles = []string{"data_access", "search_types", "archiver"} var typeMappings = map[string]string{ "decimal.Decimal": "string /* decimal.Decimal */", diff --git a/backend/pkg/api/data_access/app.go b/backend/pkg/api/data_access/app.go index e3c29d621..812549b1a 100644 --- a/backend/pkg/api/data_access/app.go +++ b/backend/pkg/api/data_access/app.go @@ -1,6 +1,7 @@ package dataaccess import ( + "context" "database/sql" "fmt" "time" @@ -19,6 +20,8 @@ type AppRepository interface { AddMobileNotificationToken(userID uint64, deviceID, notifyToken string) error GetAppSubscriptionCount(userID uint64) (uint64, error) AddMobilePurchase(tx *sql.Tx, userID uint64, paymentDetails t.MobileSubscription, verifyResponse *userservice.VerifyResponse, extSubscriptionId string) error + GetLatestBundleForNativeVersion(ctx context.Context, nativeVersion uint64) (*t.MobileAppBundleStats, error) + IncrementBundleDeliveryCount(ctx context.Context, bundleVerison uint64) error } // GetUserIdByRefreshToken basically used to confirm the claimed user id with the refresh token. Returns the userId if successful @@ -105,3 +108,13 @@ func (d *DataAccessService) AddMobilePurchase(tx *sql.Tx, userID uint64, payment return err } + +func (d *DataAccessService) GetLatestBundleForNativeVersion(ctx context.Context, nativeVersion uint64) (*t.MobileAppBundleStats, error) { + // @TODO data access + return d.dummy.GetLatestBundleForNativeVersion(ctx, nativeVersion) +} + +func (d *DataAccessService) IncrementBundleDeliveryCount(ctx context.Context, bundleVerison uint64) error { + // @TODO data access + return d.dummy.IncrementBundleDeliveryCount(ctx, bundleVerison) +} diff --git a/backend/pkg/api/data_access/dummy.go b/backend/pkg/api/data_access/dummy.go index 6197a72b2..36247ea76 100644 --- a/backend/pkg/api/data_access/dummy.go +++ b/backend/pkg/api/data_access/dummy.go @@ -11,7 +11,6 @@ import ( "github.com/go-faker/faker/v4" "github.com/go-faker/faker/v4/pkg/options" "github.com/gobitfly/beaconchain/pkg/api/enums" - "github.com/gobitfly/beaconchain/pkg/api/types" t "github.com/gobitfly/beaconchain/pkg/api/types" "github.com/gobitfly/beaconchain/pkg/userservice" "github.com/shopspring/decimal" @@ -637,7 +636,15 @@ func (d *DummyService) GetRocketPoolOverview(ctx context.Context) (*t.RocketPool return getDummyStruct[t.RocketPoolData]() } -func (d *DummyService) GetHealthz(ctx context.Context, showAll bool) types.HealthzData { - r, _ := getDummyData[types.HealthzData]() +func (d *DummyService) GetHealthz(ctx context.Context, showAll bool) t.HealthzData { + r, _ := getDummyData[t.HealthzData]() return r } + +func (d *DummyService) GetLatestBundleForNativeVersion(ctx context.Context, nativeVersion uint64) (*t.MobileAppBundleStats, error) { + return getDummyStruct[t.MobileAppBundleStats]() +} + +func (d *DummyService) IncrementBundleDeliveryCount(ctx context.Context, bundleVerison uint64) error { + return nil +} diff --git a/backend/pkg/api/data_access/healthz.go b/backend/pkg/api/data_access/healthz.go index 525ed0db7..303184532 100644 --- a/backend/pkg/api/data_access/healthz.go +++ b/backend/pkg/api/data_access/healthz.go @@ -4,6 +4,7 @@ import ( "context" "slices" + ch "github.com/ClickHouse/clickhouse-go/v2" "github.com/gobitfly/beaconchain/pkg/api/types" "github.com/gobitfly/beaconchain/pkg/commons/db" "github.com/gobitfly/beaconchain/pkg/commons/log" @@ -19,7 +20,17 @@ func (d *DataAccessService) GetHealthz(ctx context.Context, showAll bool) types. var results []types.HealthzResult var response types.HealthzData query := ` - with active_reports as ( + with clean_shutdown_events as ( + SELECT + emitter, + toNullable(inserted_at) as inserted_at + FROM + status_reports + WHERE + deployment_type = {deployment_type:String} + AND inserted_at >= now() - interval 1 days + AND event_id = {clean_shutdown_event_id:String} + ), active_reports as ( SELECT event_id, emitter, @@ -31,7 +42,8 @@ func (d *DataAccessService) GetHealthz(ctx context.Context, showAll bool) types. status, metadata FROM status_reports - WHERE expires_at > now() and deployment_type = ? + LEFT JOIN clean_shutdown_events cse ON status_reports.emitter = clean_shutdown_events.emitter + WHERE expires_at > now() and deployment_type = {deployment_type:String} and (status_reports.inserted_at < cse.inserted_at or cse.inserted_at is null) ORDER BY event_id ASC, emitter ASC, @@ -99,7 +111,7 @@ func (d *DataAccessService) GetHealthz(ctx context.Context, showAll bool) types. response.Reports = make(map[string][]types.HealthzResult) response.ReportingUUID = utils.GetUUID() response.DeploymentType = utils.Config.DeploymentType - err := db.ClickHouseReader.SelectContext(ctx, &results, query, utils.Config.DeploymentType) + err := db.ClickHouseReader.SelectContext(ctx, &results, query, ch.Named("deployment_type", utils.Config.DeploymentType), ch.Named("clean_shutdown_event_id", constants.CleanShutdownEvent)) if err != nil { response.Reports["response_error"] = []types.HealthzResult{ { diff --git a/backend/pkg/api/data_access/user.go b/backend/pkg/api/data_access/user.go index 8f566faac..598f83995 100644 --- a/backend/pkg/api/data_access/user.go +++ b/backend/pkg/api/data_access/user.go @@ -682,6 +682,7 @@ func (d *DataAccessService) GetUserDashboards(ctx context.Context, userId uint64 dbReturn := []struct { Id uint64 `db:"id"` Name string `db:"name"` + Network uint64 `db:"network"` IsArchived sql.NullString `db:"is_archived"` PublicId sql.NullString `db:"public_id"` PublicName sql.NullString `db:"public_name"` @@ -692,6 +693,7 @@ func (d *DataAccessService) GetUserDashboards(ctx context.Context, userId uint64 SELECT uvd.id, uvd.name, + uvd.network, uvd.is_archived, uvds.public_id, uvds.name AS public_name, @@ -709,6 +711,7 @@ func (d *DataAccessService) GetUserDashboards(ctx context.Context, userId uint64 validatorDashboardMap[row.Id] = &t.ValidatorDashboard{ Id: row.Id, Name: row.Name, + Network: row.Network, PublicIds: []t.VDBPublicId{}, IsArchived: row.IsArchived.Valid, ArchivedReason: row.IsArchived.String, diff --git a/backend/pkg/api/data_access/vdb_management.go b/backend/pkg/api/data_access/vdb_management.go index 3d4aa939d..9163a51ab 100644 --- a/backend/pkg/api/data_access/vdb_management.go +++ b/backend/pkg/api/data_access/vdb_management.go @@ -67,6 +67,7 @@ func (d *DataAccessService) GetValidatorDashboardInfo(ctx context.Context, dashb wg.Go(func() error { dbReturn := []struct { Name string `db:"name"` + Network uint64 `db:"network"` IsArchived sql.NullString `db:"is_archived"` PublicId sql.NullString `db:"public_id"` PublicName sql.NullString `db:"public_name"` @@ -76,6 +77,7 @@ func (d *DataAccessService) GetValidatorDashboardInfo(ctx context.Context, dashb err := d.alloyReader.SelectContext(ctx, &dbReturn, ` SELECT uvd.name, + uvd.network, uvd.is_archived, uvds.public_id, uvds.name AS public_name, @@ -95,6 +97,7 @@ func (d *DataAccessService) GetValidatorDashboardInfo(ctx context.Context, dashb mutex.Lock() result.Id = uint64(dashboardId) result.Name = dbReturn[0].Name + result.Network = dbReturn[0].Network result.IsArchived = dbReturn[0].IsArchived.Valid result.ArchivedReason = dbReturn[0].IsArchived.String @@ -273,6 +276,20 @@ func (d *DataAccessService) GetValidatorDashboardOverview(ctx context.Context, d data := t.VDBOverviewData{} eg := errgroup.Group{} var err error + + // Network + if dashboardId.Validators == nil { + eg.Go(func() error { + query := `SELECT network + FROM + users_val_dashboards + WHERE + id = $1` + return d.alloyReader.GetContext(ctx, &data.Network, query, dashboardId.Id) + }) + } + // TODO handle network of validator set dashboards + // Groups if dashboardId.Validators == nil && !dashboardId.AggregateGroups { // should have valid primary id diff --git a/backend/pkg/api/handlers/internal.go b/backend/pkg/api/handlers/internal.go index 1c4157290..d111ab7ca 100644 --- a/backend/pkg/api/handlers/internal.go +++ b/backend/pkg/api/handlers/internal.go @@ -461,6 +461,52 @@ func (h *HandlerService) InternalGetValidatorDashboardRocketPoolMinipools(w http h.PublicGetValidatorDashboardRocketPoolMinipools(w, r) } +// -------------------------------------- +// Mobile + +func (h *HandlerService) InternalGetMobileLatestBundle(w http.ResponseWriter, r *http.Request) { + var v validationError + q := r.URL.Query() + force := v.checkBool(q.Get("force"), "force") + bundleVersion := v.checkUint(q.Get("bundle_version"), "bundle_version") + nativeVersion := v.checkUint(q.Get("native_version"), "native_version") + if v.hasErrors() { + handleErr(w, r, v) + return + } + stats, err := h.dai.GetLatestBundleForNativeVersion(r.Context(), nativeVersion) + if err != nil { + handleErr(w, r, err) + return + } + var data types.MobileBundleData + data.HasNativeUpdateAvailable = stats.MaxNativeVersion > nativeVersion + // if given bundle version is smaller than the latest and delivery count is less than target count, return the latest bundle + if force || (bundleVersion < stats.LatestBundleVersion && (stats.TargetCount == 0 || stats.DeliveryCount < stats.TargetCount)) { + data.BundleUrl = stats.BundleUrl + } + response := types.GetMobileLatestBundleResponse{ + Data: data, + } + returnOk(w, r, response) +} + +func (h *HandlerService) InternalPostMobileBundleDeliveries(w http.ResponseWriter, r *http.Request) { + var v validationError + vars := mux.Vars(r) + bundleVersion := v.checkUint(vars["bundle_version"], "bundle_version") + if v.hasErrors() { + handleErr(w, r, v) + return + } + err := h.dai.IncrementBundleDeliveryCount(r.Context(), bundleVersion) + if err != nil { + handleErr(w, r, err) + return + } + returnNoContent(w, r) +} + // -------------------------------------- // Notifications diff --git a/backend/pkg/api/router.go b/backend/pkg/api/router.go index 5603e4856..0b3d6a819 100644 --- a/backend/pkg/api/router.go +++ b/backend/pkg/api/router.go @@ -93,6 +93,8 @@ func addRoutes(hs *handlers.HandlerService, publicRouter, internalRouter *mux.Ro {http.MethodGet, "/mobile/authorize", nil, hs.InternalPostMobileAuthorize}, {http.MethodPost, "/mobile/equivalent-exchange", nil, hs.InternalPostMobileEquivalentExchange}, {http.MethodPost, "/mobile/purchase", nil, hs.InternalHandleMobilePurchase}, + {http.MethodGet, "/mobile/latest-bundle", nil, hs.InternalGetMobileLatestBundle}, + {http.MethodPost, "/mobile/bundles/{bundle_version}/deliveries", nil, hs.InternalPostMobileBundleDeliveries}, {http.MethodPost, "/logout", nil, hs.InternalPostLogout}, diff --git a/backend/pkg/api/types/dashboard.go b/backend/pkg/api/types/dashboard.go index eaf49a431..d3e334722 100644 --- a/backend/pkg/api/types/dashboard.go +++ b/backend/pkg/api/types/dashboard.go @@ -7,6 +7,7 @@ type AccountDashboard struct { type ValidatorDashboard struct { Id uint64 `json:"id"` Name string `json:"name"` + Network uint64 `json:"network"` PublicIds []VDBPublicId `json:"public_ids,omitempty"` IsArchived bool `json:"is_archived"` ArchivedReason string `json:"archived_reason,omitempty" tstype:"'user' | 'dashboard_limit' | 'validator_limit' | 'group_limit'"` diff --git a/backend/pkg/api/types/data_access.go b/backend/pkg/api/types/data_access.go index fb1257f62..49d88b926 100644 --- a/backend/pkg/api/types/data_access.go +++ b/backend/pkg/api/types/data_access.go @@ -227,3 +227,14 @@ type HealthzData struct { DeploymentType string `json:"deployment_type"` Reports map[string][]HealthzResult `json:"status_reports"` } + +// ------------------------- +// Mobile structs + +type MobileAppBundleStats struct { + LatestBundleVersion uint64 + BundleUrl string + TargetCount uint64 // coalesce to 0 if column is null + DeliveryCount uint64 + MaxNativeVersion uint64 // the max native version of the whole table for the given environment +} diff --git a/backend/pkg/api/types/mobile.go b/backend/pkg/api/types/mobile.go new file mode 100644 index 000000000..7f84a999d --- /dev/null +++ b/backend/pkg/api/types/mobile.go @@ -0,0 +1,8 @@ +package types + +type MobileBundleData struct { + BundleUrl string `json:"bundle_url,omitempty"` + HasNativeUpdateAvailable bool `json:"has_native_update_available"` +} + +type GetMobileLatestBundleResponse ApiDataResponse[MobileBundleData] diff --git a/backend/pkg/api/types/validator_dashboard.go b/backend/pkg/api/types/validator_dashboard.go index cef626e57..39d5e5d0d 100644 --- a/backend/pkg/api/types/validator_dashboard.go +++ b/backend/pkg/api/types/validator_dashboard.go @@ -28,6 +28,7 @@ type VDBOverviewBalances struct { type VDBOverviewData struct { Name string `json:"name,omitempty"` + Network uint64 `json:"network"` Groups []VDBOverviewGroup `json:"groups"` Validators VDBOverviewValidators `json:"validators"` Efficiency PeriodicValues[float64] `json:"efficiency"` diff --git a/backend/pkg/commons/log/log.go b/backend/pkg/commons/log/log.go index dc9c11b0b..60684f6cb 100644 --- a/backend/pkg/commons/log/log.go +++ b/backend/pkg/commons/log/log.go @@ -36,12 +36,7 @@ func Infof(format string, args ...interface{}) { } func InfoWithFields(additionalInfos Fields, msg string) { - logFields := logrus.NewEntry(logrus.New()) - for name, info := range additionalInfos { - logFields = logFields.WithField(name, info) - } - - logFields.Info(msg) + logrus.WithFields(additionalInfos).Info(msg) } func Warn(args ...interface{}) { @@ -53,12 +48,7 @@ func Warnf(format string, args ...interface{}) { } func WarnWithFields(additionalInfos Fields, msg string) { - logFields := logrus.NewEntry(logrus.New()) - for name, info := range additionalInfos { - logFields = logFields.WithField(name, info) - } - - logFields.Warn(msg) + logrus.WithFields(additionalInfos).Warn(msg) } func Tracef(format string, args ...interface{}) { @@ -66,21 +56,11 @@ func Tracef(format string, args ...interface{}) { } func TraceWithFields(additionalInfos Fields, msg string) { - logFields := logrus.NewEntry(logrus.New()) - for name, info := range additionalInfos { - logFields = logFields.WithField(name, info) - } - - logFields.Trace(msg) + logrus.WithFields(additionalInfos).Trace(msg) } func DebugWithFields(additionalInfos Fields, msg string) { - logFields := logrus.NewEntry(logrus.New()) - for name, info := range additionalInfos { - logFields = logFields.WithField(name, info) - } - - logFields.Debug(msg) + logrus.WithFields(additionalInfos).Debug(msg) } func Debugf(format string, args ...interface{}) { @@ -96,7 +76,7 @@ func logErrorInfo(err error, callerSkip int, additionalInfos ...Fields) *logrus. } pc, fullFilePath, line, ok := runtime.Caller(callerSkip + 2) if ok { - logFields = logFields.WithFields(logrus.Fields{ + logFields = logFields.WithFields(Fields{ "_file": filepath.Base(fullFilePath), "_function": runtime.FuncForPC(pc).Name(), "_line": line, @@ -152,4 +132,4 @@ func logErrorInfo(err error, callerSkip int, additionalInfos ...Fields) *logrus. return logFields } -type Fields map[string]interface{} +type Fields = logrus.Fields diff --git a/backend/pkg/commons/utils/uuid.go b/backend/pkg/commons/utils/uuid.go index 6932f1c76..5142a1143 100644 --- a/backend/pkg/commons/utils/uuid.go +++ b/backend/pkg/commons/utils/uuid.go @@ -6,6 +6,7 @@ import ( "github.com/bwmarrin/snowflake" "github.com/gobitfly/beaconchain/pkg/commons/log" "github.com/google/uuid" + "golang.org/x/exp/rand" ) // uuid that you can get - gets set to a random value on startup/first read @@ -30,7 +31,8 @@ func GetSnowflake() int64 { return v.(*snowflake.Node).Generate().Int64() } - node, err := snowflake.NewNode(1) + nodeId := rand.Int63() & 0xFF + node, err := snowflake.NewNode(nodeId) if err != nil { log.Fatal(err, "snowflake generator failed to start", 0) return 0 diff --git a/backend/pkg/monitoring/constants/main.go b/backend/pkg/monitoring/constants/main.go index 7ffa891c2..9e20f037b 100644 --- a/backend/pkg/monitoring/constants/main.go +++ b/backend/pkg/monitoring/constants/main.go @@ -11,3 +11,5 @@ const ( Failure StatusType = "failure" Default time.Duration = -1 * time.Second ) + +const CleanShutdownEvent = "clean_shutdown" diff --git a/backend/pkg/monitoring/monitoring.go b/backend/pkg/monitoring/monitoring.go index f814d3a66..dbf021a77 100644 --- a/backend/pkg/monitoring/monitoring.go +++ b/backend/pkg/monitoring/monitoring.go @@ -1,19 +1,30 @@ package monitoring import ( + "sync" + "sync/atomic" + "github.com/gobitfly/beaconchain/pkg/commons/db" + "github.com/gobitfly/beaconchain/pkg/commons/log" "github.com/gobitfly/beaconchain/pkg/commons/metrics" "github.com/gobitfly/beaconchain/pkg/commons/types" "github.com/gobitfly/beaconchain/pkg/commons/utils" + "github.com/gobitfly/beaconchain/pkg/monitoring/constants" "github.com/gobitfly/beaconchain/pkg/monitoring/services" ) var monitoredServices []services.Service +var startedClickhouse atomic.Bool +var initMutex = sync.Mutex{} func Init(full bool) { + initMutex.Lock() + defer initMutex.Unlock() metrics.UUID.WithLabelValues(utils.GetUUID()).Set(1) // so we can find out where the uuid is set metrics.DeploymentType.WithLabelValues(utils.Config.DeploymentType).Set(1) if db.ClickHouseNativeWriter == nil { + log.Infof("initializing clickhouse writer") + startedClickhouse.Store(true) db.ClickHouseNativeWriter = db.MustInitClickhouseNative(&types.DatabaseConfig{ Username: utils.Config.ClickHouse.WriterDatabase.Username, Password: utils.Config.ClickHouse.WriterDatabase.Password, @@ -33,6 +44,7 @@ func Init(full bool) { &services.ServiceClickhouseRollings{}, &services.ServiceClickhouseEpoch{}, &services.ServiceTimeoutDetector{}, + &services.CleanShutdownSpamDetector{}, ) } @@ -42,13 +54,20 @@ func Init(full bool) { } func Start() { + log.Infof("starting monitoring services") for _, service := range monitoredServices { service.Start() } } func Stop() { + log.Infof("stopping monitoring services") for _, service := range monitoredServices { service.Stop() } + // this prevents status reports that werent shut down cleanly from triggering alerts + services.NewStatusReport(constants.CleanShutdownEvent, constants.Default, constants.Default)(constants.Success, nil) + if startedClickhouse.Load() { + db.ClickHouseNativeWriter.Close() + } } diff --git a/backend/pkg/monitoring/services/base.go b/backend/pkg/monitoring/services/base.go index 8bea11b47..04cbea094 100644 --- a/backend/pkg/monitoring/services/base.go +++ b/backend/pkg/monitoring/services/base.go @@ -3,11 +3,8 @@ package services import ( "context" "fmt" - "os" - "os/signal" "sync" "sync/atomic" - "syscall" "time" "github.com/gobitfly/beaconchain/pkg/commons/db" @@ -45,31 +42,13 @@ func (s *ServiceBase) Stop() { s.wg.Wait() } -var isShuttingDown atomic.Bool -var once sync.Once - -func autoGracefulStop() { - c := make(chan os.Signal, 1) - signal.Notify(c, os.Interrupt, syscall.SIGINT, syscall.SIGTERM) - <-c - isShuttingDown.Store(true) -} - func NewStatusReport(id string, timeout time.Duration, check_interval time.Duration) func(status constants.StatusType, metadata map[string]string) { runId := uuid.New().String() - // run if it hasnt started yet - once.Do(func() { go autoGracefulStop() }) return func(status constants.StatusType, metadata map[string]string) { // acquire snowflake synchronously flake := utils.GetSnowflake() - + now := time.Now() go func() { - // check if we are alive - if isShuttingDown.Load() { - log.Info("shutting down, not reporting status") - return - } - if metadata == nil { metadata = make(map[string]string) } @@ -82,20 +61,29 @@ func NewStatusReport(id string, timeout time.Duration, check_interval time.Durat ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() - timeouts_at := time.Now().Add(1 * time.Minute) + timeouts_at := now.Add(1 * time.Minute) if timeout != constants.Default { - timeouts_at = time.Now().Add(timeout) + timeouts_at = now.Add(timeout) } - expires_at := timeouts_at.Add(5 * time.Minute) - if check_interval >= 5*time.Minute { + expires_at := timeouts_at.Add(1 * time.Minute) + if check_interval >= 1*time.Minute { expires_at = timeouts_at.Add(check_interval) } + log.TraceWithFields(log.Fields{ + "emitter": id, + "event_id": utils.GetUUID(), + "deployment_type": utils.Config.DeploymentType, + "insert_id": flake, + "expires_at": expires_at, + "timeouts_at": timeouts_at, + "metadata": metadata, + }, "sending status report") var err error if db.ClickHouseNativeWriter != nil { err = db.ClickHouseNativeWriter.AsyncInsert( ctx, "INSERT INTO status_reports (emitter, event_id, deployment_type, insert_id, expires_at, timeouts_at, metadata) VALUES (?, ?, ?, ?, ?, ?, ?)", - true, + false, // true means wait for settlement, but we want to shoot and forget. false does mean we cant log any errors that occur during settlement utils.GetUUID(), id, utils.Config.DeploymentType, diff --git a/backend/pkg/monitoring/services/clean_shutdown_spam.go b/backend/pkg/monitoring/services/clean_shutdown_spam.go new file mode 100644 index 000000000..0dcf01391 --- /dev/null +++ b/backend/pkg/monitoring/services/clean_shutdown_spam.go @@ -0,0 +1,86 @@ +package services + +import ( + "context" + "encoding/json" + "strconv" + "time" + + "github.com/gobitfly/beaconchain/pkg/commons/db" + "github.com/gobitfly/beaconchain/pkg/commons/log" + "github.com/gobitfly/beaconchain/pkg/commons/utils" + "github.com/gobitfly/beaconchain/pkg/monitoring/constants" +) + +type CleanShutdownSpamDetector struct { + ServiceBase +} + +func (s *CleanShutdownSpamDetector) Start() { + if !s.running.CompareAndSwap(false, true) { + // already running, return error + return + } + s.wg.Add(1) + go s.internalProcess() +} + +func (s *CleanShutdownSpamDetector) internalProcess() { + defer s.wg.Done() + s.runChecks() + for { + select { + case <-s.ctx.Done(): + return + case <-time.After(30 * time.Second): + s.runChecks() + } + } +} + +func (s *CleanShutdownSpamDetector) runChecks() { + id := "monitoring_clean_shutdown_spam" + r := NewStatusReport(id, constants.Default, 30*time.Second) + r(constants.Running, nil) + if db.ClickHouseReader == nil { + r(constants.Failure, map[string]string{"error": "clickhouse reader is nil"}) + // ignore + return + } + log.Tracef("checking clean shutdown spam") + + query := ` + SELECT + emitter + FROM + status_reports + WHERE + deployment_type = ? + AND inserted_at >= now() - interval 5 minutes + AND event_id = ? + ` + ctx, cancel := context.WithTimeout(s.ctx, 15*time.Second) + defer cancel() + var emitters []string + err := db.ClickHouseReader.SelectContext(ctx, &emitters, query, utils.Config.DeploymentType, constants.CleanShutdownEvent) + if err != nil { + r(constants.Failure, map[string]string{"error": err.Error()}) + return + } + threshold := 10 + md := map[string]string{ + "count": strconv.Itoa(len(emitters)), + "threshold": strconv.Itoa(threshold), + } + if len(emitters) > threshold { + payload, err := json.Marshal(emitters) + if err != nil { + r(constants.Failure, map[string]string{"error": err.Error()}) + return + } + md["emitters"] = string(payload) + r(constants.Failure, md) + return + } + r(constants.Success, md) +} diff --git a/backend/pkg/monitoring/services/clickhouse_rollings.go b/backend/pkg/monitoring/services/clickhouse_rollings.go index 259bf5a44..8bab09dc1 100644 --- a/backend/pkg/monitoring/services/clickhouse_rollings.go +++ b/backend/pkg/monitoring/services/clickhouse_rollings.go @@ -71,7 +71,7 @@ func (s *ServiceClickhouseRollings) runChecks() { err := db.ClickHouseReader.GetContext(ctx, &tsEpochTable, ` SELECT max(epoch_timestamp) - FROM holesky.validator_dashboard_data_epoch`, + FROM validator_dashboard_data_epoch`, ) if err != nil { r(constants.Failure, map[string]string{"error": err.Error()}) @@ -81,7 +81,7 @@ func (s *ServiceClickhouseRollings) runChecks() { err = db.ClickHouseReader.GetContext(ctx, &epochRollingTable, fmt.Sprintf(` SELECT max(epoch_end) - FROM holesky.validator_dashboard_data_rolling_%s`, + FROM validator_dashboard_data_rolling_%s`, rolling, ), ) diff --git a/backend/pkg/monitoring/services/timeout_detector.go b/backend/pkg/monitoring/services/timeout_detector.go index 720bb80dc..83f85ccd3 100644 --- a/backend/pkg/monitoring/services/timeout_detector.go +++ b/backend/pkg/monitoring/services/timeout_detector.go @@ -61,7 +61,7 @@ func (s *ServiceTimeoutDetector) runChecks() { status, metadata FROM status_reports - WHERE expires_at > now() and deployment_type = ? + WHERE expires_at > now() and deployment_type = ? and emitter not in (select distinct emitter from status_reports where event_id = ? and inserted_at > now() - interval 1 days) ORDER BY event_id ASC, emitter ASC, @@ -87,6 +87,7 @@ func (s *ServiceTimeoutDetector) runChecks() { ) SELECT event_id, + emitter, status, inserted_at, expires_at, @@ -101,13 +102,14 @@ func (s *ServiceTimeoutDetector) runChecks() { defer cancel() var victims []struct { EventID string `db:"event_id"` + Emitter string `db:"emitter"` Status string `db:"status"` InsertedAt time.Time `db:"inserted_at"` ExpiresAt time.Time `db:"expires_at"` TimeoutsAt time.Time `db:"timeouts_at"` Metadata map[string]string `db:"metadata"` } - err := db.ClickHouseReader.SelectContext(ctx, &victims, query, utils.Config.DeploymentType) + err := db.ClickHouseReader.SelectContext(ctx, &victims, query, utils.Config.DeploymentType, constants.CleanShutdownEvent) if err != nil { r(constants.Failure, map[string]string{"error": err.Error()}) return diff --git a/backend/pkg/userservice/appsubscription_oracle.go b/backend/pkg/userservice/appsubscription_oracle.go index 0b078674e..0f50ed916 100644 --- a/backend/pkg/userservice/appsubscription_oracle.go +++ b/backend/pkg/userservice/appsubscription_oracle.go @@ -233,13 +233,29 @@ func rejectReason(valid bool) string { return "expired" } +// first 3 trillion dollar company and you can't reuse ids +func mapAppleProductID(productID string) string { + mappings := map[string]string{ + "orca.yearly.apple": "orca.yearly", + "orca.apple": "orca", + "dolphin.yearly.apple": "dolphin.yearly", + "dolphin.apple": "dolphin", + "guppy.yearly.apple": "guppy.yearly", + "guppy.apple": "guppy", + } + if mapped, ok := mappings[productID]; ok { + return mapped + } + return productID +} + func verifyApple(apple *api.StoreClient, receipt *types.PremiumData) (*VerifyResponse, error) { response := &VerifyResponse{ Valid: false, ExpirationDate: 0, RejectReason: "", - ProductID: receipt.ProductID, // may be changed by this function to be different than receipt.ProductID - Receipt: receipt.Receipt, // may be changed by this function to be different than receipt.Receipt + ProductID: mapAppleProductID(receipt.ProductID), // may be changed by this function to be different than receipt.ProductID + Receipt: receipt.Receipt, // may be changed by this function to be different than receipt.Receipt } if apple == nil { @@ -300,7 +316,7 @@ func verifyApple(apple *api.StoreClient, receipt *types.PremiumData) (*VerifyRes response.RejectReason = "invalid_product_id" return response, nil } - response.ProductID = productId // update response to reflect the resolved product id + response.ProductID = mapAppleProductID(productId) // update response to reflect the resolved product id expiresDateFloat, ok := claims["expiresDate"].(float64) if !ok { diff --git a/frontend/.env-example b/frontend/.env-example index 563b8dc92..8e5b8de1b 100644 --- a/frontend/.env-example +++ b/frontend/.env-example @@ -11,4 +11,5 @@ NUXT_PUBLIC_V1_DOMAIN: "" NUXT_PUBLIC_LOG_FILE: "" NUXT_PUBLIC_CHAIN_ID_BY_DEFAULT: "" NUXT_PUBLIC_MAINTENANCE_TS: "1717700652" +NUXT_PUBLIC_DEPLOYMENT_TYPE: "development" PRIVATE_SSR_SECRET: "" diff --git a/frontend/.vscode/settings.json b/frontend/.vscode/settings.json index 2ddbc4e8a..a97dbcf2a 100644 --- a/frontend/.vscode/settings.json +++ b/frontend/.vscode/settings.json @@ -1,17 +1,18 @@ { "conventionalCommits.scopes": [ - "checkout", - "ci", - "customFetch", - "DashboardChartSummaryChartFilter", - "DashboardGroupManagementModal", - "eslint", - "git", - "i18n", - "mainHeader", - "qrCode", - "vscode" -], + "checkout", + "ci", + "customFetch", + "DashboardChartSummaryChartFilter", + "DashboardGroupManagementModal", + "eslint", + "git", + "i18n", + "mainHeader", + "qrCode", + "vscode", + "DashboardValidatorManagmentModal" + ], "editor.codeActionsOnSave": { "source.fixAll.eslint": "always" }, diff --git a/frontend/components/dashboard/DashboardHeader.vue b/frontend/components/dashboard/DashboardHeader.vue index 8db7f0d54..1e1f0b3e1 100644 --- a/frontend/components/dashboard/DashboardHeader.vue +++ b/frontend/components/dashboard/DashboardHeader.vue @@ -90,10 +90,12 @@ const items = computed(() => { const cd = db as CookieDashboard return createMenuBarButton('validator', getDashboardName(cd), `${cd.hash !== undefined ? cd.hash : cd.id}`) })) - addToSortedItems($t('dashboard.header.account'), dashboards.value?.validator_dashboards?.slice(0, 1).map((db) => { - const cd = db as CookieDashboard - return createMenuBarButton('account', getDashboardName(cd), `${cd.hash ?? cd.id}`) - })) + if (showInDevelopment) { + addToSortedItems($t('dashboard.header.account'), dashboards.value?.validator_dashboards?.slice(0, 1).map((db) => { + const cd = db as CookieDashboard + return createMenuBarButton('account', getDashboardName(cd), `${cd.hash ?? cd.id}`) + })) + } const disabledTooltip = !showInDevelopment ? $t('common.coming_soon') : undefined const onNotificationsPage = dashboardType.value === 'notifications' addToSortedItems($t('notifications.title'), [ { diff --git a/frontend/components/dashboard/ValidatorManagementModal.vue b/frontend/components/dashboard/DashboardValidatorManagementModal.vue similarity index 98% rename from frontend/components/dashboard/ValidatorManagementModal.vue rename to frontend/components/dashboard/DashboardValidatorManagementModal.vue index f64946de6..94762a647 100644 --- a/frontend/components/dashboard/ValidatorManagementModal.vue +++ b/frontend/components/dashboard/DashboardValidatorManagementModal.vue @@ -553,6 +553,15 @@ const premiumLimit = computed( " /> +
+
+ {{ $t("dashboard.validator.col.status") }} +
+ +
{{ $t("dashboard.validator.col.withdrawal_credential") }} @@ -645,6 +654,7 @@ const premiumLimit = computed( display: flex; flex-direction: column; overflow-y: hidden; + justify-content: space-between; :deep(.p-datatable-wrapper) { flex-grow: 1; diff --git a/frontend/components/notifications/NotificationsClientsTable.vue b/frontend/components/notifications/NotificationsClientsTable.vue new file mode 100644 index 000000000..470396516 --- /dev/null +++ b/frontend/components/notifications/NotificationsClientsTable.vue @@ -0,0 +1,189 @@ + + + + + diff --git a/frontend/locales/en.json b/frontend/locales/en.json index 33f2e6705..302cfa091 100644 --- a/frontend/locales/en.json +++ b/frontend/locales/en.json @@ -561,6 +561,17 @@ "yes": "Yes" }, "notifications": { + "clients": { + "col": { + "client_name": "Client ", + "version": "Version" + }, + "footer":{ + "subscriptions": "Clients ({count} Subscriptions)" + }, + "search_placeholder":"Client", + "title": "Clients" + }, "col": { "dashboard": "Dashboard", "group": "Group", diff --git a/frontend/nuxt.config.ts b/frontend/nuxt.config.ts index 8c932276c..81bb64d3a 100644 --- a/frontend/nuxt.config.ts +++ b/frontend/nuxt.config.ts @@ -78,6 +78,7 @@ export default defineNuxtConfig({ apiClient: process.env.PUBLIC_API_CLIENT, apiKey: process.env.PUBLIC_API_KEY, chainIdByDefault: process.env.PUBLIC_CHAIN_ID_BY_DEFAULT, + deploymentType: process.env.PUBLIC_DEPLOYMENT_TYPE, domain: process.env.PUBLIC_DOMAIN, gitVersion, legacyApiClient: process.env.PUBLIC_LEGACY_API_CLIENT, diff --git a/frontend/pages/notifications.vue b/frontend/pages/notifications.vue index 968b4fe73..5d591e793 100644 --- a/frontend/pages/notifications.vue +++ b/frontend/pages/notifications.vue @@ -97,7 +97,8 @@ const openManageNotifications = () => { />
{ @open-dialog="openManageNotifications" /> +
diff --git a/frontend/public/mock/notifications/managementDashboard.json b/frontend/public/mock/notifications/managementDashboard.json deleted file mode 100644 index eb0d14c72..000000000 --- a/frontend/public/mock/notifications/managementDashboard.json +++ /dev/null @@ -1,31 +0,0 @@ -{ - "paging": { - "total_count": 999 - }, - "data": [ - { - "group_id": 123, - "dashboard_id": 222, - "dashboard_name": "My test dashboard", - "dashboard_type": "validator", - "subscriptions": ["missed_attestation"], - "webhook": { - "url": "", - "via_discord": false - }, - "networks": [1] - }, - { - "group_id": 123, - "dashboard_id": 222, - "dashboard_name": "My account dashboard", - "dashboard_type": "account", - "subscriptions": ["missed_attestation", "proposed_attestation"], - "webhook": { - "url": "https://discord.com/some-webhook-link", - "via_discord": false - }, - "networks": [1, 10, 42161, 8453] - } - ] -} diff --git a/frontend/stores/dashboard/useUserDashboardStore.ts b/frontend/stores/dashboard/useUserDashboardStore.ts index a8cd42ddf..0d79f1d4d 100644 --- a/frontend/stores/dashboard/useUserDashboardStore.ts +++ b/frontend/stores/dashboard/useUserDashboardStore.ts @@ -121,6 +121,7 @@ export function useUserDashboardStore() { id: res.data.id, is_archived: false, name: res.data.name, + network: res.data.network, validator_count: 0, }, ], diff --git a/frontend/stores/notifications/useNotificationsClientsStore.ts b/frontend/stores/notifications/useNotificationsClientsStore.ts new file mode 100644 index 000000000..d5dbe9a9d --- /dev/null +++ b/frontend/stores/notifications/useNotificationsClientsStore.ts @@ -0,0 +1,69 @@ +import { defineStore } from 'pinia' +import type { InternalGetUserNotificationClientsResponse } from '~/types/api/notifications' +import { API_PATH } from '~/types/customFetch' +import type { TableQueryParams } from '~/types/datatable' + +const notificationsClientStore = defineStore('notifications-clients-store', () => { + const data = ref() + return { data } +}) + +export function useNotificationsClientStore() { + const { isLoggedIn } = useUserStore() + + const { fetch } = useCustomFetch() + const { data } = storeToRefs(notificationsClientStore()) + const { + cursor, isStoredQuery, onSort, pageSize, pendingQuery, query, setCursor, setPageSize, setSearch, setStoredQuery, + } = useTableQuery({ + limit: 10, sort: 'timestamp:desc', + }, 10) + const isLoading = ref(false) + + async function loadClientsNotifications(q: TableQueryParams) { + isLoading.value = true + setStoredQuery(q) + try { + const result = await fetch( + API_PATH.NOTIFICATIONS_CLIENTS, + undefined, + undefined, + q, + ) + + isLoading.value = false + if (!isStoredQuery(q)) { + return // in case some query params change while loading + } + + data.value = result + } + catch (e) { + data.value = undefined + isLoading.value = false + } + return data.value + } + + const clientsNotifications = computed(() => { + return data.value + }) + + watch(query, (q) => { + if (q) { + isLoggedIn.value && loadClientsNotifications(q) + } + }, { immediate: true }) + + return { + clientsNotifications, + cursor, + isLoading, + onSort, + pageSize, + query: pendingQuery, + setCursor, + setPageSize, + setSearch, + } +} diff --git a/frontend/types/api/archiver.ts b/frontend/types/api/archiver.ts new file mode 100644 index 000000000..a0ecfe481 --- /dev/null +++ b/frontend/types/api/archiver.ts @@ -0,0 +1,16 @@ +// Code generated by tygo. DO NOT EDIT. +/* eslint-disable */ + +////////// +// source: archiver.go + +export interface ArchiverDashboard { + DashboardId: number /* uint64 */; + IsArchived: boolean; + GroupCount: number /* uint64 */; + ValidatorCount: number /* uint64 */; +} +export interface ArchiverDashboardArchiveReason { + DashboardId: number /* uint64 */; + ArchivedReason: any /* enums.VDBArchivedReason */; +} diff --git a/frontend/types/api/dashboard.ts b/frontend/types/api/dashboard.ts index 3ab7c3768..16a840350 100644 --- a/frontend/types/api/dashboard.ts +++ b/frontend/types/api/dashboard.ts @@ -12,6 +12,7 @@ export interface AccountDashboard { export interface ValidatorDashboard { id: number /* uint64 */; name: string; + network: number /* uint64 */; public_ids?: VDBPublicId[]; is_archived: boolean; archived_reason?: 'user' | 'dashboard_limit' | 'validator_limit' | 'group_limit'; diff --git a/frontend/types/api/mobile.ts b/frontend/types/api/mobile.ts new file mode 100644 index 000000000..d6b234a18 --- /dev/null +++ b/frontend/types/api/mobile.ts @@ -0,0 +1,12 @@ +// Code generated by tygo. DO NOT EDIT. +/* eslint-disable */ +import type { ApiDataResponse } from './common' + +////////// +// source: mobile.go + +export interface MobileBundleData { + bundle_url?: string; + has_native_update_available: boolean; +} +export type GetMobileLatestBundleResponse = ApiDataResponse; diff --git a/frontend/types/api/validator_dashboard.ts b/frontend/types/api/validator_dashboard.ts index af31ad96f..10fb9d3e1 100644 --- a/frontend/types/api/validator_dashboard.ts +++ b/frontend/types/api/validator_dashboard.ts @@ -28,6 +28,7 @@ export interface VDBOverviewBalances { } export interface VDBOverviewData { name?: string; + network: number /* uint64 */; groups: VDBOverviewGroup[]; validators: VDBOverviewValidators; efficiency: PeriodicValues; diff --git a/frontend/types/customFetch.ts b/frontend/types/customFetch.ts index dcc3ec474..18b2d3ea1 100644 --- a/frontend/types/customFetch.ts +++ b/frontend/types/customFetch.ts @@ -40,6 +40,7 @@ export enum API_PATH { LATEST_STATE = '/latestState', LOGIN = '/login', LOGOUT = '/logout', + NOTIFICATIONS_CLIENTS = '/notifications/clients', NOTIFICATIONS_DASHBOARDS = '/notifications/dashboards', NOTIFICATIONS_MACHINE = '/notifications/machines', NOTIFICATIONS_MANAGEMENT_GENERAL = '/notifications/managementGeneral', @@ -279,6 +280,10 @@ export const mapping: Record = { mock: false, path: '/logout', }, + [API_PATH.NOTIFICATIONS_CLIENTS]: { + method: 'GET', + path: '/users/me/notifications/clients', + }, [API_PATH.NOTIFICATIONS_DASHBOARDS]: { path: '/users/me/notifications/dashboards', },