From 5968845cd8d997bf5d6072447f8da582e81c3224 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Fri, 5 Jul 2024 08:09:58 +0200 Subject: [PATCH 01/94] use map for pubkey filter --- backend/pkg/notification/db.go | 9 +++---- backend/pkg/notification/notifications.go | 32 ++++++++++++++++------- 2 files changed, 26 insertions(+), 15 deletions(-) diff --git a/backend/pkg/notification/db.go b/backend/pkg/notification/db.go index eed83b0f1..cf2a217e0 100644 --- a/backend/pkg/notification/db.go +++ b/backend/pkg/notification/db.go @@ -1,15 +1,13 @@ package notification import ( - "encoding/hex" - "github.com/gobitfly/beaconchain/pkg/commons/db" "github.com/gobitfly/beaconchain/pkg/commons/types" "github.com/gobitfly/beaconchain/pkg/commons/utils" "github.com/lib/pq" ) -func GetSubsForEventFilter(eventName types.EventName) ([][]byte, map[string][]types.Subscription, error) { +func GetSubsForEventFilter(eventName types.EventName) (map[string]bool, map[string][]types.Subscription, error) { var subs []types.Subscription subQuery := ` SELECT id, user_id, event_filter, last_sent_epoch, created_epoch, event_threshold, ENCODE(unsubscribe_hash, 'hex') as unsubscribe_hash, internal_state from users_subscriptions where event_name = $1 @@ -21,7 +19,7 @@ func GetSubsForEventFilter(eventName types.EventName) ([][]byte, map[string][]ty return nil, nil, err } - filtersEncode := make([][]byte, 0, len(subs)) + filtersEncode := make(map[string]bool, len(subs)) for _, sub := range subs { if _, ok := subMap[sub.EventFilter]; !ok { subMap[sub.EventFilter] = make([]types.Subscription, 0) @@ -36,8 +34,7 @@ func GetSubsForEventFilter(eventName types.EventName) ([][]byte, map[string][]ty State: sub.State, }) - b, _ := hex.DecodeString(sub.EventFilter) - filtersEncode = append(filtersEncode, b) + filtersEncode[sub.EventFilter] = true } return filtersEncode, subMap, nil } diff --git a/backend/pkg/notification/notifications.go b/backend/pkg/notification/notifications.go index e40a83b44..ec0e841a9 100644 --- a/backend/pkg/notification/notifications.go +++ b/backend/pkg/notification/notifications.go @@ -2959,20 +2959,34 @@ func collectRocketpoolRPLCollateralNotifications(notificationsByUserID map[uint6 RPLStakeMax BigFloat `db:"max_rpl_stake"` } + // filter nodes with no minipools (anymore) because they have min/max stake of 0 + // TODO properly remove notification entry from db stakeInfoPerNode := make([]dbResult, 0) batchSize := 5000 - dataLen := len(pubkeys) - for i := 0; i < dataLen; i += batchSize { - var keys [][]byte - start := i - end := i + batchSize - - if dataLen < end { - end = dataLen + keys := make([][]byte, 0, batchSize) + for pubkey := range pubkeys { + b, err := hex.DecodeString(pubkey) + if err != nil { + log.Error(err, fmt.Sprintf("error decoding pubkey %s", pubkey), 0) + continue } + keys = append(keys, b) - keys = pubkeys[start:end] + if len(keys) > batchSize { + var partial []dbResult + err = db.WriterDb.Select(&partial, ` + SELECT address, rpl_stake, min_rpl_stake, max_rpl_stake + FROM rocketpool_nodes + WHERE address = ANY($1) AND min_rpl_stake != 0 AND max_rpl_stake != 0`, pq.ByteaArray(keys)) + if err != nil { + return err + } + stakeInfoPerNode = append(stakeInfoPerNode, partial...) + keys = make([][]byte, 0, batchSize) + } + } + if len(keys) > 0 { var partial []dbResult // filter nodes with no minipools (anymore) because they have min/max stake of 0 From 94b87dc9682c6dae3948018c16a2354ec0294d69 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Fri, 5 Jul 2024 08:25:01 +0200 Subject: [PATCH 02/94] simplify handling of pubkeys in notifications --- backend/pkg/commons/types/frontend.go | 16 ++++++++++++++++ backend/pkg/notification/db.go | 13 +++++++------ backend/pkg/notification/notifications.go | 12 ++++++------ 3 files changed, 29 insertions(+), 12 deletions(-) diff --git a/backend/pkg/commons/types/frontend.go b/backend/pkg/commons/types/frontend.go index ee589fcbb..f468b5c43 100644 --- a/backend/pkg/commons/types/frontend.go +++ b/backend/pkg/commons/types/frontend.go @@ -266,6 +266,22 @@ type Subscription struct { EventThreshold float64 `db:"event_threshold"` UnsubscribeHash sql.NullString `db:"unsubscribe_hash" swaggertype:"string"` State sql.NullString `db:"internal_state" swaggertype:"string"` + GroupId *uint32 + DashboardId *uint32 +} + +type ValidatorDashboardConfig struct { + DashboardsByUserId map[uint64]map[uint32]*ValidatorDashboard +} + +type ValidatorDashboard struct { + Name string `db:"name"` + Groups map[uint32]*ValidatorDashboardGroup +} + +type ValidatorDashboardGroup struct { + Name string `db:"name"` + Validators [][]byte } type TaggedValidators struct { diff --git a/backend/pkg/notification/db.go b/backend/pkg/notification/db.go index cf2a217e0..26838c9bb 100644 --- a/backend/pkg/notification/db.go +++ b/backend/pkg/notification/db.go @@ -7,7 +7,11 @@ import ( "github.com/lib/pq" ) -func GetSubsForEventFilter(eventName types.EventName) (map[string]bool, map[string][]types.Subscription, error) { +// Retrieves all subscription for a given event filter +// Map key corresponds to the event filter which can be +// a validator pubkey or an eth1 address (for RPL notifications) +// or a list of validators for the tax report notifications +func GetSubsForEventFilter(eventName types.EventName) (map[string][]types.Subscription, error) { var subs []types.Subscription subQuery := ` SELECT id, user_id, event_filter, last_sent_epoch, created_epoch, event_threshold, ENCODE(unsubscribe_hash, 'hex') as unsubscribe_hash, internal_state from users_subscriptions where event_name = $1 @@ -16,10 +20,9 @@ func GetSubsForEventFilter(eventName types.EventName) (map[string]bool, map[stri subMap := make(map[string][]types.Subscription, 0) err := db.FrontendWriterDB.Select(&subs, subQuery, utils.GetNetwork()+":"+string(eventName)) if err != nil { - return nil, nil, err + return nil, err } - filtersEncode := make(map[string]bool, len(subs)) for _, sub := range subs { if _, ok := subMap[sub.EventFilter]; !ok { subMap[sub.EventFilter] = make([]types.Subscription, 0) @@ -33,10 +36,8 @@ func GetSubsForEventFilter(eventName types.EventName) (map[string]bool, map[stri EventThreshold: sub.EventThreshold, State: sub.State, }) - - filtersEncode[sub.EventFilter] = true } - return filtersEncode, subMap, nil + return subMap, nil } func GetUserPushTokenByIds(ids []uint64) (map[uint64][]string, error) { diff --git a/backend/pkg/notification/notifications.go b/backend/pkg/notification/notifications.go index ec0e841a9..4a58768dc 100644 --- a/backend/pkg/notification/notifications.go +++ b/backend/pkg/notification/notifications.go @@ -1207,7 +1207,7 @@ func collectBlockProposalNotifications(notificationsByUserID map[uint64]map[type ExecRewardETH float64 } - _, subMap, err := GetSubsForEventFilter(eventName) + subMap, err := GetSubsForEventFilter(eventName) if err != nil { return fmt.Errorf("error getting subscriptions for (missed) block proposals %w", err) } @@ -1394,7 +1394,7 @@ func (n *validatorProposalNotification) GetInfoMarkdown() string { } func collectAttestationAndOfflineValidatorNotifications(notificationsByUserID map[uint64]map[types.EventName][]types.Notification, epoch uint64) error { - _, subMap, err := GetSubsForEventFilter(types.ValidatorMissedAttestationEventName) + subMap, err := GetSubsForEventFilter(types.ValidatorMissedAttestationEventName) if err != nil { return fmt.Errorf("error getting subscriptions for missted attestations %w", err) } @@ -1573,7 +1573,7 @@ func collectAttestationAndOfflineValidatorNotifications(notificationsByUserID ma return fmt.Errorf("retrieved more than %v online validators notifications: %v, exiting", onlineValidatorsLimit, len(onlineValidators)) } - _, subMap, err = GetSubsForEventFilter(types.ValidatorIsOfflineEventName) + subMap, err = GetSubsForEventFilter(types.ValidatorIsOfflineEventName) if err != nil { return fmt.Errorf("failed to get subs for %v: %v", types.ValidatorIsOfflineEventName, err) } @@ -2022,7 +2022,7 @@ func (n *validatorWithdrawalNotification) GetInfoMarkdown() string { // collectWithdrawalNotifications collects all notifications validator withdrawals func collectWithdrawalNotifications(notificationsByUserID map[uint64]map[types.EventName][]types.Notification, epoch uint64) error { // get all users that are subscribed to this event (scale: a few thousand rows depending on how many users we have) - _, subMap, err := GetSubsForEventFilter(types.ValidatorReceivedWithdrawalEventName) + subMap, err := GetSubsForEventFilter(types.ValidatorReceivedWithdrawalEventName) if err != nil { return fmt.Errorf("error getting subscriptions for missed attestations %w", err) } @@ -2947,7 +2947,7 @@ func collectRocketpoolRewardClaimRoundNotifications(notificationsByUserID map[ui } func collectRocketpoolRPLCollateralNotifications(notificationsByUserID map[uint64]map[types.EventName][]types.Notification, eventName types.EventName, epoch uint64) error { - pubkeys, subMap, err := GetSubsForEventFilter(eventName) + subMap, err := GetSubsForEventFilter(eventName) if err != nil { return fmt.Errorf("error getting subscriptions for RocketpoolRPLCollateral %w", err) } @@ -2964,7 +2964,7 @@ func collectRocketpoolRPLCollateralNotifications(notificationsByUserID map[uint6 stakeInfoPerNode := make([]dbResult, 0) batchSize := 5000 keys := make([][]byte, 0, batchSize) - for pubkey := range pubkeys { + for pubkey := range subMap { b, err := hex.DecodeString(pubkey) if err != nil { log.Error(err, fmt.Sprintf("error decoding pubkey %s", pubkey), 0) From 67b8529f8b4b8580b7472c190eb251dad5cef1a8 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Fri, 5 Jul 2024 08:54:13 +0200 Subject: [PATCH 03/94] improve typing of userid --- backend/pkg/commons/db/bigtable.go | 30 ++--- backend/pkg/commons/types/frontend.go | 18 +-- backend/pkg/notification/db.go | 16 +-- backend/pkg/notification/notifications.go | 130 +++++++++++++++------- 4 files changed, 125 insertions(+), 69 deletions(-) diff --git a/backend/pkg/commons/db/bigtable.go b/backend/pkg/commons/db/bigtable.go index edd9b3c17..7b7cd147e 100644 --- a/backend/pkg/commons/db/bigtable.go +++ b/backend/pkg/commons/db/bigtable.go @@ -194,7 +194,7 @@ func (bigtable *Bigtable) GetClient() *gcp_bigtable.Client { return bigtable.client } -func (bigtable *Bigtable) SaveMachineMetric(process string, userID uint64, machine string, data []byte) error { +func (bigtable *Bigtable) SaveMachineMetric(process string, userID types.UserId, machine string, data []byte) error { ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) defer cancel() @@ -234,7 +234,7 @@ func (bigtable *Bigtable) SaveMachineMetric(process string, userID uint64, machi return nil } -func (bigtable Bigtable) getMachineMetricNamesMap(userID uint64, searchDepth int) (map[string]bool, error) { +func (bigtable Bigtable) getMachineMetricNamesMap(userID types.UserId, searchDepth int) (map[string]bool, error) { ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30)) defer cancel() @@ -265,7 +265,7 @@ func (bigtable Bigtable) getMachineMetricNamesMap(userID uint64, searchDepth int return machineNames, nil } -func (bigtable Bigtable) GetMachineMetricsMachineNames(userID uint64) ([]string, error) { +func (bigtable Bigtable) GetMachineMetricsMachineNames(userID types.UserId) ([]string, error) { tmr := time.AfterFunc(REPORT_TIMEOUT, func() { log.WarnWithFields(log.Fields{ "userId": userID, @@ -288,7 +288,7 @@ func (bigtable Bigtable) GetMachineMetricsMachineNames(userID uint64) ([]string, return result, nil } -func (bigtable Bigtable) GetMachineMetricsMachineCount(userID uint64) (uint64, error) { +func (bigtable Bigtable) GetMachineMetricsMachineCount(userID types.UserId) (uint64, error) { tmr := time.AfterFunc(REPORT_TIMEOUT, func() { log.WarnWithFields(log.Fields{ "userId": userID, @@ -310,7 +310,7 @@ func (bigtable Bigtable) GetMachineMetricsMachineCount(userID uint64) (uint64, e return uint64(card), nil } -func (bigtable Bigtable) GetMachineMetricsNode(userID uint64, limit, offset int) ([]*types.MachineMetricNode, error) { +func (bigtable Bigtable) GetMachineMetricsNode(userID types.UserId, limit, offset int) ([]*types.MachineMetricNode, error) { tmr := time.AfterFunc(REPORT_TIMEOUT, func() { log.WarnWithFields(log.Fields{ "userId": userID, @@ -335,7 +335,7 @@ func (bigtable Bigtable) GetMachineMetricsNode(userID uint64, limit, offset int) ) } -func (bigtable Bigtable) GetMachineMetricsValidator(userID uint64, limit, offset int) ([]*types.MachineMetricValidator, error) { +func (bigtable Bigtable) GetMachineMetricsValidator(userID types.UserId, limit, offset int) ([]*types.MachineMetricValidator, error) { tmr := time.AfterFunc(REPORT_TIMEOUT, func() { log.WarnWithFields(log.Fields{ "userId": userID, @@ -360,7 +360,7 @@ func (bigtable Bigtable) GetMachineMetricsValidator(userID uint64, limit, offset ) } -func (bigtable Bigtable) GetMachineMetricsSystem(userID uint64, limit, offset int) ([]*types.MachineMetricSystem, error) { +func (bigtable Bigtable) GetMachineMetricsSystem(userID types.UserId, limit, offset int) ([]*types.MachineMetricSystem, error) { tmr := time.AfterFunc(REPORT_TIMEOUT, func() { log.WarnWithFields(log.Fields{ "userId": userID, @@ -385,7 +385,7 @@ func (bigtable Bigtable) GetMachineMetricsSystem(userID uint64, limit, offset in ) } -func getMachineMetrics[T types.MachineMetricSystem | types.MachineMetricNode | types.MachineMetricValidator](bigtable Bigtable, process string, userID uint64, limit, offset int, marshler func(data []byte, machine string) *T) ([]*T, error) { +func getMachineMetrics[T types.MachineMetricSystem | types.MachineMetricNode | types.MachineMetricValidator](bigtable Bigtable, process string, userID types.UserId, limit, offset int, marshler func(data []byte, machine string) *T) ([]*T, error) { ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*30)) defer cancel() @@ -429,7 +429,7 @@ func getMachineMetrics[T types.MachineMetricSystem | types.MachineMetricNode | t return res, nil } -func (bigtable Bigtable) GetMachineRowKey(userID uint64, process string, machine string) string { +func (bigtable Bigtable) GetMachineRowKey(userID types.UserId, process string, machine string) string { return fmt.Sprintf("u:%s:p:%s:m:%s", bigtable.reversePaddedUserID(userID), process, machine) } @@ -437,7 +437,7 @@ func (bigtable Bigtable) GetMachineRowKey(userID uint64, process string, machine // machineData contains the latest machine data in CurrentData // and 5 minute old data in fiveMinuteOldData (defined in limit) // as well as the insert timestamps of both -func (bigtable Bigtable) GetMachineMetricsForNotifications(rowKeys gcp_bigtable.RowList) (map[uint64]map[string]*types.MachineMetricSystemUser, error) { +func (bigtable Bigtable) GetMachineMetricsForNotifications(rowKeys gcp_bigtable.RowList) (map[types.UserId]map[string]*types.MachineMetricSystemUser, error) { tmr := time.AfterFunc(REPORT_TIMEOUT, func() { log.WarnWithFields(log.Fields{ "rowKeys": rowKeys, @@ -449,7 +449,7 @@ func (bigtable Bigtable) GetMachineMetricsForNotifications(rowKeys gcp_bigtable. ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(time.Second*200)) defer cancel() - res := make(map[uint64]map[string]*types.MachineMetricSystemUser) // userID -> machine -> data + res := make(map[types.UserId]map[string]*types.MachineMetricSystemUser) // userID -> machine -> data limit := 5 @@ -509,7 +509,7 @@ func (bigtable Bigtable) GetMachineMetricsForNotifications(rowKeys gcp_bigtable. } //nolint:unparam -func machineMetricRowParts(r string) (bool, uint64, string, string) { +func machineMetricRowParts(r string) (bool, types.UserId, string, string) { keySplit := strings.Split(r, ":") userID, err := strconv.ParseUint(keySplit[1], 10, 64) @@ -526,7 +526,7 @@ func machineMetricRowParts(r string) (bool, uint64, string, string) { process := keySplit[3] - return true, userID, machine, process + return true, types.UserId(userID), machine, process } func (bigtable *Bigtable) SaveValidatorBalances(epoch uint64, validators []*types.Validator) error { @@ -2678,8 +2678,8 @@ func GetCurrentDayClIncome(validator_indices []uint64) (map[uint64]int64, error) return dayIncome, nil } -func (bigtable *Bigtable) reversePaddedUserID(userID uint64) string { - return fmt.Sprintf("%09d", ^uint64(0)-userID) +func (bigtable *Bigtable) reversePaddedUserID(userID types.UserId) string { + return fmt.Sprintf("%09d", ^uint64(0)-uint64(userID)) } func (bigtable *Bigtable) reversedPaddedEpoch(epoch uint64) string { diff --git a/backend/pkg/commons/types/frontend.go b/backend/pkg/commons/types/frontend.go index f468b5c43..d91d60747 100644 --- a/backend/pkg/commons/types/frontend.go +++ b/backend/pkg/commons/types/frontend.go @@ -12,6 +12,7 @@ import ( "firebase.google.com/go/messaging" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/gobitfly/beaconchain/pkg/consapi/types" "github.com/lib/pq" "github.com/pkg/errors" "golang.org/x/text/cases" @@ -164,7 +165,7 @@ type EventNameDesc struct { } type MachineMetricSystemUser struct { - UserID uint64 + UserID UserId Machine string CurrentData *MachineMetricSystem CurrentDataInsertTs int64 @@ -255,7 +256,7 @@ type Notification interface { type Subscription struct { ID *uint64 `db:"id,omitempty"` - UserID *uint64 `db:"user_id,omitempty"` + UserID *UserId `db:"user_id,omitempty"` EventName string `db:"event_name"` EventFilter string `db:"event_filter"` LastSent *time.Time `db:"last_sent_ts"` @@ -266,22 +267,25 @@ type Subscription struct { EventThreshold float64 `db:"event_threshold"` UnsubscribeHash sql.NullString `db:"unsubscribe_hash" swaggertype:"string"` State sql.NullString `db:"internal_state" swaggertype:"string"` - GroupId *uint32 - DashboardId *uint32 + GroupId *int64 + DashboardId *int64 } +type UserId uint64 +type DashboardId uint64 +type DashboardGroupId uint64 type ValidatorDashboardConfig struct { - DashboardsByUserId map[uint64]map[uint32]*ValidatorDashboard + DashboardsByUserId map[UserId]map[DashboardId]*ValidatorDashboard } type ValidatorDashboard struct { Name string `db:"name"` - Groups map[uint32]*ValidatorDashboardGroup + Groups map[DashboardGroupId]*ValidatorDashboardGroup } type ValidatorDashboardGroup struct { Name string `db:"name"` - Validators [][]byte + Validators []types.ValidatorIndex } type TaggedValidators struct { diff --git a/backend/pkg/notification/db.go b/backend/pkg/notification/db.go index 26838c9bb..23f890ba6 100644 --- a/backend/pkg/notification/db.go +++ b/backend/pkg/notification/db.go @@ -40,14 +40,14 @@ func GetSubsForEventFilter(eventName types.EventName) (map[string][]types.Subscr return subMap, nil } -func GetUserPushTokenByIds(ids []uint64) (map[uint64][]string, error) { - pushByID := map[uint64][]string{} +func GetUserPushTokenByIds(ids []types.UserId) (map[types.UserId][]string, error) { + pushByID := map[types.UserId][]string{} if len(ids) == 0 { return pushByID, nil } var rows []struct { - ID uint64 `db:"user_id"` - Token string `db:"notification_token"` + ID types.UserId `db:"user_id"` + Token string `db:"notification_token"` } err := db.FrontendWriterDB.Select(&rows, "SELECT DISTINCT ON (user_id, notification_token) user_id, notification_token FROM users_devices WHERE (user_id = ANY($1) AND user_id NOT IN (SELECT user_id from users_notification_channels WHERE active = false and channel = $2)) AND notify_enabled = true AND active = true AND notification_token IS NOT NULL AND LENGTH(notification_token) > 20 ORDER BY user_id, notification_token, id DESC", pq.Array(ids), types.PushNotificationChannel) @@ -67,14 +67,14 @@ func GetUserPushTokenByIds(ids []uint64) (map[uint64][]string, error) { } // GetUserEmailsByIds returns the emails of users. -func GetUserEmailsByIds(ids []uint64) (map[uint64]string, error) { - mailsByID := map[uint64]string{} +func GetUserEmailsByIds(ids []types.UserId) (map[types.UserId]string, error) { + mailsByID := map[types.UserId]string{} if len(ids) == 0 { return mailsByID, nil } var rows []struct { - ID uint64 `db:"id"` - Email string `db:"email"` + ID types.UserId `db:"id"` + Email string `db:"email"` } // err := db.FrontendWriterDB.Select(&rows, "SELECT id, email FROM users WHERE id = ANY($1) AND id NOT IN (SELECT user_id from users_notification_channels WHERE active = false and channel = $2)", pq.Array(ids), types.EmailNotificationChannel) diff --git a/backend/pkg/notification/notifications.go b/backend/pkg/notification/notifications.go index 4a58768dc..c408c1cdf 100644 --- a/backend/pkg/notification/notifications.go +++ b/backend/pkg/notification/notifications.go @@ -225,8 +225,8 @@ func notificationSender() { } } -func collectNotifications(epoch uint64) (map[uint64]map[types.EventName][]types.Notification, error) { - notificationsByUserID := map[uint64]map[types.EventName][]types.Notification{} +func collectNotifications(epoch uint64) (map[types.UserId]map[types.EventName][]types.Notification, error) { + notificationsByUserID := map[types.UserId]map[types.EventName][]types.Notification{} start := time.Now() var err error var dbIsCoherent bool @@ -253,6 +253,58 @@ func collectNotifications(epoch uint64) (map[uint64]map[types.EventName][]types. log.Infof("started collecting notifications") + type dashboardDefinitionRow struct { + DashboardId types.DashboardId `db:"dashboard_id"` + DashboardName string `db:"dashboard_name"` + UserId types.UserId `db:"user_id"` + GroupId types.DashboardGroupId `db:"group_id"` + GroupName string `db:"group_name"` + ValidatorIndex types.ValidatorIndex `db:"validator_index"` + } + + log.Infof("retrieving dashboard definitions") + // TODO: add a filter to retrieve only groups that have notifications enabled + // Needs a new field in the db + var dashboardDefinitions []dashboardDefinitionRow + err = db.AlloyWriter.Select(&dashboardDefinitions, ` + select + users_val_dashboards.id as dashboard_id, + users_val_dashboards.name as dashboard_name, + users_val_dashboards.user_id, + users_val_dashboards_groups.id as group_id, + users_val_dashboards_groups.name as group_name, + users_val_dashboards_validators.validator_index + from users_val_dashboards + left join users_val_dashboards_groups on users_val_dashboards_groups.dashboard_id = users_val_dashboards.id + left join users_val_dashboards_validators on users_val_dashboards_validators.dashboard_id = users_val_dashboards_groups.dashboard_id AND users_val_dashboards_validators.group_id = users_val_dashboards_groups.id; + `) + if err != nil { + return nil, fmt.Errorf("error getting dashboard definitions: %v", err) + } + + // Now initialize the validator dashboard configuration map + validatorDashboardConfig := &types.ValidatorDashboardConfig{ + DashboardsByUserId: make(map[types.UserId]map[types.DashboardId]*types.ValidatorDashboard), + } + for _, row := range dashboardDefinitions { + if validatorDashboardConfig.DashboardsByUserId[row.UserId] == nil { + validatorDashboardConfig.DashboardsByUserId[row.UserId] = make(map[types.DashboardId]*types.ValidatorDashboard) + } + if validatorDashboardConfig.DashboardsByUserId[row.UserId][row.DashboardId] == nil { + validatorDashboardConfig.DashboardsByUserId[row.UserId][row.DashboardId] = &types.ValidatorDashboard{ + Name: row.DashboardName, + Groups: make(map[types.DashboardGroupId]*types.ValidatorDashboardGroup), + } + } + if validatorDashboardConfig.DashboardsByUserId[row.UserId][row.DashboardId].Groups[row.GroupId] == nil { + validatorDashboardConfig.DashboardsByUserId[row.UserId][row.DashboardId].Groups[row.GroupId] = &types.ValidatorDashboardGroup{ + Name: row.GroupName, + Validators: []uint64{}, + } + } + validatorDashboardConfig.DashboardsByUserId[row.UserId][row.DashboardId].Groups[row.GroupId].Validators = append(validatorDashboardConfig.DashboardsByUserId[row.UserId][row.DashboardId].Groups[row.GroupId].Validators, uint64(row.ValidatorIndex)) + } + err = collectAttestationAndOfflineValidatorNotifications(notificationsByUserID, epoch) if err != nil { metrics.Errors.WithLabelValues("notifications_collect_missed_attestation").Inc() @@ -356,8 +408,8 @@ func collectNotifications(epoch uint64) (map[uint64]map[types.EventName][]types. return notificationsByUserID, nil } -func collectUserDbNotifications(epoch uint64) (map[uint64]map[types.EventName][]types.Notification, error) { - notificationsByUserID := map[uint64]map[types.EventName][]types.Notification{} +func collectUserDbNotifications(epoch uint64) (map[types.UserId]map[types.EventName][]types.Notification, error) { + notificationsByUserID := map[types.UserId]map[types.EventName][]types.Notification{} var err error // Monitoring (premium): machine offline @@ -405,7 +457,7 @@ func collectUserDbNotifications(epoch uint64) (map[uint64]map[types.EventName][] return notificationsByUserID, nil } -func queueNotifications(notificationsByUserID map[uint64]map[types.EventName][]types.Notification, useDB *sqlx.DB) { +func queueNotifications(notificationsByUserID map[types.UserId]map[types.EventName][]types.Notification, useDB *sqlx.DB) { subByEpoch := map[uint64][]uint64{} // prevent multiple events being sent with the same subscription id @@ -541,8 +593,8 @@ func getNetwork() string { return "" } -func queuePushNotification(notificationsByUserID map[uint64]map[types.EventName][]types.Notification, useDB *sqlx.DB) error { - userIDs := []uint64{} +func queuePushNotification(notificationsByUserID map[types.UserId]map[types.EventName][]types.Notification, useDB *sqlx.DB) error { + userIDs := []types.UserId{} for userID := range notificationsByUserID { userIDs = append(userIDs, userID) } @@ -646,8 +698,8 @@ func sendPushNotifications(useDB *sqlx.DB) error { return nil } -func queueEmailNotifications(notificationsByUserID map[uint64]map[types.EventName][]types.Notification, useDB *sqlx.DB) error { - userIDs := []uint64{} +func queueEmailNotifications(notificationsByUserID map[types.UserId]map[types.EventName][]types.Notification, useDB *sqlx.DB) error { + userIDs := []types.UserId{} for userID := range notificationsByUserID { userIDs = append(userIDs, userID) } @@ -841,7 +893,7 @@ func sendEmailNotifications(useDb *sqlx.DB) error { return nil } -func queueWebhookNotifications(notificationsByUserID map[uint64]map[types.EventName][]types.Notification, useDB *sqlx.DB) error { +func queueWebhookNotifications(notificationsByUserID map[types.UserId]map[types.EventName][]types.Notification, useDB *sqlx.DB) error { for userID, userNotifications := range notificationsByUserID { var webhooks []types.UserWebhook err := useDB.Select(&webhooks, ` @@ -1198,7 +1250,7 @@ func getUrlPart(validatorIndex uint64) string { return fmt.Sprintf(` For more information visit: https://%s/validator/%v.`, utils.Config.Frontend.SiteDomain, validatorIndex, utils.Config.Frontend.SiteDomain, validatorIndex) } -func collectBlockProposalNotifications(notificationsByUserID map[uint64]map[types.EventName][]types.Notification, status uint64, eventName types.EventName, epoch uint64) error { +func collectBlockProposalNotifications(notificationsByUserID map[types.UserId]map[types.EventName][]types.Notification, status uint64, eventName types.EventName, epoch uint64) error { type dbResult struct { Proposer uint64 `db:"proposer"` Status uint64 `db:"status"` @@ -1393,7 +1445,7 @@ func (n *validatorProposalNotification) GetInfoMarkdown() string { return generalPart } -func collectAttestationAndOfflineValidatorNotifications(notificationsByUserID map[uint64]map[types.EventName][]types.Notification, epoch uint64) error { +func collectAttestationAndOfflineValidatorNotifications(notificationsByUserID map[types.UserId]map[types.EventName][]types.Notification, epoch uint64) error { subMap, err := GetSubsForEventFilter(types.ValidatorMissedAttestationEventName) if err != nil { return fmt.Errorf("error getting subscriptions for missted attestations %w", err) @@ -1898,7 +1950,7 @@ func (n *validatorGotSlashedNotification) GetInfoMarkdown() string { return generalPart } -func collectValidatorGotSlashedNotifications(notificationsByUserID map[uint64]map[types.EventName][]types.Notification, epoch uint64) error { +func collectValidatorGotSlashedNotifications(notificationsByUserID map[types.UserId]map[types.EventName][]types.Notification, epoch uint64) error { dbResult, err := db.GetValidatorsGotSlashed(epoch) if err != nil { return fmt.Errorf("error getting slashed validators from database, err: %w", err) @@ -1919,7 +1971,7 @@ func collectValidatorGotSlashedNotifications(notificationsByUserID map[uint64]ma var subscribers []struct { Ref uint64 `db:"ref"` Id uint64 `db:"id"` - UserId uint64 `db:"user_id"` + UserId types.UserId `db:"user_id"` UnsubscribeHash sql.NullString `db:"unsubscribe_hash"` } @@ -2020,7 +2072,7 @@ func (n *validatorWithdrawalNotification) GetInfoMarkdown() string { } // collectWithdrawalNotifications collects all notifications validator withdrawals -func collectWithdrawalNotifications(notificationsByUserID map[uint64]map[types.EventName][]types.Notification, epoch uint64) error { +func collectWithdrawalNotifications(notificationsByUserID map[types.UserId]map[types.EventName][]types.Notification, epoch uint64) error { // get all users that are subscribed to this event (scale: a few thousand rows depending on how many users we have) subMap, err := GetSubsForEventFilter(types.ValidatorReceivedWithdrawalEventName) if err != nil { @@ -2075,7 +2127,7 @@ func collectWithdrawalNotifications(notificationsByUserID map[uint64]map[types.E type ethClientNotification struct { SubscriptionID uint64 - UserID uint64 + UserID types.UserId Epoch uint64 EthClient string EventFilter string @@ -2183,12 +2235,12 @@ func (n *ethClientNotification) GetInfoMarkdown() string { return generalPart } -func collectEthClientNotifications(notificationsByUserID map[uint64]map[types.EventName][]types.Notification, eventName types.EventName) error { +func collectEthClientNotifications(notificationsByUserID map[types.UserId]map[types.EventName][]types.Notification, eventName types.EventName) error { updatedClients := ethclients.GetUpdatedClients() //only check if there are new updates for _, client := range updatedClients { var dbResult []struct { SubscriptionID uint64 `db:"id"` - UserID uint64 `db:"user_id"` + UserID types.UserId `db:"user_id"` Epoch uint64 `db:"created_epoch"` EventFilter string `db:"event_filter"` UnsubscribeHash sql.NullString `db:"unsubscribe_hash"` @@ -2234,13 +2286,13 @@ func collectEthClientNotifications(notificationsByUserID map[uint64]map[types.Ev type MachineEvents struct { SubscriptionID uint64 `db:"id"` - UserID uint64 `db:"user_id"` + UserID types.UserId `db:"user_id"` MachineName string `db:"machine"` UnsubscribeHash sql.NullString `db:"unsubscribe_hash"` EventThreshold float64 `db:"event_threshold"` } -func collectMonitoringMachineOffline(notificationsByUserID map[uint64]map[types.EventName][]types.Notification, epoch uint64) error { +func collectMonitoringMachineOffline(notificationsByUserID map[types.UserId]map[types.EventName][]types.Notification, epoch uint64) error { nowTs := time.Now().Unix() return collectMonitoringMachine(notificationsByUserID, types.MonitoringMachineOfflineEventName, 120, // notify condition @@ -2259,7 +2311,7 @@ func isMachineDataRecent(machineData *types.MachineMetricSystemUser) bool { return machineData.CurrentDataInsertTs >= nowTs-60*60 } -func collectMonitoringMachineDiskAlmostFull(notificationsByUserID map[uint64]map[types.EventName][]types.Notification, epoch uint64) error { +func collectMonitoringMachineDiskAlmostFull(notificationsByUserID map[types.UserId]map[types.EventName][]types.Notification, epoch uint64) error { return collectMonitoringMachine(notificationsByUserID, types.MonitoringMachineDiskAlmostFullEventName, 750, // notify condition func(subscribeData *MachineEvents, machineData *types.MachineMetricSystemUser) bool { @@ -2274,7 +2326,7 @@ func collectMonitoringMachineDiskAlmostFull(notificationsByUserID map[uint64]map ) } -func collectMonitoringMachineCPULoad(notificationsByUserID map[uint64]map[types.EventName][]types.Notification, epoch uint64) error { +func collectMonitoringMachineCPULoad(notificationsByUserID map[types.UserId]map[types.EventName][]types.Notification, epoch uint64) error { return collectMonitoringMachine(notificationsByUserID, types.MonitoringMachineCpuLoadEventName, 10, // notify condition func(subscribeData *MachineEvents, machineData *types.MachineMetricSystemUser) bool { @@ -2296,7 +2348,7 @@ func collectMonitoringMachineCPULoad(notificationsByUserID map[uint64]map[types. ) } -func collectMonitoringMachineMemoryUsage(notificationsByUserID map[uint64]map[types.EventName][]types.Notification, epoch uint64) error { +func collectMonitoringMachineMemoryUsage(notificationsByUserID map[types.UserId]map[types.EventName][]types.Notification, epoch uint64) error { return collectMonitoringMachine(notificationsByUserID, types.MonitoringMachineMemoryUsageEventName, 10, // notify condition func(subscribeData *MachineEvents, machineData *types.MachineMetricSystemUser) bool { @@ -2317,7 +2369,7 @@ func collectMonitoringMachineMemoryUsage(notificationsByUserID map[uint64]map[ty var isFirstNotificationCheck = true func collectMonitoringMachine( - notificationsByUserID map[uint64]map[types.EventName][]types.Notification, + notificationsByUserID map[types.UserId]map[types.EventName][]types.Notification, eventName types.EventName, epochWaitInBetween int, notifyConditionFulfilled func(subscribeData *MachineEvents, machineData *types.MachineMetricSystemUser) bool, @@ -2439,7 +2491,7 @@ func collectMonitoringMachine( type monitorMachineNotification struct { SubscriptionID uint64 MachineName string - UserID uint64 + UserID types.UserId Epoch uint64 EventName types.EventName UnsubscribeHash sql.NullString @@ -2518,7 +2570,7 @@ func (n *monitorMachineNotification) GetInfoMarkdown() string { type taxReportNotification struct { SubscriptionID uint64 - UserID uint64 + UserID types.UserId Epoch uint64 EventFilter string UnsubscribeHash sql.NullString @@ -2598,7 +2650,7 @@ func (n *taxReportNotification) GetInfoMarkdown() string { return n.GetInfo(false) } -func collectTaxReportNotificationNotifications(notificationsByUserID map[uint64]map[types.EventName][]types.Notification, eventName types.EventName) error { +func collectTaxReportNotificationNotifications(notificationsByUserID map[types.UserId]map[types.EventName][]types.Notification, eventName types.EventName) error { lastStatsDay, err := cache.LatestExportedStatisticDay.GetOrDefault(db.GetLastExportedStatisticDay) if err != nil { @@ -2613,7 +2665,7 @@ func collectTaxReportNotificationNotifications(notificationsByUserID map[uint64] var dbResult []struct { SubscriptionID uint64 `db:"id"` - UserID uint64 `db:"user_id"` + UserID types.UserId `db:"user_id"` Epoch uint64 `db:"created_epoch"` EventFilter string `db:"event_filter"` UnsubscribeHash sql.NullString `db:"unsubscribe_hash"` @@ -2658,7 +2710,7 @@ func collectTaxReportNotificationNotifications(notificationsByUserID map[uint64] type networkNotification struct { SubscriptionID uint64 - UserID uint64 + UserID types.UserId Epoch uint64 EventFilter string UnsubscribeHash sql.NullString @@ -2709,7 +2761,7 @@ func (n *networkNotification) GetInfoMarkdown() string { return generalPart } -func collectNetworkNotifications(notificationsByUserID map[uint64]map[types.EventName][]types.Notification, eventName types.EventName) error { +func collectNetworkNotifications(notificationsByUserID map[types.UserId]map[types.EventName][]types.Notification, eventName types.EventName) error { count := 0 err := db.WriterDb.Get(&count, ` SELECT count(ts) FROM network_liveness WHERE (headepoch-finalizedepoch) > 3 AND ts > now() - interval '60 minutes'; @@ -2722,7 +2774,7 @@ func collectNetworkNotifications(notificationsByUserID map[uint64]map[types.Even if count > 0 { var dbResult []struct { SubscriptionID uint64 `db:"id"` - UserID uint64 `db:"user_id"` + UserID types.UserId `db:"user_id"` Epoch uint64 `db:"created_epoch"` EventFilter string `db:"event_filter"` UnsubscribeHash sql.NullString `db:"unsubscribe_hash"` @@ -2763,7 +2815,7 @@ func collectNetworkNotifications(notificationsByUserID map[uint64]map[types.Even type rocketpoolNotification struct { SubscriptionID uint64 - UserID uint64 + UserID types.UserId Epoch uint64 EventFilter string EventName types.EventName @@ -2839,7 +2891,7 @@ func (n *rocketpoolNotification) GetInfoMarkdown() string { return n.GetInfo(false) } -func collectRocketpoolComissionNotifications(notificationsByUserID map[uint64]map[types.EventName][]types.Notification, eventName types.EventName) error { +func collectRocketpoolComissionNotifications(notificationsByUserID map[types.UserId]map[types.EventName][]types.Notification, eventName types.EventName) error { fee := 0.0 err := db.WriterDb.Get(&fee, ` select current_node_fee from rocketpool_network_stats order by id desc LIMIT 1; @@ -2852,7 +2904,7 @@ func collectRocketpoolComissionNotifications(notificationsByUserID map[uint64]ma if fee > 0 { var dbResult []struct { SubscriptionID uint64 `db:"id"` - UserID uint64 `db:"user_id"` + UserID types.UserId `db:"user_id"` Epoch uint64 `db:"created_epoch"` EventFilter string `db:"event_filter"` UnsubscribeHash sql.NullString `db:"unsubscribe_hash"` @@ -2893,7 +2945,7 @@ func collectRocketpoolComissionNotifications(notificationsByUserID map[uint64]ma return nil } -func collectRocketpoolRewardClaimRoundNotifications(notificationsByUserID map[uint64]map[types.EventName][]types.Notification, eventName types.EventName) error { +func collectRocketpoolRewardClaimRoundNotifications(notificationsByUserID map[types.UserId]map[types.EventName][]types.Notification, eventName types.EventName) error { var ts int64 err := db.WriterDb.Get(&ts, ` select date_part('epoch', claim_interval_time_start)::int from rocketpool_network_stats order by id desc LIMIT 1; @@ -2906,7 +2958,7 @@ func collectRocketpoolRewardClaimRoundNotifications(notificationsByUserID map[ui if ts+3*60*60 > time.Now().Unix() { var dbResult []struct { SubscriptionID uint64 `db:"id"` - UserID uint64 `db:"user_id"` + UserID types.UserId `db:"user_id"` Epoch uint64 `db:"created_epoch"` EventFilter string `db:"event_filter"` UnsubscribeHash sql.NullString `db:"unsubscribe_hash"` @@ -2946,7 +2998,7 @@ func collectRocketpoolRewardClaimRoundNotifications(notificationsByUserID map[ui return nil } -func collectRocketpoolRPLCollateralNotifications(notificationsByUserID map[uint64]map[types.EventName][]types.Notification, eventName types.EventName, epoch uint64) error { +func collectRocketpoolRPLCollateralNotifications(notificationsByUserID map[types.UserId]map[types.EventName][]types.Notification, eventName types.EventName, epoch uint64) error { subMap, err := GetSubsForEventFilter(eventName) if err != nil { return fmt.Errorf("error getting subscriptions for RocketpoolRPLCollateral %w", err) @@ -3124,7 +3176,7 @@ func bigFloat(x float64) *big.Float { return new(big.Float).SetFloat64(x) } -func collectSyncCommittee(notificationsByUserID map[uint64]map[types.EventName][]types.Notification, eventName types.EventName, epoch uint64) error { +func collectSyncCommittee(notificationsByUserID map[types.UserId]map[types.EventName][]types.Notification, eventName types.EventName, epoch uint64) error { slotsPerSyncCommittee := utils.SlotsPerSyncCommittee() currentPeriod := epoch * utils.Config.Chain.ClConfig.SlotsPerEpoch / slotsPerSyncCommittee nextPeriod := currentPeriod + 1 @@ -3152,7 +3204,7 @@ func collectSyncCommittee(notificationsByUserID map[uint64]map[types.EventName][ var dbResult []struct { SubscriptionID uint64 `db:"id"` - UserID uint64 `db:"user_id"` + UserID types.UserId `db:"user_id"` EventFilter string `db:"event_filter"` UnsubscribeHash sql.NullString `db:"unsubscribe_hash"` } From 3aa7d8e8b56800b56551b52fd361169c34b3b39c Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Fri, 5 Jul 2024 09:37:07 +0200 Subject: [PATCH 04/94] ensure that there is never more than one notification per user / type / filter combination --- backend/pkg/commons/types/frontend.go | 2 + backend/pkg/notification/notifications.go | 208 +++++++++------------- 2 files changed, 84 insertions(+), 126 deletions(-) diff --git a/backend/pkg/commons/types/frontend.go b/backend/pkg/commons/types/frontend.go index d91d60747..5367951a1 100644 --- a/backend/pkg/commons/types/frontend.go +++ b/backend/pkg/commons/types/frontend.go @@ -20,6 +20,8 @@ import ( ) type EventName string +type EventFilter string +type NotificationsPerUserId map[UserId]map[EventName]map[EventFilter]Notification const ( ValidatorBalanceDecreasedEventName EventName = "validator_balance_decreased" diff --git a/backend/pkg/notification/notifications.go b/backend/pkg/notification/notifications.go index c408c1cdf..e2db123f1 100644 --- a/backend/pkg/notification/notifications.go +++ b/backend/pkg/notification/notifications.go @@ -225,8 +225,8 @@ func notificationSender() { } } -func collectNotifications(epoch uint64) (map[types.UserId]map[types.EventName][]types.Notification, error) { - notificationsByUserID := map[types.UserId]map[types.EventName][]types.Notification{} +func collectNotifications(epoch uint64) (types.NotificationsPerUserId, error) { + notificationsByUserID := types.NotificationsPerUserId{} start := time.Now() var err error var dbIsCoherent bool @@ -408,8 +408,8 @@ func collectNotifications(epoch uint64) (map[types.UserId]map[types.EventName][] return notificationsByUserID, nil } -func collectUserDbNotifications(epoch uint64) (map[types.UserId]map[types.EventName][]types.Notification, error) { - notificationsByUserID := map[types.UserId]map[types.EventName][]types.Notification{} +func collectUserDbNotifications(epoch uint64) (types.NotificationsPerUserId, error) { + notificationsByUserID := types.NotificationsPerUserId{} var err error // Monitoring (premium): machine offline @@ -457,29 +457,9 @@ func collectUserDbNotifications(epoch uint64) (map[types.UserId]map[types.EventN return notificationsByUserID, nil } -func queueNotifications(notificationsByUserID map[types.UserId]map[types.EventName][]types.Notification, useDB *sqlx.DB) { +func queueNotifications(notificationsByUserID types.NotificationsPerUserId, useDB *sqlx.DB) { subByEpoch := map[uint64][]uint64{} - // prevent multiple events being sent with the same subscription id - for user, notifications := range notificationsByUserID { - for eventType, events := range notifications { - filteredEvents := make([]types.Notification, 0) - - for _, ev := range events { - isDuplicate := false - for _, fe := range filteredEvents { - if fe.GetSubscriptionID() == ev.GetSubscriptionID() { - isDuplicate = true - } - } - if !isDuplicate { - filteredEvents = append(filteredEvents, ev) - } - } - notificationsByUserID[user][eventType] = filteredEvents - } - } - err := queueEmailNotifications(notificationsByUserID, useDB) if err != nil { log.Error(err, "error queuing email notifications", 0) @@ -593,7 +573,7 @@ func getNetwork() string { return "" } -func queuePushNotification(notificationsByUserID map[types.UserId]map[types.EventName][]types.Notification, useDB *sqlx.DB) error { +func queuePushNotification(notificationsByUserID types.NotificationsPerUserId, useDB *sqlx.DB) error { userIDs := []types.UserId{} for userID := range notificationsByUserID { userIDs = append(userIDs, userID) @@ -611,7 +591,7 @@ func queuePushNotification(notificationsByUserID map[types.UserId]map[types.Even continue } - go func(userTokens []string, userNotifications map[types.EventName][]types.Notification) { + go func(userTokens []string, userNotifications map[types.EventName]map[types.EventFilter]types.Notification) { var batch []*messaging.Message for event, ns := range userNotifications { for _, n := range ns { @@ -698,7 +678,7 @@ func sendPushNotifications(useDB *sqlx.DB) error { return nil } -func queueEmailNotifications(notificationsByUserID map[types.UserId]map[types.EventName][]types.Notification, useDB *sqlx.DB) error { +func queueEmailNotifications(notificationsByUserID types.NotificationsPerUserId, useDB *sqlx.DB) error { userIDs := []types.UserId{} for userID := range notificationsByUserID { userIDs = append(userIDs, userID) @@ -717,7 +697,7 @@ func queueEmailNotifications(notificationsByUserID map[types.UserId]map[types.Ev // metrics.Errors.WithLabelValues("notifications_mail_not_found").Inc() continue } - go func(userEmail string, userNotifications map[types.EventName][]types.Notification) { + go func(userEmail string, userNotifications map[types.EventName]map[types.EventFilter]types.Notification) { attachments := []types.EmailAttachment{} var msg types.Email @@ -741,7 +721,8 @@ func queueEmailNotifications(notificationsByUserID map[types.UserId]map[types.Ev //nolint:gosec // this is a static string msg.Body += template.HTML(fmt.Sprintf("%s
====

", types.EventLabel[event_title])) unsubURL := "https://" + utils.Config.Frontend.SiteDomain + "/notifications/unsubscribe" - for i, n := range ns { + i := 0 + for _, n := range ns { // Find all unique notification titles for the subject title := n.GetTitle() if _, ok := notificationTitlesMap[title]; !ok { @@ -822,6 +803,7 @@ func queueEmailNotifications(notificationsByUserID map[types.UserId]map[types.Ev } metrics.NotificationsQueued.WithLabelValues("email", string(event)).Inc() + i++ } eventInfo := getEventInfo(event, ns) @@ -893,7 +875,7 @@ func sendEmailNotifications(useDb *sqlx.DB) error { return nil } -func queueWebhookNotifications(notificationsByUserID map[types.UserId]map[types.EventName][]types.Notification, useDB *sqlx.DB) error { +func queueWebhookNotifications(notificationsByUserID types.NotificationsPerUserId, useDB *sqlx.DB) error { for userID, userNotifications := range notificationsByUserID { var webhooks []types.UserWebhook err := useDB.Select(&webhooks, ` @@ -1250,7 +1232,7 @@ func getUrlPart(validatorIndex uint64) string { return fmt.Sprintf(` For more information visit: https://%s/validator/%v.`, utils.Config.Frontend.SiteDomain, validatorIndex, utils.Config.Frontend.SiteDomain, validatorIndex) } -func collectBlockProposalNotifications(notificationsByUserID map[types.UserId]map[types.EventName][]types.Notification, status uint64, eventName types.EventName, epoch uint64) error { +func collectBlockProposalNotifications(notificationsByUserID types.NotificationsPerUserId, status uint64, eventName types.EventName, epoch uint64) error { type dbResult struct { Proposer uint64 `db:"proposer"` Status uint64 `db:"status"` @@ -1342,12 +1324,12 @@ func collectBlockProposalNotifications(notificationsByUserID map[types.UserId]ma Slot: event.Slot, } if _, exists := notificationsByUserID[*sub.UserID]; !exists { - notificationsByUserID[*sub.UserID] = map[types.EventName][]types.Notification{} + notificationsByUserID[*sub.UserID] = map[types.EventName]map[types.EventFilter]types.Notification{} } if _, exists := notificationsByUserID[*sub.UserID][n.GetEventName()]; !exists { - notificationsByUserID[*sub.UserID][n.GetEventName()] = []types.Notification{} + notificationsByUserID[*sub.UserID][n.GetEventName()] = map[types.EventFilter]types.Notification{} } - notificationsByUserID[*sub.UserID][n.GetEventName()] = append(notificationsByUserID[*sub.UserID][n.GetEventName()], n) + notificationsByUserID[*sub.UserID][n.GetEventName()][types.EventFilter(n.EventFilter)] = n metrics.NotificationsCollected.WithLabelValues(string(n.GetEventName())).Inc() } } @@ -1445,7 +1427,7 @@ func (n *validatorProposalNotification) GetInfoMarkdown() string { return generalPart } -func collectAttestationAndOfflineValidatorNotifications(notificationsByUserID map[types.UserId]map[types.EventName][]types.Notification, epoch uint64) error { +func collectAttestationAndOfflineValidatorNotifications(notificationsByUserID types.NotificationsPerUserId, epoch uint64) error { subMap, err := GetSubsForEventFilter(types.ValidatorMissedAttestationEventName) if err != nil { return fmt.Errorf("error getting subscriptions for missted attestations %w", err) @@ -1530,21 +1512,12 @@ func collectAttestationAndOfflineValidatorNotifications(notificationsByUserID ma EventFilter: hex.EncodeToString(event.EventFilter), } if _, exists := notificationsByUserID[*sub.UserID]; !exists { - notificationsByUserID[*sub.UserID] = map[types.EventName][]types.Notification{} + notificationsByUserID[*sub.UserID] = map[types.EventName]map[types.EventFilter]types.Notification{} } if _, exists := notificationsByUserID[*sub.UserID][n.GetEventName()]; !exists { - notificationsByUserID[*sub.UserID][n.GetEventName()] = []types.Notification{} - } - isDuplicate := false - for _, userEvent := range notificationsByUserID[*sub.UserID][n.GetEventName()] { - if userEvent.GetSubscriptionID() == n.SubscriptionID { - isDuplicate = true - } + notificationsByUserID[*sub.UserID][n.GetEventName()] = map[types.EventFilter]types.Notification{} } - if isDuplicate { - continue - } - notificationsByUserID[*sub.UserID][n.GetEventName()] = append(notificationsByUserID[*sub.UserID][n.GetEventName()], n) + notificationsByUserID[*sub.UserID][n.GetEventName()][types.EventFilter(n.EventFilter)] = n metrics.NotificationsCollected.WithLabelValues(string(n.GetEventName())).Inc() } } @@ -1639,7 +1612,7 @@ func collectAttestationAndOfflineValidatorNotifications(notificationsByUserID ma } log.Infof("new event: validator %v detected as offline since epoch %v", validator.Index, epoch) - n := validatorIsOfflineNotification{ + n := &validatorIsOfflineNotification{ SubscriptionID: *sub.ID, ValidatorIndex: validator.Index, IsOffline: true, @@ -1650,23 +1623,13 @@ func collectAttestationAndOfflineValidatorNotifications(notificationsByUserID ma } if _, exists := notificationsByUserID[*sub.UserID]; !exists { - notificationsByUserID[*sub.UserID] = map[types.EventName][]types.Notification{} + notificationsByUserID[*sub.UserID] = map[types.EventName]map[types.EventFilter]types.Notification{} } if _, exists := notificationsByUserID[*sub.UserID][n.GetEventName()]; !exists { - notificationsByUserID[*sub.UserID][n.GetEventName()] = []types.Notification{} - } - isDuplicate := false - for _, userEvent := range notificationsByUserID[*sub.UserID][n.GetEventName()] { - if userEvent.GetSubscriptionID() == n.SubscriptionID { - isDuplicate = true - break - } - } - if isDuplicate { - log.Infof("duplicate offline notification detected") - continue + notificationsByUserID[*sub.UserID][n.GetEventName()] = map[types.EventFilter]types.Notification{} } - notificationsByUserID[*sub.UserID][n.GetEventName()] = append(notificationsByUserID[*sub.UserID][n.GetEventName()], &n) + + notificationsByUserID[*sub.UserID][n.GetEventName()][types.EventFilter(n.EventFilter)] = n metrics.NotificationsCollected.WithLabelValues(string(n.GetEventName())).Inc() } } @@ -1697,7 +1660,7 @@ func collectAttestationAndOfflineValidatorNotifications(notificationsByUserID ma log.Infof("new event: validator %v detected as online again at epoch %v", validator.Index, epoch) - n := validatorIsOfflineNotification{ + n := &validatorIsOfflineNotification{ SubscriptionID: *sub.ID, ValidatorIndex: validator.Index, IsOffline: false, @@ -1709,23 +1672,12 @@ func collectAttestationAndOfflineValidatorNotifications(notificationsByUserID ma } if _, exists := notificationsByUserID[*sub.UserID]; !exists { - notificationsByUserID[*sub.UserID] = map[types.EventName][]types.Notification{} + notificationsByUserID[*sub.UserID] = map[types.EventName]map[types.EventFilter]types.Notification{} } if _, exists := notificationsByUserID[*sub.UserID][n.GetEventName()]; !exists { - notificationsByUserID[*sub.UserID][n.GetEventName()] = []types.Notification{} - } - isDuplicate := false - for _, userEvent := range notificationsByUserID[*sub.UserID][n.GetEventName()] { - if userEvent.GetSubscriptionID() == n.SubscriptionID { - isDuplicate = true - break - } - } - if isDuplicate { - log.Infof("duplicate online notification detected") - continue + notificationsByUserID[*sub.UserID][n.GetEventName()] = map[types.EventFilter]types.Notification{} } - notificationsByUserID[*sub.UserID][n.GetEventName()] = append(notificationsByUserID[*sub.UserID][n.GetEventName()], &n) + notificationsByUserID[*sub.UserID][n.GetEventName()][types.EventFilter(n.EventFilter)] = n metrics.NotificationsCollected.WithLabelValues(string(n.GetEventName())).Inc() } } @@ -1950,7 +1902,7 @@ func (n *validatorGotSlashedNotification) GetInfoMarkdown() string { return generalPart } -func collectValidatorGotSlashedNotifications(notificationsByUserID map[types.UserId]map[types.EventName][]types.Notification, epoch uint64) error { +func collectValidatorGotSlashedNotifications(notificationsByUserID types.NotificationsPerUserId, epoch uint64) error { dbResult, err := db.GetValidatorsGotSlashed(epoch) if err != nil { return fmt.Errorf("error getting slashed validators from database, err: %w", err) @@ -2000,12 +1952,12 @@ func collectValidatorGotSlashedNotifications(notificationsByUserID map[types.Use } if _, exists := notificationsByUserID[sub.UserId]; !exists { - notificationsByUserID[sub.UserId] = map[types.EventName][]types.Notification{} + notificationsByUserID[sub.UserId] = map[types.EventName]map[types.EventFilter]types.Notification{} } if _, exists := notificationsByUserID[sub.UserId][n.GetEventName()]; !exists { - notificationsByUserID[sub.UserId][n.GetEventName()] = []types.Notification{} + notificationsByUserID[sub.UserId][n.GetEventName()] = map[types.EventFilter]types.Notification{} } - notificationsByUserID[sub.UserId][n.GetEventName()] = append(notificationsByUserID[sub.UserId][n.GetEventName()], n) + notificationsByUserID[sub.UserId][n.GetEventName()][types.EventFilter(n.EventFilter)] = n metrics.NotificationsCollected.WithLabelValues(string(n.GetEventName())).Inc() } @@ -2072,7 +2024,7 @@ func (n *validatorWithdrawalNotification) GetInfoMarkdown() string { } // collectWithdrawalNotifications collects all notifications validator withdrawals -func collectWithdrawalNotifications(notificationsByUserID map[types.UserId]map[types.EventName][]types.Notification, epoch uint64) error { +func collectWithdrawalNotifications(notificationsByUserID types.NotificationsPerUserId, epoch uint64) error { // get all users that are subscribed to this event (scale: a few thousand rows depending on how many users we have) subMap, err := GetSubsForEventFilter(types.ValidatorReceivedWithdrawalEventName) if err != nil { @@ -2111,12 +2063,12 @@ func collectWithdrawalNotifications(notificationsByUserID map[types.UserId]map[t UnsubscribeHash: sub.UnsubscribeHash, } if _, exists := notificationsByUserID[*sub.UserID]; !exists { - notificationsByUserID[*sub.UserID] = map[types.EventName][]types.Notification{} + notificationsByUserID[*sub.UserID] = map[types.EventName]map[types.EventFilter]types.Notification{} } if _, exists := notificationsByUserID[*sub.UserID][n.GetEventName()]; !exists { - notificationsByUserID[*sub.UserID][n.GetEventName()] = []types.Notification{} + notificationsByUserID[*sub.UserID][n.GetEventName()] = map[types.EventFilter]types.Notification{} } - notificationsByUserID[*sub.UserID][n.GetEventName()] = append(notificationsByUserID[*sub.UserID][n.GetEventName()], n) + notificationsByUserID[*sub.UserID][n.GetEventName()][types.EventFilter(n.EventFilter)] = n metrics.NotificationsCollected.WithLabelValues(string(n.GetEventName())).Inc() } } @@ -2235,7 +2187,7 @@ func (n *ethClientNotification) GetInfoMarkdown() string { return generalPart } -func collectEthClientNotifications(notificationsByUserID map[types.UserId]map[types.EventName][]types.Notification, eventName types.EventName) error { +func collectEthClientNotifications(notificationsByUserID types.NotificationsPerUserId, eventName types.EventName) error { updatedClients := ethclients.GetUpdatedClients() //only check if there are new updates for _, client := range updatedClients { var dbResult []struct { @@ -2272,12 +2224,12 @@ func collectEthClientNotifications(notificationsByUserID map[types.UserId]map[ty UnsubscribeHash: r.UnsubscribeHash, } if _, exists := notificationsByUserID[r.UserID]; !exists { - notificationsByUserID[r.UserID] = map[types.EventName][]types.Notification{} + notificationsByUserID[r.UserID] = map[types.EventName]map[types.EventFilter]types.Notification{} } if _, exists := notificationsByUserID[r.UserID][n.GetEventName()]; !exists { - notificationsByUserID[r.UserID][n.GetEventName()] = []types.Notification{} + notificationsByUserID[r.UserID][n.GetEventName()] = map[types.EventFilter]types.Notification{} } - notificationsByUserID[r.UserID][n.GetEventName()] = append(notificationsByUserID[r.UserID][n.GetEventName()], n) + notificationsByUserID[r.UserID][n.GetEventName()][types.EventFilter(n.EventFilter)] = n metrics.NotificationsCollected.WithLabelValues(string(n.GetEventName())).Inc() } } @@ -2292,7 +2244,7 @@ type MachineEvents struct { EventThreshold float64 `db:"event_threshold"` } -func collectMonitoringMachineOffline(notificationsByUserID map[types.UserId]map[types.EventName][]types.Notification, epoch uint64) error { +func collectMonitoringMachineOffline(notificationsByUserID types.NotificationsPerUserId, epoch uint64) error { nowTs := time.Now().Unix() return collectMonitoringMachine(notificationsByUserID, types.MonitoringMachineOfflineEventName, 120, // notify condition @@ -2311,7 +2263,7 @@ func isMachineDataRecent(machineData *types.MachineMetricSystemUser) bool { return machineData.CurrentDataInsertTs >= nowTs-60*60 } -func collectMonitoringMachineDiskAlmostFull(notificationsByUserID map[types.UserId]map[types.EventName][]types.Notification, epoch uint64) error { +func collectMonitoringMachineDiskAlmostFull(notificationsByUserID types.NotificationsPerUserId, epoch uint64) error { return collectMonitoringMachine(notificationsByUserID, types.MonitoringMachineDiskAlmostFullEventName, 750, // notify condition func(subscribeData *MachineEvents, machineData *types.MachineMetricSystemUser) bool { @@ -2326,7 +2278,7 @@ func collectMonitoringMachineDiskAlmostFull(notificationsByUserID map[types.User ) } -func collectMonitoringMachineCPULoad(notificationsByUserID map[types.UserId]map[types.EventName][]types.Notification, epoch uint64) error { +func collectMonitoringMachineCPULoad(notificationsByUserID types.NotificationsPerUserId, epoch uint64) error { return collectMonitoringMachine(notificationsByUserID, types.MonitoringMachineCpuLoadEventName, 10, // notify condition func(subscribeData *MachineEvents, machineData *types.MachineMetricSystemUser) bool { @@ -2348,7 +2300,7 @@ func collectMonitoringMachineCPULoad(notificationsByUserID map[types.UserId]map[ ) } -func collectMonitoringMachineMemoryUsage(notificationsByUserID map[types.UserId]map[types.EventName][]types.Notification, epoch uint64) error { +func collectMonitoringMachineMemoryUsage(notificationsByUserID types.NotificationsPerUserId, epoch uint64) error { return collectMonitoringMachine(notificationsByUserID, types.MonitoringMachineMemoryUsageEventName, 10, // notify condition func(subscribeData *MachineEvents, machineData *types.MachineMetricSystemUser) bool { @@ -2369,7 +2321,7 @@ func collectMonitoringMachineMemoryUsage(notificationsByUserID map[types.UserId] var isFirstNotificationCheck = true func collectMonitoringMachine( - notificationsByUserID map[types.UserId]map[types.EventName][]types.Notification, + notificationsByUserID types.NotificationsPerUserId, eventName types.EventName, epochWaitInBetween int, notifyConditionFulfilled func(subscribeData *MachineEvents, machineData *types.MachineMetricSystemUser) bool, @@ -2471,12 +2423,12 @@ func collectMonitoringMachine( } //logrus.Infof("notify %v %v", eventName, n) if _, exists := notificationsByUserID[r.UserID]; !exists { - notificationsByUserID[r.UserID] = map[types.EventName][]types.Notification{} + notificationsByUserID[r.UserID] = map[types.EventName]map[types.EventFilter]types.Notification{} } if _, exists := notificationsByUserID[r.UserID][n.GetEventName()]; !exists { - notificationsByUserID[r.UserID][n.GetEventName()] = []types.Notification{} + notificationsByUserID[r.UserID][n.GetEventName()] = map[types.EventFilter]types.Notification{} } - notificationsByUserID[r.UserID][n.GetEventName()] = append(notificationsByUserID[r.UserID][n.GetEventName()], n) + notificationsByUserID[r.UserID][n.GetEventName()][types.EventFilter(n.GetEventFilter())] = n metrics.NotificationsCollected.WithLabelValues(string(n.GetEventName())).Inc() } @@ -2650,7 +2602,7 @@ func (n *taxReportNotification) GetInfoMarkdown() string { return n.GetInfo(false) } -func collectTaxReportNotificationNotifications(notificationsByUserID map[types.UserId]map[types.EventName][]types.Notification, eventName types.EventName) error { +func collectTaxReportNotificationNotifications(notificationsByUserID types.NotificationsPerUserId, eventName types.EventName) error { lastStatsDay, err := cache.LatestExportedStatisticDay.GetOrDefault(db.GetLastExportedStatisticDay) if err != nil { @@ -2696,12 +2648,12 @@ func collectTaxReportNotificationNotifications(notificationsByUserID map[types.U UnsubscribeHash: r.UnsubscribeHash, } if _, exists := notificationsByUserID[r.UserID]; !exists { - notificationsByUserID[r.UserID] = map[types.EventName][]types.Notification{} + notificationsByUserID[r.UserID] = map[types.EventName]map[types.EventFilter]types.Notification{} } if _, exists := notificationsByUserID[r.UserID][n.GetEventName()]; !exists { - notificationsByUserID[r.UserID][n.GetEventName()] = []types.Notification{} + notificationsByUserID[r.UserID][n.GetEventName()] = map[types.EventFilter]types.Notification{} } - notificationsByUserID[r.UserID][n.GetEventName()] = append(notificationsByUserID[r.UserID][n.GetEventName()], n) + notificationsByUserID[r.UserID][n.GetEventName()][types.EventFilter(n.GetEventFilter())] = n metrics.NotificationsCollected.WithLabelValues(string(n.GetEventName())).Inc() } @@ -2761,7 +2713,7 @@ func (n *networkNotification) GetInfoMarkdown() string { return generalPart } -func collectNetworkNotifications(notificationsByUserID map[types.UserId]map[types.EventName][]types.Notification, eventName types.EventName) error { +func collectNetworkNotifications(notificationsByUserID types.NotificationsPerUserId, eventName types.EventName) error { count := 0 err := db.WriterDb.Get(&count, ` SELECT count(ts) FROM network_liveness WHERE (headepoch-finalizedepoch) > 3 AND ts > now() - interval '60 minutes'; @@ -2800,12 +2752,12 @@ func collectNetworkNotifications(notificationsByUserID map[types.UserId]map[type UnsubscribeHash: r.UnsubscribeHash, } if _, exists := notificationsByUserID[r.UserID]; !exists { - notificationsByUserID[r.UserID] = map[types.EventName][]types.Notification{} + notificationsByUserID[r.UserID] = map[types.EventName]map[types.EventFilter]types.Notification{} } if _, exists := notificationsByUserID[r.UserID][n.GetEventName()]; !exists { - notificationsByUserID[r.UserID][n.GetEventName()] = []types.Notification{} + notificationsByUserID[r.UserID][n.GetEventName()] = map[types.EventFilter]types.Notification{} } - notificationsByUserID[r.UserID][n.GetEventName()] = append(notificationsByUserID[r.UserID][n.GetEventName()], n) + notificationsByUserID[r.UserID][n.GetEventName()][types.EventFilter(n.EventFilter)] = n metrics.NotificationsCollected.WithLabelValues(string(n.GetEventName())).Inc() } } @@ -2861,7 +2813,9 @@ func (n *rocketpoolNotification) GetInfo(includeUrl bool) string { case types.RocketpoolCollateralMinReached: return fmt.Sprintf(`Your RPL collateral has reached your configured threshold at %v%%.`, n.ExtraData) case types.SyncCommitteeSoon: - return getSyncCommitteeSoonInfo([]types.Notification{n}) + return getSyncCommitteeSoonInfo(map[types.EventFilter]types.Notification{ + types.EventFilter(n.EventFilter): n, + }) } return "" @@ -2891,7 +2845,7 @@ func (n *rocketpoolNotification) GetInfoMarkdown() string { return n.GetInfo(false) } -func collectRocketpoolComissionNotifications(notificationsByUserID map[types.UserId]map[types.EventName][]types.Notification, eventName types.EventName) error { +func collectRocketpoolComissionNotifications(notificationsByUserID types.NotificationsPerUserId, eventName types.EventName) error { fee := 0.0 err := db.WriterDb.Get(&fee, ` select current_node_fee from rocketpool_network_stats order by id desc LIMIT 1; @@ -2932,12 +2886,12 @@ func collectRocketpoolComissionNotifications(notificationsByUserID map[types.Use UnsubscribeHash: r.UnsubscribeHash, } if _, exists := notificationsByUserID[r.UserID]; !exists { - notificationsByUserID[r.UserID] = map[types.EventName][]types.Notification{} + notificationsByUserID[r.UserID] = map[types.EventName]map[types.EventFilter]types.Notification{} } if _, exists := notificationsByUserID[r.UserID][n.GetEventName()]; !exists { - notificationsByUserID[r.UserID][n.GetEventName()] = []types.Notification{} + notificationsByUserID[r.UserID][n.GetEventName()] = map[types.EventFilter]types.Notification{} } - notificationsByUserID[r.UserID][n.GetEventName()] = append(notificationsByUserID[r.UserID][n.GetEventName()], n) + notificationsByUserID[r.UserID][n.GetEventName()][types.EventFilter(n.EventFilter)] = n metrics.NotificationsCollected.WithLabelValues(string(n.GetEventName())).Inc() } } @@ -2945,7 +2899,7 @@ func collectRocketpoolComissionNotifications(notificationsByUserID map[types.Use return nil } -func collectRocketpoolRewardClaimRoundNotifications(notificationsByUserID map[types.UserId]map[types.EventName][]types.Notification, eventName types.EventName) error { +func collectRocketpoolRewardClaimRoundNotifications(notificationsByUserID types.NotificationsPerUserId, eventName types.EventName) error { var ts int64 err := db.WriterDb.Get(&ts, ` select date_part('epoch', claim_interval_time_start)::int from rocketpool_network_stats order by id desc LIMIT 1; @@ -2985,12 +2939,12 @@ func collectRocketpoolRewardClaimRoundNotifications(notificationsByUserID map[ty UnsubscribeHash: r.UnsubscribeHash, } if _, exists := notificationsByUserID[r.UserID]; !exists { - notificationsByUserID[r.UserID] = map[types.EventName][]types.Notification{} + notificationsByUserID[r.UserID] = map[types.EventName]map[types.EventFilter]types.Notification{} } if _, exists := notificationsByUserID[r.UserID][n.GetEventName()]; !exists { - notificationsByUserID[r.UserID][n.GetEventName()] = []types.Notification{} + notificationsByUserID[r.UserID][n.GetEventName()] = map[types.EventFilter]types.Notification{} } - notificationsByUserID[r.UserID][n.GetEventName()] = append(notificationsByUserID[r.UserID][n.GetEventName()], n) + notificationsByUserID[r.UserID][n.GetEventName()][types.EventFilter(n.EventFilter)] = n metrics.NotificationsCollected.WithLabelValues(string(n.GetEventName())).Inc() } } @@ -2998,7 +2952,7 @@ func collectRocketpoolRewardClaimRoundNotifications(notificationsByUserID map[ty return nil } -func collectRocketpoolRPLCollateralNotifications(notificationsByUserID map[types.UserId]map[types.EventName][]types.Notification, eventName types.EventName, epoch uint64) error { +func collectRocketpoolRPLCollateralNotifications(notificationsByUserID types.NotificationsPerUserId, eventName types.EventName, epoch uint64) error { subMap, err := GetSubsForEventFilter(eventName) if err != nil { return fmt.Errorf("error getting subscriptions for RocketpoolRPLCollateral %w", err) @@ -3123,12 +3077,12 @@ func collectRocketpoolRPLCollateralNotifications(notificationsByUserID map[types UnsubscribeHash: sub.UnsubscribeHash, } if _, exists := notificationsByUserID[*sub.UserID]; !exists { - notificationsByUserID[*sub.UserID] = map[types.EventName][]types.Notification{} + notificationsByUserID[*sub.UserID] = map[types.EventName]map[types.EventFilter]types.Notification{} } if _, exists := notificationsByUserID[*sub.UserID][n.GetEventName()]; !exists { - notificationsByUserID[*sub.UserID][n.GetEventName()] = []types.Notification{} + notificationsByUserID[*sub.UserID][n.GetEventName()] = map[types.EventFilter]types.Notification{} } - notificationsByUserID[*sub.UserID][n.GetEventName()] = append(notificationsByUserID[*sub.UserID][n.GetEventName()], n) + notificationsByUserID[*sub.UserID][n.GetEventName()][types.EventFilter(n.EventFilter)] = n metrics.NotificationsCollected.WithLabelValues(string(n.GetEventName())).Inc() } @@ -3176,7 +3130,7 @@ func bigFloat(x float64) *big.Float { return new(big.Float).SetFloat64(x) } -func collectSyncCommittee(notificationsByUserID map[types.UserId]map[types.EventName][]types.Notification, eventName types.EventName, epoch uint64) error { +func collectSyncCommittee(notificationsByUserID types.NotificationsPerUserId, eventName types.EventName, epoch uint64) error { slotsPerSyncCommittee := utils.SlotsPerSyncCommittee() currentPeriod := epoch * utils.Config.Chain.ClConfig.SlotsPerEpoch / slotsPerSyncCommittee nextPeriod := currentPeriod + 1 @@ -3232,12 +3186,12 @@ func collectSyncCommittee(notificationsByUserID map[types.UserId]map[types.Event UnsubscribeHash: r.UnsubscribeHash, } if _, exists := notificationsByUserID[r.UserID]; !exists { - notificationsByUserID[r.UserID] = map[types.EventName][]types.Notification{} + notificationsByUserID[r.UserID] = map[types.EventName]map[types.EventFilter]types.Notification{} } if _, exists := notificationsByUserID[r.UserID][n.GetEventName()]; !exists { - notificationsByUserID[r.UserID][n.GetEventName()] = []types.Notification{} + notificationsByUserID[r.UserID][n.GetEventName()] = map[types.EventFilter]types.Notification{} } - notificationsByUserID[r.UserID][n.GetEventName()] = append(notificationsByUserID[r.UserID][n.GetEventName()], n) + notificationsByUserID[r.UserID][n.GetEventName()][types.EventFilter(n.EventFilter)] = n metrics.NotificationsCollected.WithLabelValues(string(n.GetEventName())).Inc() } @@ -3254,7 +3208,7 @@ type WebhookQueue struct { LastTry time.Time `db:"last_try"` } -func getEventInfo(event types.EventName, ns []types.Notification) string { +func getEventInfo(event types.EventName, ns map[types.EventFilter]types.Notification) string { switch event { case types.SyncCommitteeSoon: return getSyncCommitteeSoonInfo(ns) @@ -3265,12 +3219,13 @@ func getEventInfo(event types.EventName, ns []types.Notification) string { return "" } -func getSyncCommitteeSoonInfo(ns []types.Notification) string { +func getSyncCommitteeSoonInfo(ns map[types.EventFilter]types.Notification) string { validators := []string{} var startEpoch, endEpoch string var inTime time.Duration - for i, n := range ns { + i := 0 + for _, n := range ns { n, ok := n.(*rocketpoolNotification) if !ok { log.Error(nil, "Sync committee notification not of type rocketpoolNotification", 0) @@ -3296,6 +3251,7 @@ func getSyncCommitteeSoonInfo(ns []types.Notification) string { } inTime = inTime.Round(time.Second) } + i++ } if len(validators) > 0 { From 9661fee2a15be88d2834bb0e15d61d80dde0a5b0 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Fri, 5 Jul 2024 09:54:42 +0200 Subject: [PATCH 05/94] simplify collecting notifications --- backend/pkg/commons/types/frontend.go | 25 ++++ backend/pkg/notification/notifications.go | 168 +++++++++------------- 2 files changed, 93 insertions(+), 100 deletions(-) diff --git a/backend/pkg/commons/types/frontend.go b/backend/pkg/commons/types/frontend.go index 5367951a1..467f6f5c7 100644 --- a/backend/pkg/commons/types/frontend.go +++ b/backend/pkg/commons/types/frontend.go @@ -4,6 +4,7 @@ import ( "database/sql" "database/sql/driver" "encoding/json" + "fmt" "html/template" "math/big" "strings" @@ -12,6 +13,7 @@ import ( "firebase.google.com/go/messaging" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/gobitfly/beaconchain/pkg/commons/log" "github.com/gobitfly/beaconchain/pkg/consapi/types" "github.com/lib/pq" "github.com/pkg/errors" @@ -21,8 +23,30 @@ import ( type EventName string type EventFilter string + type NotificationsPerUserId map[UserId]map[EventName]map[EventFilter]Notification +func (npui NotificationsPerUserId) AddNotification(n Notification) { + + if n.GetUserId() == 0 { + log.Fatal(fmt.Errorf("Notification user id is 0"), fmt.Sprintf("Notification: %v", n), 0) + } + if n.GetEventName() == "" { + log.Fatal(fmt.Errorf("Notification event name is empty"), fmt.Sprintf("Notification: %v", n), 0) + } + if n.GetEventFilter() == "" { + log.Fatal(fmt.Errorf("Notification event filter is empty"), fmt.Sprintf("Notification: %v", n), 0) + } + + if _, ok := npui[n.GetUserId()]; !ok { + npui[n.GetUserId()] = make(map[EventName]map[EventFilter]Notification) + } + if _, ok := npui[n.GetUserId()][n.GetEventName()]; !ok { + npui[n.GetUserId()][EventName(n.GetEventFilter())] = make(map[EventFilter]Notification) + } + npui[n.GetUserId()][n.GetEventName()][EventFilter(n.GetEventFilter())] = n +} + const ( ValidatorBalanceDecreasedEventName EventName = "validator_balance_decreased" ValidatorMissedProposalEventName EventName = "validator_proposal_missed" @@ -252,6 +276,7 @@ type Notification interface { GetEmailAttachment() *EmailAttachment GetUnsubscribeHash() string GetInfoMarkdown() string + GetUserId() UserId } // func UnMarschal diff --git a/backend/pkg/notification/notifications.go b/backend/pkg/notification/notifications.go index e2db123f1..f39309de9 100644 --- a/backend/pkg/notification/notifications.go +++ b/backend/pkg/notification/notifications.go @@ -1315,6 +1315,7 @@ func collectBlockProposalNotifications(notificationsByUserID types.Notifications log.Infof("creating %v notification for validator %v in epoch %v", eventName, event.Proposer, epoch) n := &validatorProposalNotification{ SubscriptionID: *sub.ID, + UserID: *sub.UserID, ValidatorIndex: event.Proposer, Epoch: epoch, Status: event.Status, @@ -1323,13 +1324,7 @@ func collectBlockProposalNotifications(notificationsByUserID types.Notifications EventFilter: hex.EncodeToString(pubkey), Slot: event.Slot, } - if _, exists := notificationsByUserID[*sub.UserID]; !exists { - notificationsByUserID[*sub.UserID] = map[types.EventName]map[types.EventFilter]types.Notification{} - } - if _, exists := notificationsByUserID[*sub.UserID][n.GetEventName()]; !exists { - notificationsByUserID[*sub.UserID][n.GetEventName()] = map[types.EventFilter]types.Notification{} - } - notificationsByUserID[*sub.UserID][n.GetEventName()][types.EventFilter(n.EventFilter)] = n + notificationsByUserID.AddNotification(n) metrics.NotificationsCollected.WithLabelValues(string(n.GetEventName())).Inc() } } @@ -1339,6 +1334,7 @@ func collectBlockProposalNotifications(notificationsByUserID types.Notifications type validatorProposalNotification struct { SubscriptionID uint64 + UserID types.UserId ValidatorIndex uint64 ValidatorPublicKey string Epoch uint64 @@ -1350,6 +1346,10 @@ type validatorProposalNotification struct { UnsubscribeHash sql.NullString } +func (n *validatorProposalNotification) GetUserId() types.UserId { + return n.UserID +} + func (n *validatorProposalNotification) GetLatestState() string { return "" } @@ -1505,19 +1505,14 @@ func collectAttestationAndOfflineValidatorNotifications(notificationsByUserID ty log.Infof("creating %v notification for validator %v in epoch %v", types.ValidatorMissedAttestationEventName, event.ValidatorIndex, event.Epoch) n := &validatorAttestationNotification{ SubscriptionID: *sub.ID, + UserID: *sub.UserID, ValidatorIndex: event.ValidatorIndex, Epoch: event.Epoch, Status: event.Status, EventName: types.ValidatorMissedAttestationEventName, EventFilter: hex.EncodeToString(event.EventFilter), } - if _, exists := notificationsByUserID[*sub.UserID]; !exists { - notificationsByUserID[*sub.UserID] = map[types.EventName]map[types.EventFilter]types.Notification{} - } - if _, exists := notificationsByUserID[*sub.UserID][n.GetEventName()]; !exists { - notificationsByUserID[*sub.UserID][n.GetEventName()] = map[types.EventFilter]types.Notification{} - } - notificationsByUserID[*sub.UserID][n.GetEventName()][types.EventFilter(n.EventFilter)] = n + notificationsByUserID.AddNotification(n) metrics.NotificationsCollected.WithLabelValues(string(n.GetEventName())).Inc() } } @@ -1622,14 +1617,7 @@ func collectAttestationAndOfflineValidatorNotifications(notificationsByUserID ty EventFilter: hex.EncodeToString(validator.Pubkey), } - if _, exists := notificationsByUserID[*sub.UserID]; !exists { - notificationsByUserID[*sub.UserID] = map[types.EventName]map[types.EventFilter]types.Notification{} - } - if _, exists := notificationsByUserID[*sub.UserID][n.GetEventName()]; !exists { - notificationsByUserID[*sub.UserID][n.GetEventName()] = map[types.EventFilter]types.Notification{} - } - - notificationsByUserID[*sub.UserID][n.GetEventName()][types.EventFilter(n.EventFilter)] = n + notificationsByUserID.AddNotification(n) metrics.NotificationsCollected.WithLabelValues(string(n.GetEventName())).Inc() } } @@ -1662,6 +1650,7 @@ func collectAttestationAndOfflineValidatorNotifications(notificationsByUserID ty n := &validatorIsOfflineNotification{ SubscriptionID: *sub.ID, + UserID: *sub.UserID, ValidatorIndex: validator.Index, IsOffline: false, EventEpoch: epoch, @@ -1671,13 +1660,7 @@ func collectAttestationAndOfflineValidatorNotifications(notificationsByUserID ty EpochsOffline: epochsSinceOffline, } - if _, exists := notificationsByUserID[*sub.UserID]; !exists { - notificationsByUserID[*sub.UserID] = map[types.EventName]map[types.EventFilter]types.Notification{} - } - if _, exists := notificationsByUserID[*sub.UserID][n.GetEventName()]; !exists { - notificationsByUserID[*sub.UserID][n.GetEventName()] = map[types.EventFilter]types.Notification{} - } - notificationsByUserID[*sub.UserID][n.GetEventName()][types.EventFilter(n.EventFilter)] = n + notificationsByUserID.AddNotification(n) metrics.NotificationsCollected.WithLabelValues(string(n.GetEventName())).Inc() } } @@ -1687,6 +1670,7 @@ func collectAttestationAndOfflineValidatorNotifications(notificationsByUserID ty type validatorIsOfflineNotification struct { SubscriptionID uint64 + UserID types.UserId ValidatorIndex uint64 EventEpoch uint64 EpochsOffline uint64 @@ -1697,6 +1681,10 @@ type validatorIsOfflineNotification struct { InternalState string } +func (n *validatorIsOfflineNotification) GetUserId() types.UserId { + return n.UserID +} + func (n *validatorIsOfflineNotification) GetLatestState() string { return n.InternalState } @@ -1762,6 +1750,7 @@ func (n *validatorIsOfflineNotification) GetInfoMarkdown() string { type validatorAttestationNotification struct { SubscriptionID uint64 + UserID types.UserId ValidatorIndex uint64 ValidatorPublicKey string Epoch uint64 @@ -1771,6 +1760,10 @@ type validatorAttestationNotification struct { UnsubscribeHash sql.NullString } +func (n *validatorAttestationNotification) GetUserId() types.UserId { + return n.UserID +} + func (n *validatorAttestationNotification) GetLatestState() string { return "" } @@ -1846,6 +1839,7 @@ func (n *validatorAttestationNotification) GetInfoMarkdown() string { type validatorGotSlashedNotification struct { SubscriptionID uint64 + UserID types.UserId ValidatorIndex uint64 Epoch uint64 Slasher uint64 @@ -1854,6 +1848,10 @@ type validatorGotSlashedNotification struct { UnsubscribeHash sql.NullString } +func (n *validatorGotSlashedNotification) GetUserId() types.UserId { + return n.UserID +} + func (n *validatorGotSlashedNotification) GetLatestState() string { return "" } @@ -1943,6 +1941,7 @@ func collectValidatorGotSlashedNotifications(notificationsByUserID types.Notific n := &validatorGotSlashedNotification{ SubscriptionID: sub.Id, + UserID: sub.UserId, Slasher: event.SlasherIndex, Epoch: event.Epoch, Reason: event.Reason, @@ -1950,14 +1949,7 @@ func collectValidatorGotSlashedNotifications(notificationsByUserID types.Notific EventFilter: hex.EncodeToString(event.SlashedValidatorPubkey), UnsubscribeHash: sub.UnsubscribeHash, } - - if _, exists := notificationsByUserID[sub.UserId]; !exists { - notificationsByUserID[sub.UserId] = map[types.EventName]map[types.EventFilter]types.Notification{} - } - if _, exists := notificationsByUserID[sub.UserId][n.GetEventName()]; !exists { - notificationsByUserID[sub.UserId][n.GetEventName()] = map[types.EventFilter]types.Notification{} - } - notificationsByUserID[sub.UserId][n.GetEventName()][types.EventFilter(n.EventFilter)] = n + notificationsByUserID.AddNotification(n) metrics.NotificationsCollected.WithLabelValues(string(n.GetEventName())).Inc() } @@ -1965,6 +1957,7 @@ func collectValidatorGotSlashedNotifications(notificationsByUserID types.Notific } type validatorWithdrawalNotification struct { + UserID types.UserId SubscriptionID uint64 ValidatorIndex uint64 Epoch uint64 @@ -1975,6 +1968,10 @@ type validatorWithdrawalNotification struct { UnsubscribeHash sql.NullString } +func (n *validatorWithdrawalNotification) GetUserId() types.UserId { + return n.UserID +} + func (n *validatorWithdrawalNotification) GetLatestState() string { return "" } @@ -2054,6 +2051,7 @@ func collectWithdrawalNotifications(notificationsByUserID types.NotificationsPer // log.Infof("creating %v notification for validator %v in epoch %v", types.ValidatorReceivedWithdrawalEventName, event.ValidatorIndex, epoch) n := &validatorWithdrawalNotification{ SubscriptionID: *sub.ID, + UserID: *sub.UserID, ValidatorIndex: event.ValidatorIndex, Epoch: epoch, Slot: event.Slot, @@ -2062,13 +2060,7 @@ func collectWithdrawalNotifications(notificationsByUserID types.NotificationsPer EventFilter: hex.EncodeToString(event.Pubkey), UnsubscribeHash: sub.UnsubscribeHash, } - if _, exists := notificationsByUserID[*sub.UserID]; !exists { - notificationsByUserID[*sub.UserID] = map[types.EventName]map[types.EventFilter]types.Notification{} - } - if _, exists := notificationsByUserID[*sub.UserID][n.GetEventName()]; !exists { - notificationsByUserID[*sub.UserID][n.GetEventName()] = map[types.EventFilter]types.Notification{} - } - notificationsByUserID[*sub.UserID][n.GetEventName()][types.EventFilter(n.EventFilter)] = n + notificationsByUserID.AddNotification(n) metrics.NotificationsCollected.WithLabelValues(string(n.GetEventName())).Inc() } } @@ -2086,6 +2078,10 @@ type ethClientNotification struct { UnsubscribeHash sql.NullString } +func (n *ethClientNotification) GetUserId() types.UserId { + return n.UserID +} + func (n *ethClientNotification) GetLatestState() string { return "" } @@ -2223,13 +2219,7 @@ func collectEthClientNotifications(notificationsByUserID types.NotificationsPerU EthClient: client.Name, UnsubscribeHash: r.UnsubscribeHash, } - if _, exists := notificationsByUserID[r.UserID]; !exists { - notificationsByUserID[r.UserID] = map[types.EventName]map[types.EventFilter]types.Notification{} - } - if _, exists := notificationsByUserID[r.UserID][n.GetEventName()]; !exists { - notificationsByUserID[r.UserID][n.GetEventName()] = map[types.EventFilter]types.Notification{} - } - notificationsByUserID[r.UserID][n.GetEventName()][types.EventFilter(n.EventFilter)] = n + notificationsByUserID.AddNotification(n) metrics.NotificationsCollected.WithLabelValues(string(n.GetEventName())).Inc() } } @@ -2422,13 +2412,7 @@ func collectMonitoringMachine( UnsubscribeHash: r.UnsubscribeHash, } //logrus.Infof("notify %v %v", eventName, n) - if _, exists := notificationsByUserID[r.UserID]; !exists { - notificationsByUserID[r.UserID] = map[types.EventName]map[types.EventFilter]types.Notification{} - } - if _, exists := notificationsByUserID[r.UserID][n.GetEventName()]; !exists { - notificationsByUserID[r.UserID][n.GetEventName()] = map[types.EventFilter]types.Notification{} - } - notificationsByUserID[r.UserID][n.GetEventName()][types.EventFilter(n.GetEventFilter())] = n + notificationsByUserID.AddNotification(n) metrics.NotificationsCollected.WithLabelValues(string(n.GetEventName())).Inc() } @@ -2449,6 +2433,10 @@ type monitorMachineNotification struct { UnsubscribeHash sql.NullString } +func (n *monitorMachineNotification) GetUserId() types.UserId { + return n.UserID +} + func (n *monitorMachineNotification) GetLatestState() string { return "" } @@ -2528,6 +2516,10 @@ type taxReportNotification struct { UnsubscribeHash sql.NullString } +func (n *taxReportNotification) GetUserId() types.UserId { + return n.UserID +} + func (n *taxReportNotification) GetLatestState() string { return "" } @@ -2647,13 +2639,7 @@ func collectTaxReportNotificationNotifications(notificationsByUserID types.Notif EventFilter: r.EventFilter, UnsubscribeHash: r.UnsubscribeHash, } - if _, exists := notificationsByUserID[r.UserID]; !exists { - notificationsByUserID[r.UserID] = map[types.EventName]map[types.EventFilter]types.Notification{} - } - if _, exists := notificationsByUserID[r.UserID][n.GetEventName()]; !exists { - notificationsByUserID[r.UserID][n.GetEventName()] = map[types.EventFilter]types.Notification{} - } - notificationsByUserID[r.UserID][n.GetEventName()][types.EventFilter(n.GetEventFilter())] = n + notificationsByUserID.AddNotification(n) metrics.NotificationsCollected.WithLabelValues(string(n.GetEventName())).Inc() } @@ -2668,6 +2654,10 @@ type networkNotification struct { UnsubscribeHash sql.NullString } +func (n *networkNotification) GetUserId() types.UserId { + return n.UserID +} + func (n *networkNotification) GetLatestState() string { return "" } @@ -2751,13 +2741,8 @@ func collectNetworkNotifications(notificationsByUserID types.NotificationsPerUse EventFilter: r.EventFilter, UnsubscribeHash: r.UnsubscribeHash, } - if _, exists := notificationsByUserID[r.UserID]; !exists { - notificationsByUserID[r.UserID] = map[types.EventName]map[types.EventFilter]types.Notification{} - } - if _, exists := notificationsByUserID[r.UserID][n.GetEventName()]; !exists { - notificationsByUserID[r.UserID][n.GetEventName()] = map[types.EventFilter]types.Notification{} - } - notificationsByUserID[r.UserID][n.GetEventName()][types.EventFilter(n.EventFilter)] = n + + notificationsByUserID.AddNotification(n) metrics.NotificationsCollected.WithLabelValues(string(n.GetEventName())).Inc() } } @@ -2775,6 +2760,10 @@ type rocketpoolNotification struct { UnsubscribeHash sql.NullString } +func (n *rocketpoolNotification) GetUserId() types.UserId { + return n.UserID +} + func (n *rocketpoolNotification) GetLatestState() string { return "" } @@ -2885,13 +2874,8 @@ func collectRocketpoolComissionNotifications(notificationsByUserID types.Notific ExtraData: strconv.FormatInt(int64(fee*100), 10) + "%", UnsubscribeHash: r.UnsubscribeHash, } - if _, exists := notificationsByUserID[r.UserID]; !exists { - notificationsByUserID[r.UserID] = map[types.EventName]map[types.EventFilter]types.Notification{} - } - if _, exists := notificationsByUserID[r.UserID][n.GetEventName()]; !exists { - notificationsByUserID[r.UserID][n.GetEventName()] = map[types.EventFilter]types.Notification{} - } - notificationsByUserID[r.UserID][n.GetEventName()][types.EventFilter(n.EventFilter)] = n + + notificationsByUserID.AddNotification(n) metrics.NotificationsCollected.WithLabelValues(string(n.GetEventName())).Inc() } } @@ -2938,13 +2922,8 @@ func collectRocketpoolRewardClaimRoundNotifications(notificationsByUserID types. EventName: eventName, UnsubscribeHash: r.UnsubscribeHash, } - if _, exists := notificationsByUserID[r.UserID]; !exists { - notificationsByUserID[r.UserID] = map[types.EventName]map[types.EventFilter]types.Notification{} - } - if _, exists := notificationsByUserID[r.UserID][n.GetEventName()]; !exists { - notificationsByUserID[r.UserID][n.GetEventName()] = map[types.EventFilter]types.Notification{} - } - notificationsByUserID[r.UserID][n.GetEventName()][types.EventFilter(n.EventFilter)] = n + + notificationsByUserID.AddNotification(n) metrics.NotificationsCollected.WithLabelValues(string(n.GetEventName())).Inc() } } @@ -3076,13 +3055,8 @@ func collectRocketpoolRPLCollateralNotifications(notificationsByUserID types.Not ExtraData: strings.TrimRight(strings.TrimRight(fmt.Sprintf("%.2f", threshold*100), "0"), "."), UnsubscribeHash: sub.UnsubscribeHash, } - if _, exists := notificationsByUserID[*sub.UserID]; !exists { - notificationsByUserID[*sub.UserID] = map[types.EventName]map[types.EventFilter]types.Notification{} - } - if _, exists := notificationsByUserID[*sub.UserID][n.GetEventName()]; !exists { - notificationsByUserID[*sub.UserID][n.GetEventName()] = map[types.EventFilter]types.Notification{} - } - notificationsByUserID[*sub.UserID][n.GetEventName()][types.EventFilter(n.EventFilter)] = n + + notificationsByUserID.AddNotification(n) metrics.NotificationsCollected.WithLabelValues(string(n.GetEventName())).Inc() } @@ -3185,13 +3159,7 @@ func collectSyncCommittee(notificationsByUserID types.NotificationsPerUserId, ev ExtraData: fmt.Sprintf("%v|%v|%v", mapping[r.EventFilter], nextPeriod*utils.Config.Chain.ClConfig.EpochsPerSyncCommitteePeriod, (nextPeriod+1)*utils.Config.Chain.ClConfig.EpochsPerSyncCommitteePeriod), UnsubscribeHash: r.UnsubscribeHash, } - if _, exists := notificationsByUserID[r.UserID]; !exists { - notificationsByUserID[r.UserID] = map[types.EventName]map[types.EventFilter]types.Notification{} - } - if _, exists := notificationsByUserID[r.UserID][n.GetEventName()]; !exists { - notificationsByUserID[r.UserID][n.GetEventName()] = map[types.EventFilter]types.Notification{} - } - notificationsByUserID[r.UserID][n.GetEventName()][types.EventFilter(n.EventFilter)] = n + notificationsByUserID.AddNotification(n) metrics.NotificationsCollected.WithLabelValues(string(n.GetEventName())).Inc() } From ec0f3c9bedfa00fe196405a087d607f92094f222 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Sat, 6 Jul 2024 10:23:29 +0200 Subject: [PATCH 06/94] refractor notification structs --- backend/pkg/commons/types/frontend.go | 61 ++++ backend/pkg/notification/notifications.go | 383 ++++++---------------- 2 files changed, 157 insertions(+), 287 deletions(-) diff --git a/backend/pkg/commons/types/frontend.go b/backend/pkg/commons/types/frontend.go index 467f6f5c7..a51e5e4c2 100644 --- a/backend/pkg/commons/types/frontend.go +++ b/backend/pkg/commons/types/frontend.go @@ -279,6 +279,67 @@ type Notification interface { GetUserId() UserId } +type NotificationBaseImpl struct { + LatestState string + SubscriptionID uint64 + EventName EventName + Epoch uint64 + Info string + Title string + EventFilter string + EmailAttachment *EmailAttachment + UnsubscribeHash sql.NullString + InfoMarkdown string + UserID UserId +} + +func (n NotificationBaseImpl) GetLatestState() string { + return n.LatestState +} + +func (n NotificationBaseImpl) GetSubscriptionID() uint64 { + return n.SubscriptionID +} + +func (n NotificationBaseImpl) GetEventName() EventName { + return n.EventName +} + +func (n NotificationBaseImpl) GetEpoch() uint64 { + return n.Epoch +} + +func (n NotificationBaseImpl) GetInfo(includeUrl bool) string { + return n.Info +} + +func (n NotificationBaseImpl) GetTitle() string { + return n.Title +} + +func (n NotificationBaseImpl) GetEventFilter() string { + return n.EventFilter +} + +func (n NotificationBaseImpl) GetEmailAttachment() *EmailAttachment { + return n.EmailAttachment +} + +func (n NotificationBaseImpl) GetUnsubscribeHash() string { + if n.UnsubscribeHash.Valid { + return n.UnsubscribeHash.String + } + return "" +} + +func (n NotificationBaseImpl) GetInfoMarkdown() string { + return n.InfoMarkdown +} + +func (n NotificationBaseImpl) GetUserId() UserId { + return n.UserID +} + // func UnMarschal type Subscription struct { diff --git a/backend/pkg/notification/notifications.go b/backend/pkg/notification/notifications.go index f39309de9..b177a8d74 100644 --- a/backend/pkg/notification/notifications.go +++ b/backend/pkg/notification/notifications.go @@ -1504,13 +1504,15 @@ func collectAttestationAndOfflineValidatorNotifications(notificationsByUserID ty log.Infof("creating %v notification for validator %v in epoch %v", types.ValidatorMissedAttestationEventName, event.ValidatorIndex, event.Epoch) n := &validatorAttestationNotification{ - SubscriptionID: *sub.ID, - UserID: *sub.UserID, + NotificationBaseImpl: types.NotificationBaseImpl{ + SubscriptionID: *sub.ID, + UserID: *sub.UserID, + Epoch: event.Epoch, + EventName: types.ValidatorMissedAttestationEventName, + EventFilter: hex.EncodeToString(event.EventFilter), + }, ValidatorIndex: event.ValidatorIndex, - Epoch: event.Epoch, Status: event.Status, - EventName: types.ValidatorMissedAttestationEventName, - EventFilter: hex.EncodeToString(event.EventFilter), } notificationsByUserID.AddNotification(n) metrics.NotificationsCollected.WithLabelValues(string(n.GetEventName())).Inc() @@ -1608,13 +1610,15 @@ func collectAttestationAndOfflineValidatorNotifications(notificationsByUserID ty log.Infof("new event: validator %v detected as offline since epoch %v", validator.Index, epoch) n := &validatorIsOfflineNotification{ - SubscriptionID: *sub.ID, + NotificationBaseImpl: types.NotificationBaseImpl{ + SubscriptionID: *sub.ID, + Epoch: epoch, + EventName: types.ValidatorIsOfflineEventName, + LatestState: fmt.Sprint(epoch - 2), // first epoch the validator stopped attesting + EventFilter: hex.EncodeToString(validator.Pubkey), + }, ValidatorIndex: validator.Index, IsOffline: true, - EventEpoch: epoch, - EventName: types.ValidatorIsOfflineEventName, - InternalState: fmt.Sprint(epoch - 2), // first epoch the validator stopped attesting - EventFilter: hex.EncodeToString(validator.Pubkey), } notificationsByUserID.AddNotification(n) @@ -1632,7 +1636,7 @@ func collectAttestationAndOfflineValidatorNotifications(notificationsByUserID ty originalLastSeenEpoch, err := strconv.ParseUint(sub.State.String, 10, 64) if err != nil { - // i have no idea what just happened. + // I have no idea what just happened. return fmt.Errorf("this should never happen. couldn't parse state as uint64: %v", err) } @@ -1649,14 +1653,16 @@ func collectAttestationAndOfflineValidatorNotifications(notificationsByUserID ty log.Infof("new event: validator %v detected as online again at epoch %v", validator.Index, epoch) n := &validatorIsOfflineNotification{ - SubscriptionID: *sub.ID, - UserID: *sub.UserID, + NotificationBaseImpl: types.NotificationBaseImpl{ + SubscriptionID: *sub.ID, + UserID: *sub.UserID, + Epoch: epoch, + EventName: types.ValidatorIsOfflineEventName, + EventFilter: hex.EncodeToString(validator.Pubkey), + LatestState: "-", + }, ValidatorIndex: validator.Index, IsOffline: false, - EventEpoch: epoch, - EventName: types.ValidatorIsOfflineEventName, - InternalState: "-", - EventFilter: hex.EncodeToString(validator.Pubkey), EpochsOffline: epochsSinceOffline, } @@ -1669,50 +1675,26 @@ func collectAttestationAndOfflineValidatorNotifications(notificationsByUserID ty } type validatorIsOfflineNotification struct { - SubscriptionID uint64 - UserID types.UserId - ValidatorIndex uint64 - EventEpoch uint64 - EpochsOffline uint64 - IsOffline bool - EventName types.EventName - EventFilter string - UnsubscribeHash sql.NullString - InternalState string -} - -func (n *validatorIsOfflineNotification) GetUserId() types.UserId { - return n.UserID -} - -func (n *validatorIsOfflineNotification) GetLatestState() string { - return n.InternalState -} + types.NotificationBaseImpl -func (n *validatorIsOfflineNotification) GetSubscriptionID() uint64 { - return n.SubscriptionID -} - -func (n *validatorIsOfflineNotification) GetEventName() types.EventName { - return n.EventName -} - -func (n *validatorIsOfflineNotification) GetEpoch() uint64 { - return n.EventEpoch + ValidatorIndex uint64 + EpochsOffline uint64 + IsOffline bool } +// Overwrite specific methods func (n *validatorIsOfflineNotification) GetInfo(includeUrl bool) string { if n.IsOffline { if includeUrl { - return fmt.Sprintf(`Validator %[1]v is offline since epoch %[2]s).`, n.ValidatorIndex, n.InternalState, utils.Config.Frontend.SiteDomain) + return fmt.Sprintf(`Validator %[1]v is offline since epoch %[2]s).`, n.ValidatorIndex, n.LatestState, utils.Config.Frontend.SiteDomain) } else { - return fmt.Sprintf(`Validator %v is offline since epoch %s.`, n.ValidatorIndex, n.InternalState) + return fmt.Sprintf(`Validator %v is offline since epoch %s.`, n.ValidatorIndex, n.LatestState) } } else { if includeUrl { - return fmt.Sprintf(`Validator %[1]v is back online since epoch %[2]v (was offline for %[4]v epoch(s)).`, n.ValidatorIndex, n.EventEpoch, utils.Config.Frontend.SiteDomain, n.EpochsOffline) + return fmt.Sprintf(`Validator %[1]v is back online since epoch %[2]v (was offline for %[4]v epoch(s)).`, n.ValidatorIndex, n.Epoch, utils.Config.Frontend.SiteDomain, n.EpochsOffline) } else { - return fmt.Sprintf(`Validator %v is back online since epoch %v (was offline for %v epoch(s)).`, n.ValidatorIndex, n.EventEpoch, n.EpochsOffline) + return fmt.Sprintf(`Validator %v is back online since epoch %v (was offline for %v epoch(s)).`, n.ValidatorIndex, n.Epoch, n.EpochsOffline) } } } @@ -1725,59 +1707,24 @@ func (n *validatorIsOfflineNotification) GetTitle() string { } } -func (n *validatorIsOfflineNotification) GetEventFilter() string { - return n.EventFilter -} - -func (n *validatorIsOfflineNotification) GetEmailAttachment() *types.EmailAttachment { - return nil -} - -func (n *validatorIsOfflineNotification) GetUnsubscribeHash() string { - if n.UnsubscribeHash.Valid { - return n.UnsubscribeHash.String - } - return "" -} - func (n *validatorIsOfflineNotification) GetInfoMarkdown() string { if n.IsOffline { - return fmt.Sprintf(`Validator [%[1]v](https://%[3]v/validator/%[1]v) is offline since epoch [%[2]v](https://%[3]v/epoch/%[2]v).`, n.ValidatorIndex, n.EventEpoch, utils.Config.Frontend.SiteDomain) + return fmt.Sprintf(`Validator [%[1]v](https://%[3]v/validator/%[1]v) is offline since epoch [%[2]v](https://%[3]v/epoch/%[2]v).`, n.ValidatorIndex, n.Epoch, utils.Config.Frontend.SiteDomain) } else { - return fmt.Sprintf(`Validator [%[1]v](https://%[3]v/validator/%[1]v) is back online since epoch [%[2]v](https://%[3]v/epoch/%[2]v) (was offline for %[4]v epoch(s)).`, n.ValidatorIndex, n.EventEpoch, utils.Config.Frontend.SiteDomain, n.EpochsOffline) + return fmt.Sprintf(`Validator [%[1]v](https://%[3]v/validator/%[1]v) is back online since epoch [%[2]v](https://%[3]v/epoch/%[2]v) (was offline for %[4]v epoch(s)).`, n.ValidatorIndex, n.Epoch, utils.Config.Frontend.SiteDomain, n.EpochsOffline) } } +func (n *validatorIsOfflineNotification) GetEventName() types.EventName { + return types.ValidatorIsOfflineEventName +} + type validatorAttestationNotification struct { - SubscriptionID uint64 - UserID types.UserId + types.NotificationBaseImpl + ValidatorIndex uint64 ValidatorPublicKey string - Epoch uint64 Status uint64 // * Can be 0 = scheduled | missed, 1 executed - EventName types.EventName - EventFilter string - UnsubscribeHash sql.NullString -} - -func (n *validatorAttestationNotification) GetUserId() types.UserId { - return n.UserID -} - -func (n *validatorAttestationNotification) GetLatestState() string { - return "" -} - -func (n *validatorAttestationNotification) GetSubscriptionID() uint64 { - return n.SubscriptionID -} - -func (n *validatorAttestationNotification) GetEventName() types.EventName { - return n.EventName -} - -func (n *validatorAttestationNotification) GetEpoch() uint64 { - return n.Epoch } func (n *validatorAttestationNotification) GetInfo(includeUrl bool) string { @@ -1811,21 +1758,6 @@ func (n *validatorAttestationNotification) GetTitle() string { return "-" } -func (n *validatorAttestationNotification) GetEventFilter() string { - return n.EventFilter -} - -func (n *validatorAttestationNotification) GetEmailAttachment() *types.EmailAttachment { - return nil -} - -func (n *validatorAttestationNotification) GetUnsubscribeHash() string { - if n.UnsubscribeHash.Valid { - return n.UnsubscribeHash.String - } - return "" -} - func (n *validatorAttestationNotification) GetInfoMarkdown() string { var generalPart = "" switch n.Status { @@ -1837,46 +1769,16 @@ func (n *validatorAttestationNotification) GetInfoMarkdown() string { return generalPart } -type validatorGotSlashedNotification struct { - SubscriptionID uint64 - UserID types.UserId - ValidatorIndex uint64 - Epoch uint64 - Slasher uint64 - Reason string - EventFilter string - UnsubscribeHash sql.NullString -} - -func (n *validatorGotSlashedNotification) GetUserId() types.UserId { - return n.UserID -} - -func (n *validatorGotSlashedNotification) GetLatestState() string { - return "" -} - -func (n *validatorGotSlashedNotification) GetUnsubscribeHash() string { - if n.UnsubscribeHash.Valid { - return n.UnsubscribeHash.String - } - return "" -} - -func (n *validatorGotSlashedNotification) GetEmailAttachment() *types.EmailAttachment { - return nil -} - -func (n *validatorGotSlashedNotification) GetSubscriptionID() uint64 { - return n.SubscriptionID +func (n *validatorAttestationNotification) GetEventName() types.EventName { + return types.ValidatorMissedAttestationEventName } -func (n *validatorGotSlashedNotification) GetEpoch() uint64 { - return n.Epoch -} +type validatorGotSlashedNotification struct { + types.NotificationBaseImpl -func (n *validatorGotSlashedNotification) GetEventName() types.EventName { - return types.ValidatorGotSlashedEventName + ValidatorIndex uint64 + Slasher uint64 + Reason string } func (n *validatorGotSlashedNotification) GetInfo(includeUrl bool) string { @@ -1891,8 +1793,8 @@ func (n *validatorGotSlashedNotification) GetTitle() string { return "Validator got Slashed" } -func (n *validatorGotSlashedNotification) GetEventFilter() string { - return n.EventFilter +func (n *validatorGotSlashedNotification) GetEventName() types.EventName { + return types.ValidatorGotSlashedEventName } func (n *validatorGotSlashedNotification) GetInfoMarkdown() string { @@ -1940,14 +1842,16 @@ func collectValidatorGotSlashedNotifications(notificationsByUserID types.Notific log.Infof("creating %v notification for validator %v in epoch %v", event.SlashedValidatorPubkey, event.Reason, epoch) n := &validatorGotSlashedNotification{ - SubscriptionID: sub.Id, - UserID: sub.UserId, - Slasher: event.SlasherIndex, - Epoch: event.Epoch, - Reason: event.Reason, - ValidatorIndex: event.SlashedValidatorIndex, - EventFilter: hex.EncodeToString(event.SlashedValidatorPubkey), - UnsubscribeHash: sub.UnsubscribeHash, + NotificationBaseImpl: types.NotificationBaseImpl{ + SubscriptionID: sub.Id, + UserID: sub.UserId, + Epoch: event.Epoch, + EventFilter: hex.EncodeToString(event.SlashedValidatorPubkey), + UnsubscribeHash: sub.UnsubscribeHash, + }, + Slasher: event.SlasherIndex, + Reason: event.Reason, + ValidatorIndex: event.SlashedValidatorIndex, } notificationsByUserID.AddNotification(n) metrics.NotificationsCollected.WithLabelValues(string(n.GetEventName())).Inc() @@ -1957,42 +1861,13 @@ func collectValidatorGotSlashedNotifications(notificationsByUserID types.Notific } type validatorWithdrawalNotification struct { - UserID types.UserId - SubscriptionID uint64 - ValidatorIndex uint64 - Epoch uint64 - Slot uint64 - Amount uint64 - Address []byte - EventFilter string - UnsubscribeHash sql.NullString -} - -func (n *validatorWithdrawalNotification) GetUserId() types.UserId { - return n.UserID -} + types.NotificationBaseImpl -func (n *validatorWithdrawalNotification) GetLatestState() string { - return "" -} - -func (n *validatorWithdrawalNotification) GetUnsubscribeHash() string { - if n.UnsubscribeHash.Valid { - return n.UnsubscribeHash.String - } - return "" -} - -func (n *validatorWithdrawalNotification) GetEmailAttachment() *types.EmailAttachment { - return nil -} - -func (n *validatorWithdrawalNotification) GetSubscriptionID() uint64 { - return n.SubscriptionID -} - -func (n *validatorWithdrawalNotification) GetEpoch() uint64 { - return n.Epoch + ValidatorIndex uint64 + Epoch uint64 + Slot uint64 + Amount uint64 + Address []byte } func (n *validatorWithdrawalNotification) GetEventName() types.EventName { @@ -2011,10 +1886,6 @@ func (n *validatorWithdrawalNotification) GetTitle() string { return "Withdrawal Processed" } -func (n *validatorWithdrawalNotification) GetEventFilter() string { - return n.EventFilter -} - func (n *validatorWithdrawalNotification) GetInfoMarkdown() string { generalPart := fmt.Sprintf(`An automatic withdrawal of %[2]v has been processed for validator [%[1]v](https://%[6]v/validator/%[1]v) during slot [%[3]v](https://%[6]v/slot/%[3]v). The funds have been sent to: [%[4]v](https://%[6]v/address/0x%[5]x).`, n.ValidatorIndex, utils.FormatClCurrencyString(n.Amount, utils.Config.Frontend.MainCurrency, 6, true, false, false), n.Slot, utils.FormatHashRaw(n.Address), n.Address, utils.Config.Frontend.SiteDomain) return generalPart @@ -2050,15 +1921,17 @@ func collectWithdrawalNotifications(notificationsByUserID types.NotificationsPer } // log.Infof("creating %v notification for validator %v in epoch %v", types.ValidatorReceivedWithdrawalEventName, event.ValidatorIndex, epoch) n := &validatorWithdrawalNotification{ - SubscriptionID: *sub.ID, - UserID: *sub.UserID, - ValidatorIndex: event.ValidatorIndex, - Epoch: epoch, - Slot: event.Slot, - Amount: event.Amount, - Address: event.Address, - EventFilter: hex.EncodeToString(event.Pubkey), - UnsubscribeHash: sub.UnsubscribeHash, + NotificationBaseImpl: types.NotificationBaseImpl{ + SubscriptionID: *sub.ID, + UserID: *sub.UserID, + EventFilter: hex.EncodeToString(event.Pubkey), + UnsubscribeHash: sub.UnsubscribeHash, + }, + ValidatorIndex: event.ValidatorIndex, + Epoch: epoch, + Slot: event.Slot, + Amount: event.Amount, + Address: event.Address, } notificationsByUserID.AddNotification(n) metrics.NotificationsCollected.WithLabelValues(string(n.GetEventName())).Inc() @@ -2070,39 +1943,9 @@ func collectWithdrawalNotifications(notificationsByUserID types.NotificationsPer } type ethClientNotification struct { - SubscriptionID uint64 - UserID types.UserId - Epoch uint64 - EthClient string - EventFilter string - UnsubscribeHash sql.NullString -} - -func (n *ethClientNotification) GetUserId() types.UserId { - return n.UserID -} - -func (n *ethClientNotification) GetLatestState() string { - return "" -} - -func (n *ethClientNotification) GetUnsubscribeHash() string { - if n.UnsubscribeHash.Valid { - return n.UnsubscribeHash.String - } - return "" -} - -func (n *ethClientNotification) GetEmailAttachment() *types.EmailAttachment { - return nil -} + types.NotificationBaseImpl -func (n *ethClientNotification) GetSubscriptionID() uint64 { - return n.SubscriptionID -} - -func (n *ethClientNotification) GetEpoch() uint64 { - return n.Epoch + EthClient string } func (n *ethClientNotification) GetEventName() types.EventName { @@ -2147,10 +1990,6 @@ func (n *ethClientNotification) GetTitle() string { return fmt.Sprintf("New %s update", n.EthClient) } -func (n *ethClientNotification) GetEventFilter() string { - return n.EventFilter -} - func (n *ethClientNotification) GetInfoMarkdown() string { url := "" switch n.EthClient { @@ -2212,12 +2051,14 @@ func collectEthClientNotifications(notificationsByUserID types.NotificationsPerU for _, r := range dbResult { n := ðClientNotification{ - SubscriptionID: r.SubscriptionID, - UserID: r.UserID, - Epoch: r.Epoch, - EventFilter: r.EventFilter, - EthClient: client.Name, - UnsubscribeHash: r.UnsubscribeHash, + NotificationBaseImpl: types.NotificationBaseImpl{ + SubscriptionID: r.SubscriptionID, + UserID: r.UserID, + Epoch: r.Epoch, + EventFilter: r.EventFilter, + UnsubscribeHash: r.UnsubscribeHash, + }, + EthClient: client.Name, } notificationsByUserID.AddNotification(n) metrics.NotificationsCollected.WithLabelValues(string(n.GetEventName())).Inc() @@ -2404,12 +2245,14 @@ func collectMonitoringMachine( for _, r := range result { n := &monitorMachineNotification{ - SubscriptionID: r.SubscriptionID, - MachineName: r.MachineName, - UserID: r.UserID, - EventName: eventName, - Epoch: epoch, - UnsubscribeHash: r.UnsubscribeHash, + NotificationBaseImpl: types.NotificationBaseImpl{ + SubscriptionID: r.SubscriptionID, + UserID: r.UserID, + EventName: eventName, + Epoch: epoch, + UnsubscribeHash: r.UnsubscribeHash, + }, + MachineName: r.MachineName, } //logrus.Infof("notify %v %v", eventName, n) notificationsByUserID.AddNotification(n) @@ -2425,43 +2268,9 @@ func collectMonitoringMachine( } type monitorMachineNotification struct { - SubscriptionID uint64 - MachineName string - UserID types.UserId - Epoch uint64 - EventName types.EventName - UnsubscribeHash sql.NullString -} + types.NotificationBaseImpl -func (n *monitorMachineNotification) GetUserId() types.UserId { - return n.UserID -} - -func (n *monitorMachineNotification) GetLatestState() string { - return "" -} - -func (n *monitorMachineNotification) GetUnsubscribeHash() string { - if n.UnsubscribeHash.Valid { - return n.UnsubscribeHash.String - } - return "" -} - -func (n *monitorMachineNotification) GetEmailAttachment() *types.EmailAttachment { - return nil -} - -func (n *monitorMachineNotification) GetSubscriptionID() uint64 { - return n.SubscriptionID -} - -func (n *monitorMachineNotification) GetEpoch() uint64 { - return n.Epoch -} - -func (n *monitorMachineNotification) GetEventName() types.EventName { - return n.EventName + MachineName string } func (n *monitorMachineNotification) GetInfo(includeUrl bool) string { From 7e235b241195533dd37db02a5ff93b00e6c067bf Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Sat, 6 Jul 2024 10:30:15 +0200 Subject: [PATCH 07/94] add missing structs --- backend/pkg/notification/notifications.go | 258 ++++++---------------- 1 file changed, 65 insertions(+), 193 deletions(-) diff --git a/backend/pkg/notification/notifications.go b/backend/pkg/notification/notifications.go index b177a8d74..3a2dc4ce8 100644 --- a/backend/pkg/notification/notifications.go +++ b/backend/pkg/notification/notifications.go @@ -1314,14 +1314,16 @@ func collectBlockProposalNotifications(notificationsByUserID types.Notifications } log.Infof("creating %v notification for validator %v in epoch %v", eventName, event.Proposer, epoch) n := &validatorProposalNotification{ - SubscriptionID: *sub.ID, - UserID: *sub.UserID, + NotificationBaseImpl: types.NotificationBaseImpl{ + SubscriptionID: *sub.ID, + UserID: *sub.UserID, + Epoch: epoch, + EventName: eventName, + EventFilter: hex.EncodeToString(pubkey), + }, ValidatorIndex: event.Proposer, - Epoch: epoch, Status: event.Status, - EventName: eventName, Reward: event.ExecRewardETH, - EventFilter: hex.EncodeToString(pubkey), Slot: event.Slot, } notificationsByUserID.AddNotification(n) @@ -1333,48 +1335,12 @@ func collectBlockProposalNotifications(notificationsByUserID types.Notifications } type validatorProposalNotification struct { - SubscriptionID uint64 - UserID types.UserId - ValidatorIndex uint64 - ValidatorPublicKey string - Epoch uint64 - Slot uint64 - Status uint64 // * Can be 0 = scheduled, 1 executed, 2 missed */ - EventName types.EventName - EventFilter string - Reward float64 - UnsubscribeHash sql.NullString -} - -func (n *validatorProposalNotification) GetUserId() types.UserId { - return n.UserID -} - -func (n *validatorProposalNotification) GetLatestState() string { - return "" -} - -func (n *validatorProposalNotification) GetUnsubscribeHash() string { - if n.UnsubscribeHash.Valid { - return n.UnsubscribeHash.String - } - return "" -} - -func (n *validatorProposalNotification) GetEmailAttachment() *types.EmailAttachment { - return nil -} - -func (n *validatorProposalNotification) GetSubscriptionID() uint64 { - return n.SubscriptionID -} - -func (n *validatorProposalNotification) GetEpoch() uint64 { - return n.Epoch -} + types.NotificationBaseImpl -func (n *validatorProposalNotification) GetEventName() types.EventName { - return n.EventName + ValidatorIndex uint64 + Slot uint64 + Status uint64 // * Can be 0 = scheduled, 1 executed, 2 missed */ + Reward float64 } func (n *validatorProposalNotification) GetInfo(includeUrl bool) string { @@ -1409,10 +1375,6 @@ func (n *validatorProposalNotification) GetTitle() string { return "-" } -func (n *validatorProposalNotification) GetEventFilter() string { - return n.EventFilter -} - func (n *validatorProposalNotification) GetInfoMarkdown() string { var generalPart = "" switch n.Status { @@ -2318,26 +2280,7 @@ func (n *monitorMachineNotification) GetInfoMarkdown() string { } type taxReportNotification struct { - SubscriptionID uint64 - UserID types.UserId - Epoch uint64 - EventFilter string - UnsubscribeHash sql.NullString -} - -func (n *taxReportNotification) GetUserId() types.UserId { - return n.UserID -} - -func (n *taxReportNotification) GetLatestState() string { - return "" -} - -func (n *taxReportNotification) GetUnsubscribeHash() string { - if n.UnsubscribeHash.Valid { - return n.UnsubscribeHash.String - } - return "" + types.NotificationBaseImpl } func (n *taxReportNotification) GetEmailAttachment() *types.EmailAttachment { @@ -2374,14 +2317,6 @@ func (n *taxReportNotification) GetEmailAttachment() *types.EmailAttachment { return &types.EmailAttachment{Attachment: pdf, Name: fmt.Sprintf("income_history_%v_%v.pdf", firstDay.Format("20060102"), lastDay.Format("20060102"))} } -func (n *taxReportNotification) GetSubscriptionID() uint64 { - return n.SubscriptionID -} - -func (n *taxReportNotification) GetEpoch() uint64 { - return n.Epoch -} - func (n *taxReportNotification) GetEventName() types.EventName { return types.TaxReportEventName } @@ -2442,11 +2377,13 @@ func collectTaxReportNotificationNotifications(notificationsByUserID types.Notif for _, r := range dbResult { n := &taxReportNotification{ - SubscriptionID: r.SubscriptionID, - UserID: r.UserID, - Epoch: r.Epoch, - EventFilter: r.EventFilter, - UnsubscribeHash: r.UnsubscribeHash, + NotificationBaseImpl: types.NotificationBaseImpl{ + SubscriptionID: r.SubscriptionID, + UserID: r.UserID, + Epoch: r.Epoch, + EventFilter: r.EventFilter, + UnsubscribeHash: r.UnsubscribeHash, + }, } notificationsByUserID.AddNotification(n) metrics.NotificationsCollected.WithLabelValues(string(n.GetEventName())).Inc() @@ -2456,38 +2393,7 @@ func collectTaxReportNotificationNotifications(notificationsByUserID types.Notif } type networkNotification struct { - SubscriptionID uint64 - UserID types.UserId - Epoch uint64 - EventFilter string - UnsubscribeHash sql.NullString -} - -func (n *networkNotification) GetUserId() types.UserId { - return n.UserID -} - -func (n *networkNotification) GetLatestState() string { - return "" -} - -func (n *networkNotification) GetUnsubscribeHash() string { - if n.UnsubscribeHash.Valid { - return n.UnsubscribeHash.String - } - return "" -} - -func (n *networkNotification) GetEmailAttachment() *types.EmailAttachment { - return nil -} - -func (n *networkNotification) GetSubscriptionID() uint64 { - return n.SubscriptionID -} - -func (n *networkNotification) GetEpoch() uint64 { - return n.Epoch + types.NotificationBaseImpl } func (n *networkNotification) GetEventName() types.EventName { @@ -2503,10 +2409,6 @@ func (n *networkNotification) GetTitle() string { return "Beaconchain Network Issues" } -func (n *networkNotification) GetEventFilter() string { - return n.EventFilter -} - func (n *networkNotification) GetInfoMarkdown() string { generalPart := fmt.Sprintf(`Network experienced finality issues ([view chart](https://%v/charts/network_liveness)).`, utils.Config.Frontend.SiteDomain) return generalPart @@ -2544,11 +2446,13 @@ func collectNetworkNotifications(notificationsByUserID types.NotificationsPerUse for _, r := range dbResult { n := &networkNotification{ - SubscriptionID: r.SubscriptionID, - UserID: r.UserID, - Epoch: r.Epoch, - EventFilter: r.EventFilter, - UnsubscribeHash: r.UnsubscribeHash, + NotificationBaseImpl: types.NotificationBaseImpl{ + SubscriptionID: r.SubscriptionID, + UserID: r.UserID, + Epoch: r.Epoch, + EventFilter: r.EventFilter, + UnsubscribeHash: r.UnsubscribeHash, + }, } notificationsByUserID.AddNotification(n) @@ -2560,44 +2464,8 @@ func collectNetworkNotifications(notificationsByUserID types.NotificationsPerUse } type rocketpoolNotification struct { - SubscriptionID uint64 - UserID types.UserId - Epoch uint64 - EventFilter string - EventName types.EventName - ExtraData string - UnsubscribeHash sql.NullString -} - -func (n *rocketpoolNotification) GetUserId() types.UserId { - return n.UserID -} - -func (n *rocketpoolNotification) GetLatestState() string { - return "" -} - -func (n *rocketpoolNotification) GetUnsubscribeHash() string { - if n.UnsubscribeHash.Valid { - return n.UnsubscribeHash.String - } - return "" -} - -func (n *rocketpoolNotification) GetEmailAttachment() *types.EmailAttachment { - return nil -} - -func (n *rocketpoolNotification) GetSubscriptionID() uint64 { - return n.SubscriptionID -} - -func (n *rocketpoolNotification) GetEpoch() uint64 { - return n.Epoch -} - -func (n *rocketpoolNotification) GetEventName() types.EventName { - return n.EventName + types.NotificationBaseImpl + ExtraData string } func (n *rocketpoolNotification) GetInfo(includeUrl bool) string { @@ -2635,10 +2503,6 @@ func (n *rocketpoolNotification) GetTitle() string { return "" } -func (n *rocketpoolNotification) GetEventFilter() string { - return n.EventFilter -} - func (n *rocketpoolNotification) GetInfoMarkdown() string { return n.GetInfo(false) } @@ -2675,13 +2539,15 @@ func collectRocketpoolComissionNotifications(notificationsByUserID types.Notific for _, r := range dbResult { n := &rocketpoolNotification{ - SubscriptionID: r.SubscriptionID, - UserID: r.UserID, - Epoch: r.Epoch, - EventFilter: r.EventFilter, - EventName: eventName, - ExtraData: strconv.FormatInt(int64(fee*100), 10) + "%", - UnsubscribeHash: r.UnsubscribeHash, + NotificationBaseImpl: types.NotificationBaseImpl{ + SubscriptionID: r.SubscriptionID, + UserID: r.UserID, + Epoch: r.Epoch, + EventFilter: r.EventFilter, + EventName: eventName, + UnsubscribeHash: r.UnsubscribeHash, + }, + ExtraData: strconv.FormatInt(int64(fee*100), 10) + "%", } notificationsByUserID.AddNotification(n) @@ -2724,12 +2590,14 @@ func collectRocketpoolRewardClaimRoundNotifications(notificationsByUserID types. for _, r := range dbResult { n := &rocketpoolNotification{ - SubscriptionID: r.SubscriptionID, - UserID: r.UserID, - Epoch: r.Epoch, - EventFilter: r.EventFilter, - EventName: eventName, - UnsubscribeHash: r.UnsubscribeHash, + NotificationBaseImpl: types.NotificationBaseImpl{ + SubscriptionID: r.SubscriptionID, + UserID: r.UserID, + Epoch: r.Epoch, + EventFilter: r.EventFilter, + EventName: eventName, + UnsubscribeHash: r.UnsubscribeHash, + }, } notificationsByUserID.AddNotification(n) @@ -2856,13 +2724,15 @@ func collectRocketpoolRPLCollateralNotifications(notificationsByUserID types.Not } n := &rocketpoolNotification{ - SubscriptionID: *sub.ID, - UserID: *sub.UserID, - Epoch: epoch, - EventFilter: sub.EventFilter, - EventName: eventName, - ExtraData: strings.TrimRight(strings.TrimRight(fmt.Sprintf("%.2f", threshold*100), "0"), "."), - UnsubscribeHash: sub.UnsubscribeHash, + NotificationBaseImpl: types.NotificationBaseImpl{ + SubscriptionID: *sub.ID, + UserID: *sub.UserID, + Epoch: epoch, + EventFilter: sub.EventFilter, + EventName: eventName, + UnsubscribeHash: sub.UnsubscribeHash, + }, + ExtraData: strings.TrimRight(strings.TrimRight(fmt.Sprintf("%.2f", threshold*100), "0"), "."), } notificationsByUserID.AddNotification(n) @@ -2960,13 +2830,15 @@ func collectSyncCommittee(notificationsByUserID types.NotificationsPerUserId, ev for _, r := range dbResult { n := &rocketpoolNotification{ - SubscriptionID: r.SubscriptionID, - UserID: r.UserID, - Epoch: epoch, - EventFilter: r.EventFilter, - EventName: eventName, - ExtraData: fmt.Sprintf("%v|%v|%v", mapping[r.EventFilter], nextPeriod*utils.Config.Chain.ClConfig.EpochsPerSyncCommitteePeriod, (nextPeriod+1)*utils.Config.Chain.ClConfig.EpochsPerSyncCommitteePeriod), - UnsubscribeHash: r.UnsubscribeHash, + NotificationBaseImpl: types.NotificationBaseImpl{ + SubscriptionID: r.SubscriptionID, + UserID: r.UserID, + Epoch: epoch, + EventFilter: r.EventFilter, + EventName: eventName, + UnsubscribeHash: r.UnsubscribeHash, + }, + ExtraData: fmt.Sprintf("%v|%v|%v", mapping[r.EventFilter], nextPeriod*utils.Config.Chain.ClConfig.EpochsPerSyncCommitteePeriod, (nextPeriod+1)*utils.Config.Chain.ClConfig.EpochsPerSyncCommitteePeriod), } notificationsByUserID.AddNotification(n) metrics.NotificationsCollected.WithLabelValues(string(n.GetEventName())).Inc() From bf095cc9ed932e12c53de3e68636723b60e4878f Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Mon, 8 Jul 2024 10:21:21 +0200 Subject: [PATCH 08/94] refractor GetSubsForEventFilter --- backend/pkg/notification/db.go | 51 ++- backend/pkg/notification/notifications.go | 382 +++++++++++----------- 2 files changed, 235 insertions(+), 198 deletions(-) diff --git a/backend/pkg/notification/db.go b/backend/pkg/notification/db.go index 23f890ba6..36810d6b3 100644 --- a/backend/pkg/notification/db.go +++ b/backend/pkg/notification/db.go @@ -1,6 +1,7 @@ package notification import ( + "github.com/doug-martin/goqu/v9" "github.com/gobitfly/beaconchain/pkg/commons/db" "github.com/gobitfly/beaconchain/pkg/commons/types" "github.com/gobitfly/beaconchain/pkg/commons/utils" @@ -11,14 +12,54 @@ import ( // Map key corresponds to the event filter which can be // a validator pubkey or an eth1 address (for RPL notifications) // or a list of validators for the tax report notifications -func GetSubsForEventFilter(eventName types.EventName) (map[string][]types.Subscription, error) { +// optionally it is possible to set a filter on the last sent ts and the event filter +// fields +func GetSubsForEventFilter(eventName types.EventName, lastSentFilter string, lastSentFilterArgs []interface{}, eventFilters []string) (map[string][]types.Subscription, error) { var subs []types.Subscription - subQuery := ` - SELECT id, user_id, event_filter, last_sent_epoch, created_epoch, event_threshold, ENCODE(unsubscribe_hash, 'hex') as unsubscribe_hash, internal_state from users_subscriptions where event_name = $1 - ` + + // subQuery := ` + // SELECT + // id, + // user_id, + // event_filter, + // last_sent_epoch, + // created_epoch, + // event_threshold, + // ENCODE(unsubscribe_hash, 'hex') as unsubscribe_hash, + // internal_state + // from users_subscriptions + // where event_name = $1 + // ` + + ds := goqu.Dialect("postgres").From("users_subscriptions").Select( + goqu.C("id"), + goqu.C("user_id"), + goqu.C("event_filter"), + goqu.C("last_sent_epoch"), + goqu.C("created_epoch"), + goqu.C("event_threshold"), + goqu.L("ENCODE(unsubscribe_hash, 'hex') as unsubscribe_hash"), + goqu.C("internal_state"), + ).Where(goqu.C("event_name").Eq(utils.GetNetwork() + ":" + string(eventName))) + + if lastSentFilter != "" { + if len(lastSentFilterArgs) > 0 { + ds = ds.Where(goqu.L(lastSentFilter, lastSentFilterArgs...)) + } else { + ds = ds.Where(goqu.L(lastSentFilter)) + } + } + if len(eventFilters) > 0 { + ds = ds.Where(goqu.L("event_filter = ANY(?)", pq.StringArray(eventFilters))) + } + + query, args, err := ds.Prepared(true).ToSQL() + if err != nil { + return nil, err + } subMap := make(map[string][]types.Subscription, 0) - err := db.FrontendWriterDB.Select(&subs, subQuery, utils.GetNetwork()+":"+string(eventName)) + err = db.FrontendWriterDB.Select(&subs, query, args) if err != nil { return nil, err } diff --git a/backend/pkg/notification/notifications.go b/backend/pkg/notification/notifications.go index 3a2dc4ce8..0f1a71b3e 100644 --- a/backend/pkg/notification/notifications.go +++ b/backend/pkg/notification/notifications.go @@ -1241,7 +1241,7 @@ func collectBlockProposalNotifications(notificationsByUserID types.Notifications ExecRewardETH float64 } - subMap, err := GetSubsForEventFilter(eventName) + subMap, err := GetSubsForEventFilter(eventName, "", nil, nil) if err != nil { return fmt.Errorf("error getting subscriptions for (missed) block proposals %w", err) } @@ -1390,7 +1390,7 @@ func (n *validatorProposalNotification) GetInfoMarkdown() string { } func collectAttestationAndOfflineValidatorNotifications(notificationsByUserID types.NotificationsPerUserId, epoch uint64) error { - subMap, err := GetSubsForEventFilter(types.ValidatorMissedAttestationEventName) + subMap, err := GetSubsForEventFilter(types.ValidatorMissedAttestationEventName, "", nil, nil) if err != nil { return fmt.Errorf("error getting subscriptions for missted attestations %w", err) } @@ -1557,7 +1557,7 @@ func collectAttestationAndOfflineValidatorNotifications(notificationsByUserID ty return fmt.Errorf("retrieved more than %v online validators notifications: %v, exiting", onlineValidatorsLimit, len(onlineValidators)) } - subMap, err = GetSubsForEventFilter(types.ValidatorIsOfflineEventName) + subMap, err = GetSubsForEventFilter(types.ValidatorIsOfflineEventName, "", nil, nil) if err != nil { return fmt.Errorf("failed to get subs for %v: %v", types.ValidatorIsOfflineEventName, err) } @@ -1772,6 +1772,7 @@ func collectValidatorGotSlashedNotifications(notificationsByUserID types.Notific query := "" resultsLen := len(dbResult) for i, event := range dbResult { + // TODO: clarify why we need the id here?! query += fmt.Sprintf(`SELECT %d AS ref, id, user_id, ENCODE(unsubscribe_hash, 'hex') AS unsubscribe_hash from users_subscriptions where event_name = $1 AND event_filter = '%x'`, i, event.SlashedValidatorPubkey) if i < resultsLen-1 { query += " UNION " @@ -1856,7 +1857,7 @@ func (n *validatorWithdrawalNotification) GetInfoMarkdown() string { // collectWithdrawalNotifications collects all notifications validator withdrawals func collectWithdrawalNotifications(notificationsByUserID types.NotificationsPerUserId, epoch uint64) error { // get all users that are subscribed to this event (scale: a few thousand rows depending on how many users we have) - subMap, err := GetSubsForEventFilter(types.ValidatorReceivedWithdrawalEventName) + subMap, err := GetSubsForEventFilter(types.ValidatorReceivedWithdrawalEventName, "", nil, nil) if err != nil { return fmt.Errorf("error getting subscriptions for missed attestations %w", err) } @@ -1987,43 +1988,43 @@ func (n *ethClientNotification) GetInfoMarkdown() string { func collectEthClientNotifications(notificationsByUserID types.NotificationsPerUserId, eventName types.EventName) error { updatedClients := ethclients.GetUpdatedClients() //only check if there are new updates for _, client := range updatedClients { - var dbResult []struct { - SubscriptionID uint64 `db:"id"` - UserID types.UserId `db:"user_id"` - Epoch uint64 `db:"created_epoch"` - EventFilter string `db:"event_filter"` - UnsubscribeHash sql.NullString `db:"unsubscribe_hash"` - } - - err := db.FrontendWriterDB.Select(&dbResult, ` - SELECT us.id, us.user_id, us.created_epoch, us.event_filter, ENCODE(us.unsubscribe_hash, 'hex') AS unsubscribe_hash - FROM users_subscriptions AS us - WHERE - us.event_name=$1 - AND - us.event_filter=$2 - AND - ((us.last_sent_ts <= NOW() - INTERVAL '2 DAY' AND TO_TIMESTAMP($3) > us.last_sent_ts) OR us.last_sent_ts IS NULL) - `, - eventName, strings.ToLower(client.Name), client.Date.Unix()) // was last notification sent 2 days ago for this client + // err := db.FrontendWriterDB.Select(&dbResult, ` + // SELECT us.id, us.user_id, us.created_epoch, us.event_filter, ENCODE(us.unsubscribe_hash, 'hex') AS unsubscribe_hash + // FROM users_subscriptions AS us + // WHERE + // us.event_name=$1 + // AND + // us.event_filter=$2 + // AND + // ((us.last_sent_ts <= NOW() - INTERVAL '2 DAY' AND TO_TIMESTAMP($3) > us.last_sent_ts) OR us.last_sent_ts IS NULL) + // `, + // eventName, strings.ToLower(client.Name), client.Date.Unix()) // was last notification sent 2 days ago for this client + + dbResult, err := GetSubsForEventFilter( + eventName, + "(us.last_sent_ts <= NOW() - INTERVAL '2 DAY' AND TO_TIMESTAMP(?) > us.last_sent_ts) OR us.last_sent_ts IS NULL", + []interface{}{client.Date.Unix()}, + []string{strings.ToLower(client.Name)}) if err != nil { return err } - for _, r := range dbResult { - n := ðClientNotification{ - NotificationBaseImpl: types.NotificationBaseImpl{ - SubscriptionID: r.SubscriptionID, - UserID: r.UserID, - Epoch: r.Epoch, - EventFilter: r.EventFilter, - UnsubscribeHash: r.UnsubscribeHash, - }, - EthClient: client.Name, + for _, subs := range dbResult { + for _, sub := range subs { + n := ðClientNotification{ + NotificationBaseImpl: types.NotificationBaseImpl{ + SubscriptionID: *sub.ID, + UserID: *sub.UserID, + Epoch: sub.CreatedEpoch, + EventFilter: sub.EventFilter, + UnsubscribeHash: sub.UnsubscribeHash, + }, + EthClient: client.Name, + } + notificationsByUserID.AddNotification(n) + metrics.NotificationsCollected.WithLabelValues(string(n.GetEventName())).Inc() } - notificationsByUserID.AddNotification(n) - metrics.NotificationsCollected.WithLabelValues(string(n.GetEventName())).Inc() } } return nil @@ -2041,7 +2042,7 @@ func collectMonitoringMachineOffline(notificationsByUserID types.NotificationsPe nowTs := time.Now().Unix() return collectMonitoringMachine(notificationsByUserID, types.MonitoringMachineOfflineEventName, 120, // notify condition - func(_ *MachineEvents, machineData *types.MachineMetricSystemUser) bool { + func(subscribeData *types.Subscription, machineData *types.MachineMetricSystemUser) bool { if machineData.CurrentDataInsertTs < nowTs-10*60 && machineData.CurrentDataInsertTs > nowTs-90*60 { return true } @@ -2059,7 +2060,7 @@ func isMachineDataRecent(machineData *types.MachineMetricSystemUser) bool { func collectMonitoringMachineDiskAlmostFull(notificationsByUserID types.NotificationsPerUserId, epoch uint64) error { return collectMonitoringMachine(notificationsByUserID, types.MonitoringMachineDiskAlmostFullEventName, 750, // notify condition - func(subscribeData *MachineEvents, machineData *types.MachineMetricSystemUser) bool { + func(subscribeData *types.Subscription, machineData *types.MachineMetricSystemUser) bool { if !isMachineDataRecent(machineData) { return false } @@ -2074,7 +2075,7 @@ func collectMonitoringMachineDiskAlmostFull(notificationsByUserID types.Notifica func collectMonitoringMachineCPULoad(notificationsByUserID types.NotificationsPerUserId, epoch uint64) error { return collectMonitoringMachine(notificationsByUserID, types.MonitoringMachineCpuLoadEventName, 10, // notify condition - func(subscribeData *MachineEvents, machineData *types.MachineMetricSystemUser) bool { + func(subscribeData *types.Subscription, machineData *types.MachineMetricSystemUser) bool { if !isMachineDataRecent(machineData) { return false } @@ -2096,7 +2097,7 @@ func collectMonitoringMachineCPULoad(notificationsByUserID types.NotificationsPe func collectMonitoringMachineMemoryUsage(notificationsByUserID types.NotificationsPerUserId, epoch uint64) error { return collectMonitoringMachine(notificationsByUserID, types.MonitoringMachineMemoryUsageEventName, 10, // notify condition - func(subscribeData *MachineEvents, machineData *types.MachineMetricSystemUser) bool { + func(subscribeData *types.Subscription, machineData *types.MachineMetricSystemUser) bool { if !isMachineDataRecent(machineData) { return false } @@ -2117,21 +2118,23 @@ func collectMonitoringMachine( notificationsByUserID types.NotificationsPerUserId, eventName types.EventName, epochWaitInBetween int, - notifyConditionFulfilled func(subscribeData *MachineEvents, machineData *types.MachineMetricSystemUser) bool, + notifyConditionFulfilled func(subscribeData *types.Subscription, machineData *types.MachineMetricSystemUser) bool, epoch uint64, ) error { - var allSubscribed []MachineEvents + var allSubscribed []*types.Subscription + // event_filter == machine name + // TODO: clarify why we need grouping here?! err := db.FrontendWriterDB.Select(&allSubscribed, `SELECT us.user_id, max(us.id) AS id, ENCODE((array_agg(us.unsubscribe_hash))[1], 'hex') AS unsubscribe_hash, - event_filter AS machine, + event_filter, COALESCE(event_threshold, 0) AS event_threshold FROM users_subscriptions us WHERE us.event_name = $1 AND us.created_epoch <= $2 AND (us.last_sent_epoch < ($2 - $3) OR us.last_sent_epoch IS NULL) - group by us.user_id, machine, event_threshold`, + group by us.user_id, event_filter, event_threshold`, eventName, epoch, epochWaitInBetween) if err != nil { return err @@ -2139,7 +2142,7 @@ func collectMonitoringMachine( rowKeys := gcp_bigtable.RowList{} for _, data := range allSubscribed { - rowKeys = append(rowKeys, db.BigtableClient.GetMachineRowKey(data.UserID, "system", data.MachineName)) + rowKeys = append(rowKeys, db.BigtableClient.GetMachineRowKey(*data.UserID, "system", data.EventFilter)) } machineDataOfSubscribed, err := db.BigtableClient.GetMachineMetricsForNotifications(rowKeys) @@ -2147,20 +2150,20 @@ func collectMonitoringMachine( return err } - var result []MachineEvents + var result []*types.Subscription for _, data := range allSubscribed { localData := data // Create a local copy of the data variable - machineMap, found := machineDataOfSubscribed[localData.UserID] + machineMap, found := machineDataOfSubscribed[*localData.UserID] if !found { continue } - currentMachineData, found := machineMap[localData.MachineName] + currentMachineData, found := machineMap[localData.EventFilter] if !found { continue } //logrus.Infof("currentMachineData %v | %v | %v | %v", currentMachine.CurrentDataInsertTs, currentMachine.CompareDataInsertTs, currentMachine.UserID, currentMachine.Machine) - if notifyConditionFulfilled(&localData, currentMachineData) { + if notifyConditionFulfilled(localData, currentMachineData) { result = append(result, localData) } } @@ -2208,13 +2211,13 @@ func collectMonitoringMachine( for _, r := range result { n := &monitorMachineNotification{ NotificationBaseImpl: types.NotificationBaseImpl{ - SubscriptionID: r.SubscriptionID, - UserID: r.UserID, + SubscriptionID: *r.ID, + UserID: *r.UserID, EventName: eventName, Epoch: epoch, UnsubscribeHash: r.UnsubscribeHash, }, - MachineName: r.MachineName, + MachineName: r.EventFilter, } //logrus.Infof("notify %v %v", eventName, n) notificationsByUserID.AddNotification(n) @@ -2351,42 +2354,37 @@ func collectTaxReportNotificationNotifications(notificationsByUserID types.Notif return nil } - var dbResult []struct { - SubscriptionID uint64 `db:"id"` - UserID types.UserId `db:"user_id"` - Epoch uint64 `db:"created_epoch"` - EventFilter string `db:"event_filter"` - UnsubscribeHash sql.NullString `db:"unsubscribe_hash"` - } - - name := string(eventName) - if utils.Config.Chain.ClConfig.ConfigName != "" { - name = utils.Config.Chain.ClConfig.ConfigName + ":" + name - } - - err = db.FrontendWriterDB.Select(&dbResult, ` - SELECT us.id, us.user_id, us.created_epoch, us.event_filter, ENCODE(us.unsubscribe_hash, 'hex') AS unsubscribe_hash - FROM users_subscriptions AS us - WHERE us.event_name=$1 AND (us.last_sent_ts < $2 OR (us.last_sent_ts IS NULL AND us.created_ts < $2)); - `, - name, firstDayOfMonth) + // err = db.FrontendWriterDB.Select(&dbResult, ` + // SELECT us.id, us.user_id, us.created_epoch, us.event_filter, ENCODE(us.unsubscribe_hash, 'hex') AS unsubscribe_hash + // FROM users_subscriptions AS us + // WHERE us.event_name=$1 AND (us.last_sent_ts < $2 OR (us.last_sent_ts IS NULL AND us.created_ts < $2)); + // `, + // name, firstDayOfMonth) + dbResults, err := GetSubsForEventFilter( + types.TaxReportEventName, + "us.last_sent_ts < ? OR (us.last_sent_ts IS NULL AND us.created_ts < ?)", + []interface{}{firstDayOfMonth, firstDayOfMonth}, + nil, + ) if err != nil { return err } - for _, r := range dbResult { - n := &taxReportNotification{ - NotificationBaseImpl: types.NotificationBaseImpl{ - SubscriptionID: r.SubscriptionID, - UserID: r.UserID, - Epoch: r.Epoch, - EventFilter: r.EventFilter, - UnsubscribeHash: r.UnsubscribeHash, - }, + for _, subs := range dbResults { + for _, sub := range subs { + n := &taxReportNotification{ + NotificationBaseImpl: types.NotificationBaseImpl{ + SubscriptionID: *sub.ID, + UserID: *sub.UserID, + Epoch: sub.CreatedEpoch, + EventFilter: sub.EventFilter, + UnsubscribeHash: sub.UnsubscribeHash, + }, + } + notificationsByUserID.AddNotification(n) + metrics.NotificationsCollected.WithLabelValues(string(n.GetEventName())).Inc() } - notificationsByUserID.AddNotification(n) - metrics.NotificationsCollected.WithLabelValues(string(n.GetEventName())).Inc() } return nil @@ -2425,38 +2423,38 @@ func collectNetworkNotifications(notificationsByUserID types.NotificationsPerUse } if count > 0 { - var dbResult []struct { - SubscriptionID uint64 `db:"id"` - UserID types.UserId `db:"user_id"` - Epoch uint64 `db:"created_epoch"` - EventFilter string `db:"event_filter"` - UnsubscribeHash sql.NullString `db:"unsubscribe_hash"` - } - - err := db.FrontendWriterDB.Select(&dbResult, ` - SELECT us.id, us.user_id, us.created_epoch, us.event_filter, ENCODE(us.unsubscribe_hash, 'hex') AS unsubscribe_hash - FROM users_subscriptions AS us - WHERE us.event_name=$1 AND (us.last_sent_ts <= NOW() - INTERVAL '1 hour' OR us.last_sent_ts IS NULL); - `, - utils.GetNetwork()+":"+string(eventName)) - + // err := db.FrontendWriterDB.Select(&dbResult, ` + // SELECT us.id, us.user_id, us.created_epoch, us.event_filter, ENCODE(us.unsubscribe_hash, 'hex') AS unsubscribe_hash + // FROM users_subscriptions AS us + // WHERE us.event_name=$1 AND (us.last_sent_ts <= NOW() - INTERVAL '1 hour' OR us.last_sent_ts IS NULL); + // `, + // utils.GetNetwork()+":"+string(eventName)) + + dbResult, err := GetSubsForEventFilter( + eventName, + "us.last_sent_ts <= NOW() - INTERVAL '1 hour' OR us.last_sent_ts IS NULL", + nil, + nil, + ) if err != nil { return err } - for _, r := range dbResult { - n := &networkNotification{ - NotificationBaseImpl: types.NotificationBaseImpl{ - SubscriptionID: r.SubscriptionID, - UserID: r.UserID, - Epoch: r.Epoch, - EventFilter: r.EventFilter, - UnsubscribeHash: r.UnsubscribeHash, - }, - } + for _, subs := range dbResult { + for _, sub := range subs { + n := &networkNotification{ + NotificationBaseImpl: types.NotificationBaseImpl{ + SubscriptionID: *sub.ID, + UserID: *sub.UserID, + Epoch: sub.CreatedEpoch, + EventFilter: sub.EventFilter, + UnsubscribeHash: sub.UnsubscribeHash, + }, + } - notificationsByUserID.AddNotification(n) - metrics.NotificationsCollected.WithLabelValues(string(n.GetEventName())).Inc() + notificationsByUserID.AddNotification(n) + metrics.NotificationsCollected.WithLabelValues(string(n.GetEventName())).Inc() + } } } @@ -2518,40 +2516,40 @@ func collectRocketpoolComissionNotifications(notificationsByUserID types.Notific } if fee > 0 { - var dbResult []struct { - SubscriptionID uint64 `db:"id"` - UserID types.UserId `db:"user_id"` - Epoch uint64 `db:"created_epoch"` - EventFilter string `db:"event_filter"` - UnsubscribeHash sql.NullString `db:"unsubscribe_hash"` - } - - err := db.FrontendWriterDB.Select(&dbResult, ` - SELECT us.id, us.user_id, us.created_epoch, us.event_filter, ENCODE(us.unsubscribe_hash, 'hex') AS unsubscribe_hash - FROM users_subscriptions AS us - WHERE us.event_name=$1 AND (us.last_sent_ts <= NOW() - INTERVAL '8 hours' OR us.last_sent_ts IS NULL) AND (us.event_threshold <= $2 OR (us.event_threshold < 0 AND us.event_threshold * -1 >= $2)); - `, - utils.GetNetwork()+":"+string(eventName), fee) - + // err := db.FrontendWriterDB.Select(&dbResult, ` + // SELECT us.id, us.user_id, us.created_epoch, us.event_filter, ENCODE(us.unsubscribe_hash, 'hex') AS unsubscribe_hash + // FROM users_subscriptions AS us + // WHERE us.event_name=$1 AND (us.last_sent_ts <= NOW() - INTERVAL '8 hours' OR us.last_sent_ts IS NULL) AND (us.event_threshold <= $2 OR (us.event_threshold < 0 AND us.event_threshold * -1 >= $2)); + // `, + // utils.GetNetwork()+":"+string(eventName), fee) + + dbResult, err := GetSubsForEventFilter( + eventName, + "(us.last_sent_ts <= NOW() - INTERVAL '8 hours' OR us.last_sent_ts IS NULL) AND (us.event_threshold <= ? OR (us.event_threshold < 0 AND us.event_threshold * -1 >= ?)", + []interface{}{fee, fee}, + nil, + ) if err != nil { return err } - for _, r := range dbResult { - n := &rocketpoolNotification{ - NotificationBaseImpl: types.NotificationBaseImpl{ - SubscriptionID: r.SubscriptionID, - UserID: r.UserID, - Epoch: r.Epoch, - EventFilter: r.EventFilter, - EventName: eventName, - UnsubscribeHash: r.UnsubscribeHash, - }, - ExtraData: strconv.FormatInt(int64(fee*100), 10) + "%", - } + for _, subs := range dbResult { + for _, sub := range subs { + n := &rocketpoolNotification{ + NotificationBaseImpl: types.NotificationBaseImpl{ + SubscriptionID: *sub.ID, + UserID: *sub.UserID, + Epoch: sub.CreatedEpoch, + EventFilter: sub.EventFilter, + EventName: eventName, + UnsubscribeHash: sub.UnsubscribeHash, + }, + ExtraData: strconv.FormatInt(int64(fee*100), 10) + "%", + } - notificationsByUserID.AddNotification(n) - metrics.NotificationsCollected.WithLabelValues(string(n.GetEventName())).Inc() + notificationsByUserID.AddNotification(n) + metrics.NotificationsCollected.WithLabelValues(string(n.GetEventName())).Inc() + } } } @@ -2569,39 +2567,41 @@ func collectRocketpoolRewardClaimRoundNotifications(notificationsByUserID types. } if ts+3*60*60 > time.Now().Unix() { - var dbResult []struct { - SubscriptionID uint64 `db:"id"` - UserID types.UserId `db:"user_id"` - Epoch uint64 `db:"created_epoch"` - EventFilter string `db:"event_filter"` - UnsubscribeHash sql.NullString `db:"unsubscribe_hash"` - } - - err := db.FrontendWriterDB.Select(&dbResult, ` - SELECT us.id, us.user_id, us.created_epoch, us.event_filter, ENCODE(us.unsubscribe_hash, 'hex') AS unsubscribe_hash - FROM users_subscriptions AS us - WHERE us.event_name=$1 AND (us.last_sent_ts <= NOW() - INTERVAL '5 hours' OR us.last_sent_ts IS NULL); - `, - utils.GetNetwork()+":"+string(eventName)) - + // var dbResult []*types.Subscription + + // err := db.FrontendWriterDB.Select(&dbResult, ` + // SELECT us.id, us.user_id, us.created_epoch, us.event_filter, ENCODE(us.unsubscribe_hash, 'hex') AS unsubscribe_hash + // FROM users_subscriptions AS us + // WHERE us.event_name=$1 AND (us.last_sent_ts <= NOW() - INTERVAL '5 hours' OR us.last_sent_ts IS NULL); + // `, + // utils.GetNetwork()+":"+string(eventName)) + + dbResult, err := GetSubsForEventFilter( + eventName, + "us.last_sent_ts <= NOW() - INTERVAL '5 hours' OR us.last_sent_ts IS NULL", + nil, + nil, + ) if err != nil { return err } - for _, r := range dbResult { - n := &rocketpoolNotification{ - NotificationBaseImpl: types.NotificationBaseImpl{ - SubscriptionID: r.SubscriptionID, - UserID: r.UserID, - Epoch: r.Epoch, - EventFilter: r.EventFilter, - EventName: eventName, - UnsubscribeHash: r.UnsubscribeHash, - }, - } + for _, subs := range dbResult { + for _, sub := range subs { + n := &rocketpoolNotification{ + NotificationBaseImpl: types.NotificationBaseImpl{ + SubscriptionID: *sub.ID, + UserID: *sub.UserID, + Epoch: sub.CreatedEpoch, + EventFilter: sub.EventFilter, + EventName: eventName, + UnsubscribeHash: sub.UnsubscribeHash, + }, + } - notificationsByUserID.AddNotification(n) - metrics.NotificationsCollected.WithLabelValues(string(n.GetEventName())).Inc() + notificationsByUserID.AddNotification(n) + metrics.NotificationsCollected.WithLabelValues(string(n.GetEventName())).Inc() + } } } @@ -2609,7 +2609,7 @@ func collectRocketpoolRewardClaimRoundNotifications(notificationsByUserID types. } func collectRocketpoolRPLCollateralNotifications(notificationsByUserID types.NotificationsPerUserId, eventName types.EventName, epoch uint64) error { - subMap, err := GetSubsForEventFilter(eventName) + subMap, err := GetSubsForEventFilter(eventName, "", nil, nil) if err != nil { return fmt.Errorf("error getting subscriptions for RocketpoolRPLCollateral %w", err) } @@ -2809,39 +2809,35 @@ func collectSyncCommittee(notificationsByUserID types.NotificationsPerUserId, ev pubKeys = append(pubKeys, val.PubKey) } - var dbResult []struct { - SubscriptionID uint64 `db:"id"` - UserID types.UserId `db:"user_id"` - EventFilter string `db:"event_filter"` - UnsubscribeHash sql.NullString `db:"unsubscribe_hash"` - } - - err = db.FrontendWriterDB.Select(&dbResult, ` - SELECT us.id, us.user_id, us.event_filter, ENCODE(us.unsubscribe_hash, 'hex') as unsubscribe_hash - FROM users_subscriptions AS us - WHERE us.event_name=$1 AND (us.last_sent_ts <= NOW() - INTERVAL '26 hours' OR us.last_sent_ts IS NULL) AND event_filter = ANY($2); - `, - utils.GetNetwork()+":"+string(eventName), pq.StringArray(pubKeys), - ) + dbResult, err := GetSubsForEventFilter(eventName, "us.last_sent_ts <= NOW() - INTERVAL '26 hours' OR us.last_sent_ts IS NULL", nil, pubKeys) + // err = db.FrontendWriterDB.Select(&dbResult, ` + // SELECT us.id, us.user_id, us.event_filter, ENCODE(us.unsubscribe_hash, 'hex') as unsubscribe_hash + // FROM users_subscriptions AS us + // WHERE us.event_name=$1 AND (us.last_sent_ts <= NOW() - INTERVAL '26 hours' OR us.last_sent_ts IS NULL) AND event_filter = ANY($2); + // `, + // utils.GetNetwork()+":"+string(eventName), pq.StringArray(pubKeys), + // ) if err != nil { return err } - for _, r := range dbResult { - n := &rocketpoolNotification{ - NotificationBaseImpl: types.NotificationBaseImpl{ - SubscriptionID: r.SubscriptionID, - UserID: r.UserID, - Epoch: epoch, - EventFilter: r.EventFilter, - EventName: eventName, - UnsubscribeHash: r.UnsubscribeHash, - }, - ExtraData: fmt.Sprintf("%v|%v|%v", mapping[r.EventFilter], nextPeriod*utils.Config.Chain.ClConfig.EpochsPerSyncCommitteePeriod, (nextPeriod+1)*utils.Config.Chain.ClConfig.EpochsPerSyncCommitteePeriod), + for _, subs := range dbResult { + for _, sub := range subs { + n := &rocketpoolNotification{ + NotificationBaseImpl: types.NotificationBaseImpl{ + SubscriptionID: *sub.ID, + UserID: *sub.UserID, + Epoch: epoch, + EventFilter: sub.EventFilter, + EventName: eventName, + UnsubscribeHash: sub.UnsubscribeHash, + }, + ExtraData: fmt.Sprintf("%v|%v|%v", mapping[sub.EventFilter], nextPeriod*utils.Config.Chain.ClConfig.EpochsPerSyncCommitteePeriod, (nextPeriod+1)*utils.Config.Chain.ClConfig.EpochsPerSyncCommitteePeriod), + } + notificationsByUserID.AddNotification(n) + metrics.NotificationsCollected.WithLabelValues(string(n.GetEventName())).Inc() } - notificationsByUserID.AddNotification(n) - metrics.NotificationsCollected.WithLabelValues(string(n.GetEventName())).Inc() } return nil From f89ce9738c86f21d5b4067c878358308324ef59b Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Mon, 8 Jul 2024 10:29:47 +0200 Subject: [PATCH 09/94] fix linter --- backend/pkg/commons/types/frontend.go | 1 - backend/pkg/notification/notifications.go | 13 ++++++++++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/backend/pkg/commons/types/frontend.go b/backend/pkg/commons/types/frontend.go index a51e5e4c2..f9669b9ec 100644 --- a/backend/pkg/commons/types/frontend.go +++ b/backend/pkg/commons/types/frontend.go @@ -27,7 +27,6 @@ type EventFilter string type NotificationsPerUserId map[UserId]map[EventName]map[EventFilter]Notification func (npui NotificationsPerUserId) AddNotification(n Notification) { - if n.GetUserId() == 0 { log.Fatal(fmt.Errorf("Notification user id is 0"), fmt.Sprintf("Notification: %v", n), 0) } diff --git a/backend/pkg/notification/notifications.go b/backend/pkg/notification/notifications.go index 0f1a71b3e..c81d29e6a 100644 --- a/backend/pkg/notification/notifications.go +++ b/backend/pkg/notification/notifications.go @@ -231,6 +231,7 @@ func collectNotifications(epoch uint64) (types.NotificationsPerUserId, error) { var err error var dbIsCoherent bool + // do a consistency check to make sure that we have all the data we need in the db err = db.WriterDb.Get(&dbIsCoherent, ` SELECT NOT (array[false] && array_agg(is_coherent)) AS is_coherent @@ -263,6 +264,8 @@ func collectNotifications(epoch uint64) (types.NotificationsPerUserId, error) { } log.Infof("retrieving dashboard definitions") + // Retrieve all dashboard definitions to be able to retrieve validators included in + // the group notification subscriptions // TODO: add a filter to retrieve only groups that have notifications enabled // Needs a new field in the db var dashboardDefinitions []dashboardDefinitionRow @@ -305,6 +308,8 @@ func collectNotifications(epoch uint64) (types.NotificationsPerUserId, error) { validatorDashboardConfig.DashboardsByUserId[row.UserId][row.DashboardId].Groups[row.GroupId].Validators = append(validatorDashboardConfig.DashboardsByUserId[row.UserId][row.DashboardId].Groups[row.GroupId].Validators, uint64(row.ValidatorIndex)) } + // TODO: pass the validatorDashboardConfig to the notification collection functions + err = collectAttestationAndOfflineValidatorNotifications(notificationsByUserID, epoch) if err != nil { metrics.Errors.WithLabelValues("notifications_collect_missed_attestation").Inc() @@ -1389,7 +1394,9 @@ func (n *validatorProposalNotification) GetInfoMarkdown() string { return generalPart } +// collectAttestationAndOfflineValidatorNotifications collects notifications for missed attestations and offline validators func collectAttestationAndOfflineValidatorNotifications(notificationsByUserID types.NotificationsPerUserId, epoch uint64) error { + // Retrieve subscriptions for missed attestations subMap, err := GetSubsForEventFilter(types.ValidatorMissedAttestationEventName, "", nil, nil) if err != nil { return fmt.Errorf("error getting subscriptions for missted attestations %w", err) @@ -1403,12 +1410,13 @@ func collectAttestationAndOfflineValidatorNotifications(notificationsByUserID ty } // get attestations for all validators for the last 4 epochs - + // we need 4 epochs so that can detect the online / offline status of validators validators, err := db.GetValidatorIndices() if err != nil { return err } + // this reads the submitted attestations for the last 4 epochs participationPerEpoch, err := db.GetValidatorAttestationHistoryForNotifications(epoch-3, epoch) if err != nil { return fmt.Errorf("error getting validator attestations from db %w", err) @@ -1988,7 +1996,6 @@ func (n *ethClientNotification) GetInfoMarkdown() string { func collectEthClientNotifications(notificationsByUserID types.NotificationsPerUserId, eventName types.EventName) error { updatedClients := ethclients.GetUpdatedClients() //only check if there are new updates for _, client := range updatedClients { - // err := db.FrontendWriterDB.Select(&dbResult, ` // SELECT us.id, us.user_id, us.created_epoch, us.event_filter, ENCODE(us.unsubscribe_hash, 'hex') AS unsubscribe_hash // FROM users_subscriptions AS us @@ -2362,7 +2369,7 @@ func collectTaxReportNotificationNotifications(notificationsByUserID types.Notif // name, firstDayOfMonth) dbResults, err := GetSubsForEventFilter( - types.TaxReportEventName, + eventName, "us.last_sent_ts < ? OR (us.last_sent_ts IS NULL AND us.created_ts < ?)", []interface{}{firstDayOfMonth, firstDayOfMonth}, nil, From 370bff2970a29c62f63903478c89e4dc7e188185 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Mon, 8 Jul 2024 12:18:58 +0200 Subject: [PATCH 10/94] fix bug in AddNotification function --- backend/pkg/commons/types/frontend.go | 4 +- backend/pkg/notification/notifications.go | 96 ++++++++++++----------- 2 files changed, 53 insertions(+), 47 deletions(-) diff --git a/backend/pkg/commons/types/frontend.go b/backend/pkg/commons/types/frontend.go index f9669b9ec..2a4f0d8ca 100644 --- a/backend/pkg/commons/types/frontend.go +++ b/backend/pkg/commons/types/frontend.go @@ -41,7 +41,7 @@ func (npui NotificationsPerUserId) AddNotification(n Notification) { npui[n.GetUserId()] = make(map[EventName]map[EventFilter]Notification) } if _, ok := npui[n.GetUserId()][n.GetEventName()]; !ok { - npui[n.GetUserId()][EventName(n.GetEventFilter())] = make(map[EventFilter]Notification) + npui[n.GetUserId()][n.GetEventName()] = make(map[EventFilter]Notification) } npui[n.GetUserId()][n.GetEventName()][EventFilter(n.GetEventFilter())] = n } @@ -344,7 +344,7 @@ func (n NotificationBaseImpl) GetUserId() UserId { type Subscription struct { ID *uint64 `db:"id,omitempty"` UserID *UserId `db:"user_id,omitempty"` - EventName string `db:"event_name"` + EventName EventName `db:"event_name"` EventFilter string `db:"event_filter"` LastSent *time.Time `db:"last_sent_ts"` LastEpoch *uint64 `db:"last_sent_epoch"` diff --git a/backend/pkg/notification/notifications.go b/backend/pkg/notification/notifications.go index c81d29e6a..e803f8514 100644 --- a/backend/pkg/notification/notifications.go +++ b/backend/pkg/notification/notifications.go @@ -309,7 +309,9 @@ func collectNotifications(epoch uint64) (types.NotificationsPerUserId, error) { } // TODO: pass the validatorDashboardConfig to the notification collection functions - + // The following functions will collect the notifications and add them to the + // notificationsByUserID map. The notifications will be queued and sent later + // by the notification sender process err = collectAttestationAndOfflineValidatorNotifications(notificationsByUserID, epoch) if err != nil { metrics.Errors.WithLabelValues("notifications_collect_missed_attestation").Inc() @@ -352,7 +354,7 @@ func collectNotifications(epoch uint64) (types.NotificationsPerUserId, error) { } log.Infof("collecting withdrawal notifications took: %v", time.Since(start)) - err = collectNetworkNotifications(notificationsByUserID, types.NetworkLivenessIncreasedEventName) + err = collectNetworkNotifications(notificationsByUserID) if err != nil { metrics.Errors.WithLabelValues("notifications_collect_network").Inc() return nil, fmt.Errorf("error collecting network notifications: %v", err) @@ -372,7 +374,7 @@ func collectNotifications(epoch uint64) (types.NotificationsPerUserId, error) { return nil, fmt.Errorf("error collecting rocketpool notifications: %v", err) } } else { - err = collectRocketpoolComissionNotifications(notificationsByUserID, types.RocketpoolCommissionThresholdEventName) + err = collectRocketpoolComissionNotifications(notificationsByUserID) if err != nil { //nolint:misspell metrics.Errors.WithLabelValues("notifications_collect_rocketpool_comission").Inc() @@ -380,7 +382,7 @@ func collectNotifications(epoch uint64) (types.NotificationsPerUserId, error) { } log.Infof("collecting rocketpool commissions took: %v", time.Since(start)) - err = collectRocketpoolRewardClaimRoundNotifications(notificationsByUserID, types.RocketpoolNewClaimRoundStartedEventName) + err = collectRocketpoolRewardClaimRoundNotifications(notificationsByUserID) if err != nil { metrics.Errors.WithLabelValues("notifications_collect_rocketpool_reward_claim").Inc() return nil, fmt.Errorf("error collecting new rocketpool claim round: %v", err) @@ -403,7 +405,7 @@ func collectNotifications(epoch uint64) (types.NotificationsPerUserId, error) { } } - err = collectSyncCommittee(notificationsByUserID, types.SyncCommitteeSoon, epoch) + err = collectSyncCommittee(notificationsByUserID, epoch) if err != nil { metrics.Errors.WithLabelValues("notifications_collect_sync_committee").Inc() return nil, fmt.Errorf("error collecting sync committee: %v", err) @@ -446,14 +448,14 @@ func collectUserDbNotifications(epoch uint64) (types.NotificationsPerUserId, err } // New ETH clients - err = collectEthClientNotifications(notificationsByUserID, types.EthClientUpdateEventName) + err = collectEthClientNotifications(notificationsByUserID) if err != nil { metrics.Errors.WithLabelValues("notifications_collect_eth_client").Inc() return nil, fmt.Errorf("error collecting Eth client notifications: %v", err) } //Tax Report - err = collectTaxReportNotificationNotifications(notificationsByUserID, types.TaxReportEventName) + err = collectTaxReportNotificationNotifications(notificationsByUserID) if err != nil { metrics.Errors.WithLabelValues("notifications_collect_tax_report").Inc() return nil, fmt.Errorf("error collecting tax report notifications: %v", err) @@ -735,6 +737,8 @@ func queueEmailNotifications(notificationsByUserID types.NotificationsPerUserId, notificationTitles = append(notificationTitles, title) } + // TODO: this is bad and will break in case there are a lot of unsubscribe hashes to generate + // the unsubscribe hash should be set when we add the subscription to the db unsubHash := n.GetUnsubscribeHash() if unsubHash == "" { id := n.GetSubscriptionID() @@ -760,10 +764,8 @@ func queueEmailNotifications(notificationsByUserID types.NotificationsPerUserId, `, id) if err != nil { log.Error(err, "error getting user subscription by subscription id", 0) - err = tx.Rollback() - if err != nil { - log.Error(err, "error rolling back transaction", 0) - } + utils.Rollback(tx) + continue } raw := fmt.Sprintf("%v%v%v%v", sub.ID, sub.UserID, sub.EventName, sub.CreatedTime) @@ -772,19 +774,15 @@ func queueEmailNotifications(notificationsByUserID types.NotificationsPerUserId, _, err = tx.Exec("UPDATE users_subscriptions set unsubscribe_hash = $1 WHERE id = $2", digest[:], id) if err != nil { log.Error(err, "error updating users subscriptions table with unsubscribe hash", 0) - err = tx.Rollback() - if err != nil { - log.Error(err, "error rolling back transaction", 0) - } + utils.Rollback(tx) + continue } err = tx.Commit() if err != nil { log.Error(err, "error committing transaction to update users subscriptions with an unsubscribe hash", 0) - err = tx.Rollback() - if err != nil { - log.Error(err, "error rolling back transaction", 0) - } + utils.Rollback(tx) + continue } unsubHash = hex.EncodeToString(digest[:]) @@ -1323,7 +1321,7 @@ func collectBlockProposalNotifications(notificationsByUserID types.Notifications SubscriptionID: *sub.ID, UserID: *sub.UserID, Epoch: epoch, - EventName: eventName, + EventName: sub.EventName, EventFilter: hex.EncodeToString(pubkey), }, ValidatorIndex: event.Proposer, @@ -1478,7 +1476,7 @@ func collectAttestationAndOfflineValidatorNotifications(notificationsByUserID ty SubscriptionID: *sub.ID, UserID: *sub.UserID, Epoch: event.Epoch, - EventName: types.ValidatorMissedAttestationEventName, + EventName: sub.EventName, EventFilter: hex.EncodeToString(event.EventFilter), }, ValidatorIndex: event.ValidatorIndex, @@ -1583,9 +1581,10 @@ func collectAttestationAndOfflineValidatorNotifications(notificationsByUserID ty NotificationBaseImpl: types.NotificationBaseImpl{ SubscriptionID: *sub.ID, Epoch: epoch, - EventName: types.ValidatorIsOfflineEventName, + EventName: sub.EventName, LatestState: fmt.Sprint(epoch - 2), // first epoch the validator stopped attesting EventFilter: hex.EncodeToString(validator.Pubkey), + UserID: *sub.UserID, }, ValidatorIndex: validator.Index, IsOffline: true, @@ -1627,7 +1626,7 @@ func collectAttestationAndOfflineValidatorNotifications(notificationsByUserID ty SubscriptionID: *sub.ID, UserID: *sub.UserID, Epoch: epoch, - EventName: types.ValidatorIsOfflineEventName, + EventName: sub.EventName, EventFilter: hex.EncodeToString(validator.Pubkey), LatestState: "-", }, @@ -1781,7 +1780,7 @@ func collectValidatorGotSlashedNotifications(notificationsByUserID types.Notific resultsLen := len(dbResult) for i, event := range dbResult { // TODO: clarify why we need the id here?! - query += fmt.Sprintf(`SELECT %d AS ref, id, user_id, ENCODE(unsubscribe_hash, 'hex') AS unsubscribe_hash from users_subscriptions where event_name = $1 AND event_filter = '%x'`, i, event.SlashedValidatorPubkey) + query += fmt.Sprintf(`SELECT %d AS ref, id, user_id, ENCODE(unsubscribe_hash, 'hex') AS unsubscribe_hash, event_name from users_subscriptions where event_name = $1 AND event_filter = '%x'`, i, event.SlashedValidatorPubkey) if i < resultsLen-1 { query += " UNION " } @@ -1792,10 +1791,11 @@ func collectValidatorGotSlashedNotifications(notificationsByUserID types.Notific } var subscribers []struct { - Ref uint64 `db:"ref"` - Id uint64 `db:"id"` - UserId types.UserId `db:"user_id"` - UnsubscribeHash sql.NullString `db:"unsubscribe_hash"` + Ref uint64 `db:"ref"` + Id uint64 `db:"id"` + UserId types.UserId `db:"user_id"` + UnsubscribeHash sql.NullString `db:"unsubscribe_hash"` + EventName types.EventName `db:"event_name"` } name := string(types.ValidatorGotSlashedEventName) @@ -1819,6 +1819,7 @@ func collectValidatorGotSlashedNotifications(notificationsByUserID types.Notific Epoch: event.Epoch, EventFilter: hex.EncodeToString(event.SlashedValidatorPubkey), UnsubscribeHash: sub.UnsubscribeHash, + EventName: sub.EventName, }, Slasher: event.SlasherIndex, Reason: event.Reason, @@ -1897,6 +1898,7 @@ func collectWithdrawalNotifications(notificationsByUserID types.NotificationsPer UserID: *sub.UserID, EventFilter: hex.EncodeToString(event.Pubkey), UnsubscribeHash: sub.UnsubscribeHash, + EventName: sub.EventName, }, ValidatorIndex: event.ValidatorIndex, Epoch: epoch, @@ -1993,7 +1995,7 @@ func (n *ethClientNotification) GetInfoMarkdown() string { return generalPart } -func collectEthClientNotifications(notificationsByUserID types.NotificationsPerUserId, eventName types.EventName) error { +func collectEthClientNotifications(notificationsByUserID types.NotificationsPerUserId) error { updatedClients := ethclients.GetUpdatedClients() //only check if there are new updates for _, client := range updatedClients { // err := db.FrontendWriterDB.Select(&dbResult, ` @@ -2009,7 +2011,7 @@ func collectEthClientNotifications(notificationsByUserID types.NotificationsPerU // eventName, strings.ToLower(client.Name), client.Date.Unix()) // was last notification sent 2 days ago for this client dbResult, err := GetSubsForEventFilter( - eventName, + types.EthClientUpdateEventName, "(us.last_sent_ts <= NOW() - INTERVAL '2 DAY' AND TO_TIMESTAMP(?) > us.last_sent_ts) OR us.last_sent_ts IS NULL", []interface{}{client.Date.Unix()}, []string{strings.ToLower(client.Name)}) @@ -2026,6 +2028,7 @@ func collectEthClientNotifications(notificationsByUserID types.NotificationsPerU Epoch: sub.CreatedEpoch, EventFilter: sub.EventFilter, UnsubscribeHash: sub.UnsubscribeHash, + EventName: sub.EventName, }, EthClient: client.Name, } @@ -2220,9 +2223,10 @@ func collectMonitoringMachine( NotificationBaseImpl: types.NotificationBaseImpl{ SubscriptionID: *r.ID, UserID: *r.UserID, - EventName: eventName, + EventName: r.EventName, Epoch: epoch, UnsubscribeHash: r.UnsubscribeHash, + EventFilter: r.EventFilter, }, MachineName: r.EventFilter, } @@ -2348,7 +2352,7 @@ func (n *taxReportNotification) GetInfoMarkdown() string { return n.GetInfo(false) } -func collectTaxReportNotificationNotifications(notificationsByUserID types.NotificationsPerUserId, eventName types.EventName) error { +func collectTaxReportNotificationNotifications(notificationsByUserID types.NotificationsPerUserId) error { lastStatsDay, err := cache.LatestExportedStatisticDay.GetOrDefault(db.GetLastExportedStatisticDay) if err != nil { @@ -2369,7 +2373,7 @@ func collectTaxReportNotificationNotifications(notificationsByUserID types.Notif // name, firstDayOfMonth) dbResults, err := GetSubsForEventFilter( - eventName, + types.TaxReportEventName, "us.last_sent_ts < ? OR (us.last_sent_ts IS NULL AND us.created_ts < ?)", []interface{}{firstDayOfMonth, firstDayOfMonth}, nil, @@ -2387,6 +2391,7 @@ func collectTaxReportNotificationNotifications(notificationsByUserID types.Notif Epoch: sub.CreatedEpoch, EventFilter: sub.EventFilter, UnsubscribeHash: sub.UnsubscribeHash, + EventName: sub.EventName, }, } notificationsByUserID.AddNotification(n) @@ -2419,7 +2424,7 @@ func (n *networkNotification) GetInfoMarkdown() string { return generalPart } -func collectNetworkNotifications(notificationsByUserID types.NotificationsPerUserId, eventName types.EventName) error { +func collectNetworkNotifications(notificationsByUserID types.NotificationsPerUserId) error { count := 0 err := db.WriterDb.Get(&count, ` SELECT count(ts) FROM network_liveness WHERE (headepoch-finalizedepoch) > 3 AND ts > now() - interval '60 minutes'; @@ -2438,7 +2443,7 @@ func collectNetworkNotifications(notificationsByUserID types.NotificationsPerUse // utils.GetNetwork()+":"+string(eventName)) dbResult, err := GetSubsForEventFilter( - eventName, + types.NetworkLivenessIncreasedEventName, "us.last_sent_ts <= NOW() - INTERVAL '1 hour' OR us.last_sent_ts IS NULL", nil, nil, @@ -2456,6 +2461,7 @@ func collectNetworkNotifications(notificationsByUserID types.NotificationsPerUse Epoch: sub.CreatedEpoch, EventFilter: sub.EventFilter, UnsubscribeHash: sub.UnsubscribeHash, + EventName: sub.EventName, }, } @@ -2512,7 +2518,7 @@ func (n *rocketpoolNotification) GetInfoMarkdown() string { return n.GetInfo(false) } -func collectRocketpoolComissionNotifications(notificationsByUserID types.NotificationsPerUserId, eventName types.EventName) error { +func collectRocketpoolComissionNotifications(notificationsByUserID types.NotificationsPerUserId) error { fee := 0.0 err := db.WriterDb.Get(&fee, ` select current_node_fee from rocketpool_network_stats order by id desc LIMIT 1; @@ -2531,7 +2537,7 @@ func collectRocketpoolComissionNotifications(notificationsByUserID types.Notific // utils.GetNetwork()+":"+string(eventName), fee) dbResult, err := GetSubsForEventFilter( - eventName, + types.RocketpoolCommissionThresholdEventName, "(us.last_sent_ts <= NOW() - INTERVAL '8 hours' OR us.last_sent_ts IS NULL) AND (us.event_threshold <= ? OR (us.event_threshold < 0 AND us.event_threshold * -1 >= ?)", []interface{}{fee, fee}, nil, @@ -2548,7 +2554,7 @@ func collectRocketpoolComissionNotifications(notificationsByUserID types.Notific UserID: *sub.UserID, Epoch: sub.CreatedEpoch, EventFilter: sub.EventFilter, - EventName: eventName, + EventName: sub.EventName, UnsubscribeHash: sub.UnsubscribeHash, }, ExtraData: strconv.FormatInt(int64(fee*100), 10) + "%", @@ -2563,7 +2569,7 @@ func collectRocketpoolComissionNotifications(notificationsByUserID types.Notific return nil } -func collectRocketpoolRewardClaimRoundNotifications(notificationsByUserID types.NotificationsPerUserId, eventName types.EventName) error { +func collectRocketpoolRewardClaimRoundNotifications(notificationsByUserID types.NotificationsPerUserId) error { var ts int64 err := db.WriterDb.Get(&ts, ` select date_part('epoch', claim_interval_time_start)::int from rocketpool_network_stats order by id desc LIMIT 1; @@ -2584,7 +2590,7 @@ func collectRocketpoolRewardClaimRoundNotifications(notificationsByUserID types. // utils.GetNetwork()+":"+string(eventName)) dbResult, err := GetSubsForEventFilter( - eventName, + types.RocketpoolNewClaimRoundStartedEventName, "us.last_sent_ts <= NOW() - INTERVAL '5 hours' OR us.last_sent_ts IS NULL", nil, nil, @@ -2601,7 +2607,7 @@ func collectRocketpoolRewardClaimRoundNotifications(notificationsByUserID types. UserID: *sub.UserID, Epoch: sub.CreatedEpoch, EventFilter: sub.EventFilter, - EventName: eventName, + EventName: sub.EventName, UnsubscribeHash: sub.UnsubscribeHash, }, } @@ -2736,7 +2742,7 @@ func collectRocketpoolRPLCollateralNotifications(notificationsByUserID types.Not UserID: *sub.UserID, Epoch: epoch, EventFilter: sub.EventFilter, - EventName: eventName, + EventName: sub.EventName, UnsubscribeHash: sub.UnsubscribeHash, }, ExtraData: strings.TrimRight(strings.TrimRight(fmt.Sprintf("%.2f", threshold*100), "0"), "."), @@ -2790,7 +2796,7 @@ func bigFloat(x float64) *big.Float { return new(big.Float).SetFloat64(x) } -func collectSyncCommittee(notificationsByUserID types.NotificationsPerUserId, eventName types.EventName, epoch uint64) error { +func collectSyncCommittee(notificationsByUserID types.NotificationsPerUserId, epoch uint64) error { slotsPerSyncCommittee := utils.SlotsPerSyncCommittee() currentPeriod := epoch * utils.Config.Chain.ClConfig.SlotsPerEpoch / slotsPerSyncCommittee nextPeriod := currentPeriod + 1 @@ -2816,7 +2822,7 @@ func collectSyncCommittee(notificationsByUserID types.NotificationsPerUserId, ev pubKeys = append(pubKeys, val.PubKey) } - dbResult, err := GetSubsForEventFilter(eventName, "us.last_sent_ts <= NOW() - INTERVAL '26 hours' OR us.last_sent_ts IS NULL", nil, pubKeys) + dbResult, err := GetSubsForEventFilter(types.SyncCommitteeSoon, "us.last_sent_ts <= NOW() - INTERVAL '26 hours' OR us.last_sent_ts IS NULL", nil, pubKeys) // err = db.FrontendWriterDB.Select(&dbResult, ` // SELECT us.id, us.user_id, us.event_filter, ENCODE(us.unsubscribe_hash, 'hex') as unsubscribe_hash // FROM users_subscriptions AS us @@ -2837,7 +2843,7 @@ func collectSyncCommittee(notificationsByUserID types.NotificationsPerUserId, ev UserID: *sub.UserID, Epoch: epoch, EventFilter: sub.EventFilter, - EventName: eventName, + EventName: sub.EventName, UnsubscribeHash: sub.UnsubscribeHash, }, ExtraData: fmt.Sprintf("%v|%v|%v", mapping[sub.EventFilter], nextPeriod*utils.Config.Chain.ClConfig.EpochsPerSyncCommitteePeriod, (nextPeriod+1)*utils.Config.Chain.ClConfig.EpochsPerSyncCommitteePeriod), From 65f0c8f055cef326cb5b468fd20586185134ff08 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Mon, 8 Jul 2024 13:48:31 +0200 Subject: [PATCH 11/94] remove direct email unsubscribe --- backend/pkg/commons/types/frontend.go | 23 +-- backend/pkg/notification/db.go | 11 +- backend/pkg/notification/notifications.go | 185 +++++++--------------- 3 files changed, 66 insertions(+), 153 deletions(-) diff --git a/backend/pkg/commons/types/frontend.go b/backend/pkg/commons/types/frontend.go index 2a4f0d8ca..9c8f45c04 100644 --- a/backend/pkg/commons/types/frontend.go +++ b/backend/pkg/commons/types/frontend.go @@ -273,7 +273,6 @@ type Notification interface { GetTitle() string GetEventFilter() string GetEmailAttachment() *EmailAttachment - GetUnsubscribeHash() string GetInfoMarkdown() string GetUserId() UserId } @@ -287,7 +286,6 @@ type NotificationBaseImpl struct { Title string EventFilter string EmailAttachment *EmailAttachment - UnsubscribeHash sql.NullString InfoMarkdown string UserID UserId } @@ -324,13 +322,6 @@ func (n NotificationBaseImpl) GetEmailAttachment() *EmailAttachment { return n.EmailAttachment } -func (n NotificationBaseImpl) GetUnsubscribeHash() string { - if n.UnsubscribeHash.Valid { - return n.UnsubscribeHash.String - } - return "" -} - func (n NotificationBaseImpl) GetInfoMarkdown() string { return n.InfoMarkdown } @@ -349,13 +340,12 @@ type Subscription struct { LastSent *time.Time `db:"last_sent_ts"` LastEpoch *uint64 `db:"last_sent_epoch"` // Channels pq.StringArray `db:"channels"` - CreatedTime time.Time `db:"created_ts"` - CreatedEpoch uint64 `db:"created_epoch"` - EventThreshold float64 `db:"event_threshold"` - UnsubscribeHash sql.NullString `db:"unsubscribe_hash" swaggertype:"string"` - State sql.NullString `db:"internal_state" swaggertype:"string"` - GroupId *int64 - DashboardId *int64 + CreatedTime time.Time `db:"created_ts"` + CreatedEpoch uint64 `db:"created_epoch"` + EventThreshold float64 `db:"event_threshold"` + State sql.NullString `db:"internal_state" swaggertype:"string"` + GroupId *int64 + DashboardId *int64 } type UserId uint64 @@ -562,7 +552,6 @@ type Email struct { Title string Body template.HTML SubscriptionManageURL template.HTML - UnsubURL template.HTML } type UserWebhook struct { diff --git a/backend/pkg/notification/db.go b/backend/pkg/notification/db.go index 36810d6b3..cbcb9d497 100644 --- a/backend/pkg/notification/db.go +++ b/backend/pkg/notification/db.go @@ -38,7 +38,6 @@ func GetSubsForEventFilter(eventName types.EventName, lastSentFilter string, las goqu.C("last_sent_epoch"), goqu.C("created_epoch"), goqu.C("event_threshold"), - goqu.L("ENCODE(unsubscribe_hash, 'hex') as unsubscribe_hash"), goqu.C("internal_state"), ).Where(goqu.C("event_name").Eq(utils.GetNetwork() + ":" + string(eventName))) @@ -68,15 +67,7 @@ func GetSubsForEventFilter(eventName types.EventName, lastSentFilter string, las if _, ok := subMap[sub.EventFilter]; !ok { subMap[sub.EventFilter] = make([]types.Subscription, 0) } - subMap[sub.EventFilter] = append(subMap[sub.EventFilter], types.Subscription{ - UserID: sub.UserID, - ID: sub.ID, - LastEpoch: sub.LastEpoch, - EventFilter: sub.EventFilter, - CreatedEpoch: sub.CreatedEpoch, - EventThreshold: sub.EventThreshold, - State: sub.State, - }) + subMap[sub.EventFilter] = append(subMap[sub.EventFilter], sub) } return subMap, nil } diff --git a/backend/pkg/notification/notifications.go b/backend/pkg/notification/notifications.go index e803f8514..2d7b6ff7c 100644 --- a/backend/pkg/notification/notifications.go +++ b/backend/pkg/notification/notifications.go @@ -3,7 +3,6 @@ package notification import ( "bytes" "context" - "crypto/sha256" "database/sql" "database/sql/driver" "encoding/hex" @@ -11,7 +10,6 @@ import ( "errors" "fmt" - "html" "html/template" "io" "math/big" @@ -598,6 +596,9 @@ func queuePushNotification(notificationsByUserID types.NotificationsPerUserId, u continue } + // todo: this looks like a flawed approach to queue the notifications + // this will issue one db write per user, which is not optimal + // we should batch the notifications and write them in one go go func(userTokens []string, userNotifications map[types.EventName]map[types.EventFilter]types.Notification) { var batch []*messaging.Message for event, ns := range userNotifications { @@ -727,7 +728,6 @@ func queueEmailNotifications(notificationsByUserID types.NotificationsPerUserId, } //nolint:gosec // this is a static string msg.Body += template.HTML(fmt.Sprintf("%s
====

", types.EventLabel[event_title])) - unsubURL := "https://" + utils.Config.Frontend.SiteDomain + "/notifications/unsubscribe" i := 0 for _, n := range ns { // Find all unique notification titles for the subject @@ -737,64 +737,6 @@ func queueEmailNotifications(notificationsByUserID types.NotificationsPerUserId, notificationTitles = append(notificationTitles, title) } - // TODO: this is bad and will break in case there are a lot of unsubscribe hashes to generate - // the unsubscribe hash should be set when we add the subscription to the db - unsubHash := n.GetUnsubscribeHash() - if unsubHash == "" { - id := n.GetSubscriptionID() - - tx, err := db.FrontendWriterDB.Beginx() - if err != nil { - log.Error(err, "error starting transaction", 0) - } - var sub types.Subscription - err = tx.Get(&sub, ` - SELECT - id, - user_id, - event_name, - event_filter, - last_sent_ts, - last_sent_epoch, - created_ts, - created_epoch, - event_threshold - FROM users_subscriptions - WHERE id = $1 - `, id) - if err != nil { - log.Error(err, "error getting user subscription by subscription id", 0) - utils.Rollback(tx) - continue - } - - raw := fmt.Sprintf("%v%v%v%v", sub.ID, sub.UserID, sub.EventName, sub.CreatedTime) - digest := sha256.Sum256([]byte(raw)) - - _, err = tx.Exec("UPDATE users_subscriptions set unsubscribe_hash = $1 WHERE id = $2", digest[:], id) - if err != nil { - log.Error(err, "error updating users subscriptions table with unsubscribe hash", 0) - utils.Rollback(tx) - continue - } - - err = tx.Commit() - if err != nil { - log.Error(err, "error committing transaction to update users subscriptions with an unsubscribe hash", 0) - utils.Rollback(tx) - continue - } - - unsubHash = hex.EncodeToString(digest[:]) - } - if i == 0 { - unsubURL += "?hash=" + html.EscapeString(unsubHash) - } else { - unsubURL += "&hash=" + html.EscapeString(unsubHash) - } - //nolint:gosec // this is a static string - msg.UnsubURL = template.HTML(fmt.Sprintf(`Unsubscribe`, unsubURL)) - if event != types.SyncCommitteeSoon { // SyncCommitteeSoon notifications are summed up in getEventInfo for all validators //nolint:gosec // this is a static string @@ -1150,6 +1092,8 @@ func sendDiscordNotifications(useDB *sqlx.DB) error { notifMap[n.Content.Webhook.ID] = append(notifMap[n.Content.Webhook.ID], n) } for _, webhook := range webhookMap { + // todo: this has the potential to spin up thousands of go routines + // should use an errgroup instead if we decide to keep the aproach go func(webhook types.UserWebhook, reqs []types.TransitDiscord) { defer func() { // update retries counters in db based on end result @@ -1780,7 +1724,7 @@ func collectValidatorGotSlashedNotifications(notificationsByUserID types.Notific resultsLen := len(dbResult) for i, event := range dbResult { // TODO: clarify why we need the id here?! - query += fmt.Sprintf(`SELECT %d AS ref, id, user_id, ENCODE(unsubscribe_hash, 'hex') AS unsubscribe_hash, event_name from users_subscriptions where event_name = $1 AND event_filter = '%x'`, i, event.SlashedValidatorPubkey) + query += fmt.Sprintf(`SELECT %d AS ref, id, user_id, event_name from users_subscriptions where event_name = $1 AND event_filter = '%x'`, i, event.SlashedValidatorPubkey) if i < resultsLen-1 { query += " UNION " } @@ -1791,11 +1735,10 @@ func collectValidatorGotSlashedNotifications(notificationsByUserID types.Notific } var subscribers []struct { - Ref uint64 `db:"ref"` - Id uint64 `db:"id"` - UserId types.UserId `db:"user_id"` - UnsubscribeHash sql.NullString `db:"unsubscribe_hash"` - EventName types.EventName `db:"event_name"` + Ref uint64 `db:"ref"` + Id uint64 `db:"id"` + UserId types.UserId `db:"user_id"` + EventName types.EventName `db:"event_name"` } name := string(types.ValidatorGotSlashedEventName) @@ -1814,12 +1757,11 @@ func collectValidatorGotSlashedNotifications(notificationsByUserID types.Notific n := &validatorGotSlashedNotification{ NotificationBaseImpl: types.NotificationBaseImpl{ - SubscriptionID: sub.Id, - UserID: sub.UserId, - Epoch: event.Epoch, - EventFilter: hex.EncodeToString(event.SlashedValidatorPubkey), - UnsubscribeHash: sub.UnsubscribeHash, - EventName: sub.EventName, + SubscriptionID: sub.Id, + UserID: sub.UserId, + Epoch: event.Epoch, + EventFilter: hex.EncodeToString(event.SlashedValidatorPubkey), + EventName: sub.EventName, }, Slasher: event.SlasherIndex, Reason: event.Reason, @@ -1894,11 +1836,10 @@ func collectWithdrawalNotifications(notificationsByUserID types.NotificationsPer // log.Infof("creating %v notification for validator %v in epoch %v", types.ValidatorReceivedWithdrawalEventName, event.ValidatorIndex, epoch) n := &validatorWithdrawalNotification{ NotificationBaseImpl: types.NotificationBaseImpl{ - SubscriptionID: *sub.ID, - UserID: *sub.UserID, - EventFilter: hex.EncodeToString(event.Pubkey), - UnsubscribeHash: sub.UnsubscribeHash, - EventName: sub.EventName, + SubscriptionID: *sub.ID, + UserID: *sub.UserID, + EventFilter: hex.EncodeToString(event.Pubkey), + EventName: sub.EventName, }, ValidatorIndex: event.ValidatorIndex, Epoch: epoch, @@ -2023,12 +1964,11 @@ func collectEthClientNotifications(notificationsByUserID types.NotificationsPerU for _, sub := range subs { n := ðClientNotification{ NotificationBaseImpl: types.NotificationBaseImpl{ - SubscriptionID: *sub.ID, - UserID: *sub.UserID, - Epoch: sub.CreatedEpoch, - EventFilter: sub.EventFilter, - UnsubscribeHash: sub.UnsubscribeHash, - EventName: sub.EventName, + SubscriptionID: *sub.ID, + UserID: *sub.UserID, + Epoch: sub.CreatedEpoch, + EventFilter: sub.EventFilter, + EventName: sub.EventName, }, EthClient: client.Name, } @@ -2221,12 +2161,11 @@ func collectMonitoringMachine( for _, r := range result { n := &monitorMachineNotification{ NotificationBaseImpl: types.NotificationBaseImpl{ - SubscriptionID: *r.ID, - UserID: *r.UserID, - EventName: r.EventName, - Epoch: epoch, - UnsubscribeHash: r.UnsubscribeHash, - EventFilter: r.EventFilter, + SubscriptionID: *r.ID, + UserID: *r.UserID, + EventName: r.EventName, + Epoch: epoch, + EventFilter: r.EventFilter, }, MachineName: r.EventFilter, } @@ -2386,12 +2325,11 @@ func collectTaxReportNotificationNotifications(notificationsByUserID types.Notif for _, sub := range subs { n := &taxReportNotification{ NotificationBaseImpl: types.NotificationBaseImpl{ - SubscriptionID: *sub.ID, - UserID: *sub.UserID, - Epoch: sub.CreatedEpoch, - EventFilter: sub.EventFilter, - UnsubscribeHash: sub.UnsubscribeHash, - EventName: sub.EventName, + SubscriptionID: *sub.ID, + UserID: *sub.UserID, + Epoch: sub.CreatedEpoch, + EventFilter: sub.EventFilter, + EventName: sub.EventName, }, } notificationsByUserID.AddNotification(n) @@ -2456,12 +2394,11 @@ func collectNetworkNotifications(notificationsByUserID types.NotificationsPerUse for _, sub := range subs { n := &networkNotification{ NotificationBaseImpl: types.NotificationBaseImpl{ - SubscriptionID: *sub.ID, - UserID: *sub.UserID, - Epoch: sub.CreatedEpoch, - EventFilter: sub.EventFilter, - UnsubscribeHash: sub.UnsubscribeHash, - EventName: sub.EventName, + SubscriptionID: *sub.ID, + UserID: *sub.UserID, + Epoch: sub.CreatedEpoch, + EventFilter: sub.EventFilter, + EventName: sub.EventName, }, } @@ -2550,12 +2487,11 @@ func collectRocketpoolComissionNotifications(notificationsByUserID types.Notific for _, sub := range subs { n := &rocketpoolNotification{ NotificationBaseImpl: types.NotificationBaseImpl{ - SubscriptionID: *sub.ID, - UserID: *sub.UserID, - Epoch: sub.CreatedEpoch, - EventFilter: sub.EventFilter, - EventName: sub.EventName, - UnsubscribeHash: sub.UnsubscribeHash, + SubscriptionID: *sub.ID, + UserID: *sub.UserID, + Epoch: sub.CreatedEpoch, + EventFilter: sub.EventFilter, + EventName: sub.EventName, }, ExtraData: strconv.FormatInt(int64(fee*100), 10) + "%", } @@ -2603,12 +2539,11 @@ func collectRocketpoolRewardClaimRoundNotifications(notificationsByUserID types. for _, sub := range subs { n := &rocketpoolNotification{ NotificationBaseImpl: types.NotificationBaseImpl{ - SubscriptionID: *sub.ID, - UserID: *sub.UserID, - Epoch: sub.CreatedEpoch, - EventFilter: sub.EventFilter, - EventName: sub.EventName, - UnsubscribeHash: sub.UnsubscribeHash, + SubscriptionID: *sub.ID, + UserID: *sub.UserID, + Epoch: sub.CreatedEpoch, + EventFilter: sub.EventFilter, + EventName: sub.EventName, }, } @@ -2738,12 +2673,11 @@ func collectRocketpoolRPLCollateralNotifications(notificationsByUserID types.Not n := &rocketpoolNotification{ NotificationBaseImpl: types.NotificationBaseImpl{ - SubscriptionID: *sub.ID, - UserID: *sub.UserID, - Epoch: epoch, - EventFilter: sub.EventFilter, - EventName: sub.EventName, - UnsubscribeHash: sub.UnsubscribeHash, + SubscriptionID: *sub.ID, + UserID: *sub.UserID, + Epoch: epoch, + EventFilter: sub.EventFilter, + EventName: sub.EventName, }, ExtraData: strings.TrimRight(strings.TrimRight(fmt.Sprintf("%.2f", threshold*100), "0"), "."), } @@ -2839,12 +2773,11 @@ func collectSyncCommittee(notificationsByUserID types.NotificationsPerUserId, ep for _, sub := range subs { n := &rocketpoolNotification{ NotificationBaseImpl: types.NotificationBaseImpl{ - SubscriptionID: *sub.ID, - UserID: *sub.UserID, - Epoch: epoch, - EventFilter: sub.EventFilter, - EventName: sub.EventName, - UnsubscribeHash: sub.UnsubscribeHash, + SubscriptionID: *sub.ID, + UserID: *sub.UserID, + Epoch: epoch, + EventFilter: sub.EventFilter, + EventName: sub.EventName, }, ExtraData: fmt.Sprintf("%v|%v|%v", mapping[sub.EventFilter], nextPeriod*utils.Config.Chain.ClConfig.EpochsPerSyncCommitteePeriod, (nextPeriod+1)*utils.Config.Chain.ClConfig.EpochsPerSyncCommitteePeriod), } From c85a5e5ff247b763c609fb46ad2d7e1a6a5f472f Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Tue, 9 Jul 2024 12:25:51 +0200 Subject: [PATCH 12/94] add handling of orphaned block notifications --- backend/pkg/notification/notifications.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/backend/pkg/notification/notifications.go b/backend/pkg/notification/notifications.go index 2d7b6ff7c..5ca1a1108 100644 --- a/backend/pkg/notification/notifications.go +++ b/backend/pkg/notification/notifications.go @@ -1306,6 +1306,8 @@ func (n *validatorProposalNotification) GetInfo(includeUrl bool) string { generalPart = fmt.Sprintf(`Validator %s proposed block at slot %s with %v %v execution reward.`, vali, slot, n.Reward, utils.Config.Frontend.ElCurrency) case 2: generalPart = fmt.Sprintf(`Validator %s missed a block proposal at slot %s.`, vali, slot) + case 3: + generalPart = fmt.Sprintf(`Validator %s had an orphaned block proposal at slot %s.`, vali, slot) } return generalPart + suffix } @@ -1318,6 +1320,8 @@ func (n *validatorProposalNotification) GetTitle() string { return "New Block Proposal" case 2: return "Block Proposal Missed" + case 3: + return "Block Proposal Missed (Orphaned)" } return "-" } @@ -1331,6 +1335,8 @@ func (n *validatorProposalNotification) GetInfoMarkdown() string { generalPart = fmt.Sprintf(`Validator [%[2]v](https://%[1]v/validator/%[2]v) proposed a new block at slot [%[3]v](https://%[1]v/slot/%[3]v) with %[4]v %[5]v execution reward.`, utils.Config.Frontend.SiteDomain, n.ValidatorIndex, n.Slot, n.Reward, utils.Config.Frontend.ElCurrency) case 2: generalPart = fmt.Sprintf(`Validator [%[2]v](https://%[1]v/validator/%[2]v) missed a block proposal at slot [%[3]v](https://%[1]v/slot/%[3]v).`, utils.Config.Frontend.SiteDomain, n.ValidatorIndex, n.Slot) + case 3: + generalPart = fmt.Sprintf(`Validator [%[2]v](https://%[1]v/validator/%[2]v) had an orphaned block proposal at slot [%[3]v](https://%[1]v/slot/%[3]v).`, utils.Config.Frontend.SiteDomain, n.ValidatorIndex, n.Slot) } return generalPart From 2ca40f7f1633ee5447e5bd81cb673d2b90cd9c33 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Tue, 9 Jul 2024 12:50:44 +0200 Subject: [PATCH 13/94] implement code review changes --- backend/pkg/commons/db/db.go | 18 +-- backend/pkg/commons/types/exporter.go | 9 ++ backend/pkg/commons/types/frontend.go | 7 +- backend/pkg/notification/db.go | 1 + backend/pkg/notification/notifications.go | 142 +++++++++++----------- 5 files changed, 88 insertions(+), 89 deletions(-) diff --git a/backend/pkg/commons/db/db.go b/backend/pkg/commons/db/db.go index f85f3641e..fe13e3e67 100644 --- a/backend/pkg/commons/db/db.go +++ b/backend/pkg/commons/db/db.go @@ -954,22 +954,8 @@ func GetTotalEligibleEther() (uint64, error) { } // GetValidatorsGotSlashed returns the validators that got slashed after `epoch` either by an attestation violation or a proposer violation -func GetValidatorsGotSlashed(epoch uint64) ([]struct { - Epoch uint64 `db:"epoch"` - SlasherIndex uint64 `db:"slasher"` - SlasherPubkey string `db:"slasher_pubkey"` - SlashedValidatorIndex uint64 `db:"slashedvalidator"` - SlashedValidatorPubkey []byte `db:"slashedvalidator_pubkey"` - Reason string `db:"reason"` -}, error) { - var dbResult []struct { - Epoch uint64 `db:"epoch"` - SlasherIndex uint64 `db:"slasher"` - SlasherPubkey string `db:"slasher_pubkey"` - SlashedValidatorIndex uint64 `db:"slashedvalidator"` - SlashedValidatorPubkey []byte `db:"slashedvalidator_pubkey"` - Reason string `db:"reason"` - } +func GetValidatorsGotSlashed(epoch uint64) ([]*types.SlashingInfo, error) { + var dbResult []*types.SlashingInfo err := ReaderDb.Select(&dbResult, ` WITH slashings AS ( diff --git a/backend/pkg/commons/types/exporter.go b/backend/pkg/commons/types/exporter.go index 5a3f3a397..197638472 100644 --- a/backend/pkg/commons/types/exporter.go +++ b/backend/pkg/commons/types/exporter.go @@ -709,3 +709,12 @@ type RedisCachedValidatorsMapping struct { Epoch Epoch Mapping []*CachedValidator } + +type SlashingInfo struct { + Epoch uint64 `db:"epoch"` + SlasherIndex uint64 `db:"slasher"` + SlasherPubkey string `db:"slasher_pubkey"` + SlashedValidatorIndex uint64 `db:"slashedvalidator"` + SlashedValidatorPubkey []byte `db:"slashedvalidator_pubkey"` + Reason string `db:"reason"` +} diff --git a/backend/pkg/commons/types/frontend.go b/backend/pkg/commons/types/frontend.go index 9c8f45c04..5c90a8be3 100644 --- a/backend/pkg/commons/types/frontend.go +++ b/backend/pkg/commons/types/frontend.go @@ -33,9 +33,10 @@ func (npui NotificationsPerUserId) AddNotification(n Notification) { if n.GetEventName() == "" { log.Fatal(fmt.Errorf("Notification event name is empty"), fmt.Sprintf("Notification: %v", n), 0) } - if n.GetEventFilter() == "" { - log.Fatal(fmt.Errorf("Notification event filter is empty"), fmt.Sprintf("Notification: %v", n), 0) - } + // next check is disabled as there are events that do not require a filter (rocketpool, network events) + // if n.GetEventFilter() == "" { + // log.Fatal(fmt.Errorf("Notification event filter is empty"), fmt.Sprintf("Notification: %v", n), 0) + // } if _, ok := npui[n.GetUserId()]; !ok { npui[n.GetUserId()] = make(map[EventName]map[EventFilter]Notification) diff --git a/backend/pkg/notification/db.go b/backend/pkg/notification/db.go index cbcb9d497..7b8948d34 100644 --- a/backend/pkg/notification/db.go +++ b/backend/pkg/notification/db.go @@ -12,6 +12,7 @@ import ( // Map key corresponds to the event filter which can be // a validator pubkey or an eth1 address (for RPL notifications) // or a list of validators for the tax report notifications +// or a machine name for machine notifications or a eth client name for ethereum client update notifications // optionally it is possible to set a filter on the last sent ts and the event filter // fields func GetSubsForEventFilter(eventName types.EventName, lastSentFilter string, lastSentFilterArgs []interface{}, eventFilters []string) (map[string][]types.Subscription, error) { diff --git a/backend/pkg/notification/notifications.go b/backend/pkg/notification/notifications.go index 5ca1a1108..6b6e0ea0c 100644 --- a/backend/pkg/notification/notifications.go +++ b/backend/pkg/notification/notifications.go @@ -1726,55 +1726,44 @@ func collectValidatorGotSlashedNotifications(notificationsByUserID types.Notific if err != nil { return fmt.Errorf("error getting slashed validators from database, err: %w", err) } - query := "" - resultsLen := len(dbResult) - for i, event := range dbResult { - // TODO: clarify why we need the id here?! - query += fmt.Sprintf(`SELECT %d AS ref, id, user_id, event_name from users_subscriptions where event_name = $1 AND event_filter = '%x'`, i, event.SlashedValidatorPubkey) - if i < resultsLen-1 { - query += " UNION " - } - } - - if query == "" { - return nil - } - - var subscribers []struct { - Ref uint64 `db:"ref"` - Id uint64 `db:"id"` - UserId types.UserId `db:"user_id"` - EventName types.EventName `db:"event_name"` + slashedPubkeys := make([]string, 0, len(dbResult)) + pubkeyToSlashingInfoMap := make(map[string]*types.SlashingInfo) + for _, event := range dbResult { + pubkeyStr := hex.EncodeToString(event.SlashedValidatorPubkey) + slashedPubkeys = append(slashedPubkeys, pubkeyStr) + pubkeyToSlashingInfoMap[pubkeyStr] = event } - name := string(types.ValidatorGotSlashedEventName) - if utils.Config.Chain.ClConfig.ConfigName != "" { - name = utils.Config.Chain.ClConfig.ConfigName + ":" + name - } - err = db.FrontendWriterDB.Select(&subscribers, query, name) + subscribedUsers, err := GetSubsForEventFilter(types.ValidatorGotSlashedEventName, "", nil, slashedPubkeys) if err != nil { - return fmt.Errorf("error querying subscribers, err: %w", err) + return fmt.Errorf("failed to get subs for %v: %v", types.ValidatorGotSlashedEventName, err) } - for _, sub := range subscribers { - event := dbResult[sub.Ref] + for _, subs := range subscribedUsers { + for _, sub := range subs { - log.Infof("creating %v notification for validator %v in epoch %v", event.SlashedValidatorPubkey, event.Reason, epoch) + event := pubkeyToSlashingInfoMap[sub.EventFilter] + if event == nil { + log.Error(fmt.Errorf("error retrieving slashing info for public key %s", sub.EventFilter), "", 0) + continue + } + log.Infof("creating %v notification for validator %v in epoch %v", event.Reason, sub.EventFilter, epoch) - n := &validatorGotSlashedNotification{ - NotificationBaseImpl: types.NotificationBaseImpl{ - SubscriptionID: sub.Id, - UserID: sub.UserId, - Epoch: event.Epoch, - EventFilter: hex.EncodeToString(event.SlashedValidatorPubkey), - EventName: sub.EventName, - }, - Slasher: event.SlasherIndex, - Reason: event.Reason, - ValidatorIndex: event.SlashedValidatorIndex, + n := &validatorGotSlashedNotification{ + NotificationBaseImpl: types.NotificationBaseImpl{ + SubscriptionID: *sub.ID, + UserID: *sub.UserID, + Epoch: epoch, + EventFilter: sub.EventFilter, + EventName: sub.EventName, + }, + Slasher: event.SlasherIndex, + Reason: event.Reason, + ValidatorIndex: event.SlashedValidatorIndex, + } + notificationsByUserID.AddNotification(n) + metrics.NotificationsCollected.WithLabelValues(string(n.GetEventName())).Inc() } - notificationsByUserID.AddNotification(n) - metrics.NotificationsCollected.WithLabelValues(string(n.GetEventName())).Inc() } return nil @@ -2077,28 +2066,39 @@ func collectMonitoringMachine( notifyConditionFulfilled func(subscribeData *types.Subscription, machineData *types.MachineMetricSystemUser) bool, epoch uint64, ) error { - var allSubscribed []*types.Subscription // event_filter == machine name + + dbResult, err := GetSubsForEventFilter( + eventName, + "us.created_epoch <= ? AND (us.last_sent_epoch < (? - ?) OR us.last_sent_epoch IS NULL)", + []interface{}{epoch, epoch, epochWaitInBetween}, + nil, + ) + // TODO: clarify why we need grouping here?! - err := db.FrontendWriterDB.Select(&allSubscribed, - `SELECT - us.user_id, - max(us.id) AS id, - ENCODE((array_agg(us.unsubscribe_hash))[1], 'hex') AS unsubscribe_hash, - event_filter, - COALESCE(event_threshold, 0) AS event_threshold - FROM users_subscriptions us - WHERE us.event_name = $1 AND us.created_epoch <= $2 - AND (us.last_sent_epoch < ($2 - $3) OR us.last_sent_epoch IS NULL) - group by us.user_id, event_filter, event_threshold`, - eventName, epoch, epochWaitInBetween) + // err := db.FrontendWriterDB.Select(&allSubscribed, + // `SELECT + // us.user_id, + // max(us.id) AS id, + // ENCODE((array_agg(us.unsubscribe_hash))[1], 'hex') AS unsubscribe_hash, + // event_filter, + // COALESCE(event_threshold, 0) AS event_threshold + // FROM users_subscriptions us + // WHERE us.event_name = $1 AND us.created_epoch <= $2 + // AND (us.last_sent_epoch < ($2 - $3) OR us.last_sent_epoch IS NULL) + // group by us.user_id, event_filter, event_threshold`, + // eventName, epoch, epochWaitInBetween) if err != nil { return err } rowKeys := gcp_bigtable.RowList{} - for _, data := range allSubscribed { - rowKeys = append(rowKeys, db.BigtableClient.GetMachineRowKey(*data.UserID, "system", data.EventFilter)) + totalSubscribed := 0 + for _, data := range dbResult { + for _, sub := range data { + rowKeys = append(rowKeys, db.BigtableClient.GetMachineRowKey(*sub.UserID, "system", sub.EventFilter)) + totalSubscribed++ + } } machineDataOfSubscribed, err := db.BigtableClient.GetMachineMetricsForNotifications(rowKeys) @@ -2107,20 +2107,22 @@ func collectMonitoringMachine( } var result []*types.Subscription - for _, data := range allSubscribed { - localData := data // Create a local copy of the data variable - machineMap, found := machineDataOfSubscribed[*localData.UserID] - if !found { - continue - } - currentMachineData, found := machineMap[localData.EventFilter] - if !found { - continue - } + for _, data := range dbResult { + for _, sub := range data { + localData := sub // Create a local copy of the data variable + machineMap, found := machineDataOfSubscribed[*localData.UserID] + if !found { + continue + } + currentMachineData, found := machineMap[localData.EventFilter] + if !found { + continue + } - //logrus.Infof("currentMachineData %v | %v | %v | %v", currentMachine.CurrentDataInsertTs, currentMachine.CompareDataInsertTs, currentMachine.UserID, currentMachine.Machine) - if notifyConditionFulfilled(localData, currentMachineData) { - result = append(result, localData) + //logrus.Infof("currentMachineData %v | %v | %v | %v", currentMachine.CurrentDataInsertTs, currentMachine.CompareDataInsertTs, currentMachine.UserID, currentMachine.Machine) + if notifyConditionFulfilled(&localData, currentMachineData) { + result = append(result, &localData) + } } } @@ -2158,7 +2160,7 @@ func collectMonitoringMachine( subRatioThreshold = subFirstRatioThreshold isFirstNotificationCheck = false } - if float64(len(result))/float64(len(allSubscribed)) >= subRatioThreshold { + if float64(len(result))/float64(totalSubscribed) >= subRatioThreshold { log.Error(nil, fmt.Errorf("error too many users would be notified concerning: %v", eventName), 0) return nil } From f1a2ef83d0f1c545354f8337402440b9cd621c5e Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Tue, 9 Jul 2024 12:53:53 +0200 Subject: [PATCH 14/94] please linter --- backend/pkg/notification/notifications.go | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/pkg/notification/notifications.go b/backend/pkg/notification/notifications.go index 6b6e0ea0c..51cb926ae 100644 --- a/backend/pkg/notification/notifications.go +++ b/backend/pkg/notification/notifications.go @@ -1741,7 +1741,6 @@ func collectValidatorGotSlashedNotifications(notificationsByUserID types.Notific for _, subs := range subscribedUsers { for _, sub := range subs { - event := pubkeyToSlashingInfoMap[sub.EventFilter] if event == nil { log.Error(fmt.Errorf("error retrieving slashing info for public key %s", sub.EventFilter), "", 0) From 274816226cd18f5dc8e1cee2bab64fe6cad92883 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Wed, 10 Jul 2024 10:29:46 +0200 Subject: [PATCH 15/94] remove internal state for notifications --- backend/pkg/commons/types/frontend.go | 12 ++++---- backend/pkg/notification/db.go | 1 - backend/pkg/notification/notifications.go | 35 ++++++++++++----------- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/backend/pkg/commons/types/frontend.go b/backend/pkg/commons/types/frontend.go index 5c90a8be3..767aca38b 100644 --- a/backend/pkg/commons/types/frontend.go +++ b/backend/pkg/commons/types/frontend.go @@ -341,12 +341,12 @@ type Subscription struct { LastSent *time.Time `db:"last_sent_ts"` LastEpoch *uint64 `db:"last_sent_epoch"` // Channels pq.StringArray `db:"channels"` - CreatedTime time.Time `db:"created_ts"` - CreatedEpoch uint64 `db:"created_epoch"` - EventThreshold float64 `db:"event_threshold"` - State sql.NullString `db:"internal_state" swaggertype:"string"` - GroupId *int64 - DashboardId *int64 + CreatedTime time.Time `db:"created_ts"` + CreatedEpoch uint64 `db:"created_epoch"` + EventThreshold float64 `db:"event_threshold"` + // State sql.NullString `db:"internal_state" swaggertype:"string"` + GroupId *int64 + DashboardId *int64 } type UserId uint64 diff --git a/backend/pkg/notification/db.go b/backend/pkg/notification/db.go index 7b8948d34..4363c12f7 100644 --- a/backend/pkg/notification/db.go +++ b/backend/pkg/notification/db.go @@ -39,7 +39,6 @@ func GetSubsForEventFilter(eventName types.EventName, lastSentFilter string, las goqu.C("last_sent_epoch"), goqu.C("created_epoch"), goqu.C("event_threshold"), - goqu.C("internal_state"), ).Where(goqu.C("event_name").Eq(utils.GetNetwork() + ":" + string(eventName))) if lastSentFilter != "" { diff --git a/backend/pkg/notification/notifications.go b/backend/pkg/notification/notifications.go index 51cb926ae..76bedf688 100644 --- a/backend/pkg/notification/notifications.go +++ b/backend/pkg/notification/notifications.go @@ -492,6 +492,8 @@ func queueNotifications(notificationsByUserID types.NotificationsPerUserId, useD } } } + + // obsolete as notifications are anyway sent on a per-epoch basis for epoch, subIDs := range subByEpoch { // update that we've queued the subscription (last sent rather means last queued) err := db.UpdateSubscriptionsLastSent(subIDs, time.Now(), epoch, useDB) @@ -520,6 +522,7 @@ func queueNotifications(notificationsByUserID types.NotificationsPerUserId, useD } } + // no need to batch here as the internal state will become obsolete for state, subs := range stateToSub { subArray := make([]int64, 0) for subID := range subs { @@ -1549,21 +1552,21 @@ func collectAttestationAndOfflineValidatorNotifications(notificationsByUserID ty t := hex.EncodeToString(validator.Pubkey) subs := subMap[t] for _, sub := range subs { - if sub.State.String == "" || sub.State.String == "-" { // discard online notifications that do not have a corresponding offline notification - continue - } + // if sub.State.String == "" || sub.State.String == "-" { // discard online notifications that do not have a corresponding offline notification + // continue + // } - originalLastSeenEpoch, err := strconv.ParseUint(sub.State.String, 10, 64) - if err != nil { - // I have no idea what just happened. - return fmt.Errorf("this should never happen. couldn't parse state as uint64: %v", err) - } + // originalLastSeenEpoch, err := strconv.ParseUint(sub.State.String, 10, 64) + // if err != nil { + // // I have no idea what just happened. + // return fmt.Errorf("this should never happen. couldn't parse state as uint64: %v", err) + // } - epochsSinceOffline := epoch - originalLastSeenEpoch + // epochsSinceOffline := epoch - originalLastSeenEpoch - if epochsSinceOffline > epoch { // fix overflow - epochsSinceOffline = 4 - } + // if epochsSinceOffline > epoch { // fix overflow + // epochsSinceOffline = 4 + // } if sub.UserID == nil || sub.ID == nil { return fmt.Errorf("error expected userId and subId to be defined but got user: %v, sub: %v", sub.UserID, sub.ID) @@ -1582,7 +1585,6 @@ func collectAttestationAndOfflineValidatorNotifications(notificationsByUserID ty }, ValidatorIndex: validator.Index, IsOffline: false, - EpochsOffline: epochsSinceOffline, } notificationsByUserID.AddNotification(n) @@ -1597,7 +1599,6 @@ type validatorIsOfflineNotification struct { types.NotificationBaseImpl ValidatorIndex uint64 - EpochsOffline uint64 IsOffline bool } @@ -1611,9 +1612,9 @@ func (n *validatorIsOfflineNotification) GetInfo(includeUrl bool) string { } } else { if includeUrl { - return fmt.Sprintf(`Validator %[1]v is back online since epoch %[2]v (was offline for %[4]v epoch(s)).`, n.ValidatorIndex, n.Epoch, utils.Config.Frontend.SiteDomain, n.EpochsOffline) + return fmt.Sprintf(`Validator %[1]v is back online since epoch %[2]v.`, n.ValidatorIndex, n.Epoch, utils.Config.Frontend.SiteDomain) } else { - return fmt.Sprintf(`Validator %v is back online since epoch %v (was offline for %v epoch(s)).`, n.ValidatorIndex, n.Epoch, n.EpochsOffline) + return fmt.Sprintf(`Validator %v is back online since epoch %v.`, n.ValidatorIndex, n.Epoch) } } } @@ -1630,7 +1631,7 @@ func (n *validatorIsOfflineNotification) GetInfoMarkdown() string { if n.IsOffline { return fmt.Sprintf(`Validator [%[1]v](https://%[3]v/validator/%[1]v) is offline since epoch [%[2]v](https://%[3]v/epoch/%[2]v).`, n.ValidatorIndex, n.Epoch, utils.Config.Frontend.SiteDomain) } else { - return fmt.Sprintf(`Validator [%[1]v](https://%[3]v/validator/%[1]v) is back online since epoch [%[2]v](https://%[3]v/epoch/%[2]v) (was offline for %[4]v epoch(s)).`, n.ValidatorIndex, n.Epoch, utils.Config.Frontend.SiteDomain, n.EpochsOffline) + return fmt.Sprintf(`Validator [%[1]v](https://%[3]v/validator/%[1]v) is back online since epoch [%[2]v](https://%[3]v/epoch/%[2]v).`, n.ValidatorIndex, n.Epoch, utils.Config.Frontend.SiteDomain) } } From 161652041681b02afa609ed8620731b301d1ff58 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Mon, 2 Sep 2024 14:46:24 +0200 Subject: [PATCH 16/94] fix(notifications): properly log error --- backend/pkg/notification/notifications.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/pkg/notification/notifications.go b/backend/pkg/notification/notifications.go index 76bedf688..ba06f4132 100644 --- a/backend/pkg/notification/notifications.go +++ b/backend/pkg/notification/notifications.go @@ -166,7 +166,7 @@ func notificationSender() { if err != nil { log.Error(err, "error getting advisory lock from db", 0) - conn.Close() + err := conn.Close() if err != nil { log.Error(err, "error returning connection to connection pool", 0) } From bc57bfd680e69bd97c75a239b659f35ca510f80b Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Mon, 9 Sep 2024 09:54:53 +0000 Subject: [PATCH 17/94] simplify extraction of user ids from the notification map --- backend/pkg/notification/notifications.go | 63 +++++++++++------------ 1 file changed, 30 insertions(+), 33 deletions(-) diff --git a/backend/pkg/notification/notifications.go b/backend/pkg/notification/notifications.go index e68f5f3f6..71ec9e0eb 100644 --- a/backend/pkg/notification/notifications.go +++ b/backend/pkg/notification/notifications.go @@ -8,6 +8,8 @@ import ( "encoding/hex" "encoding/json" "errors" + "maps" + "slices" "fmt" "html/template" @@ -283,28 +285,28 @@ func collectNotifications(epoch uint64) (types.NotificationsPerUserId, error) { return nil, fmt.Errorf("error getting dashboard definitions: %v", err) } - // Now initialize the validator dashboard configuration map - validatorDashboardConfig := &types.ValidatorDashboardConfig{ - DashboardsByUserId: make(map[types.UserId]map[types.DashboardId]*types.ValidatorDashboard), - } - for _, row := range dashboardDefinitions { - if validatorDashboardConfig.DashboardsByUserId[row.UserId] == nil { - validatorDashboardConfig.DashboardsByUserId[row.UserId] = make(map[types.DashboardId]*types.ValidatorDashboard) - } - if validatorDashboardConfig.DashboardsByUserId[row.UserId][row.DashboardId] == nil { - validatorDashboardConfig.DashboardsByUserId[row.UserId][row.DashboardId] = &types.ValidatorDashboard{ - Name: row.DashboardName, - Groups: make(map[types.DashboardGroupId]*types.ValidatorDashboardGroup), - } - } - if validatorDashboardConfig.DashboardsByUserId[row.UserId][row.DashboardId].Groups[row.GroupId] == nil { - validatorDashboardConfig.DashboardsByUserId[row.UserId][row.DashboardId].Groups[row.GroupId] = &types.ValidatorDashboardGroup{ - Name: row.GroupName, - Validators: []uint64{}, - } - } - validatorDashboardConfig.DashboardsByUserId[row.UserId][row.DashboardId].Groups[row.GroupId].Validators = append(validatorDashboardConfig.DashboardsByUserId[row.UserId][row.DashboardId].Groups[row.GroupId].Validators, uint64(row.ValidatorIndex)) - } + // // Now initialize the validator dashboard configuration map + // validatorDashboardConfig := &types.ValidatorDashboardConfig{ + // DashboardsByUserId: make(map[types.UserId]map[types.DashboardId]*types.ValidatorDashboard), + // } + // for _, row := range dashboardDefinitions { + // if validatorDashboardConfig.DashboardsByUserId[row.UserId] == nil { + // validatorDashboardConfig.DashboardsByUserId[row.UserId] = make(map[types.DashboardId]*types.ValidatorDashboard) + // } + // if validatorDashboardConfig.DashboardsByUserId[row.UserId][row.DashboardId] == nil { + // validatorDashboardConfig.DashboardsByUserId[row.UserId][row.DashboardId] = &types.ValidatorDashboard{ + // Name: row.DashboardName, + // Groups: make(map[types.DashboardGroupId]*types.ValidatorDashboardGroup), + // } + // } + // if validatorDashboardConfig.DashboardsByUserId[row.UserId][row.DashboardId].Groups[row.GroupId] == nil { + // validatorDashboardConfig.DashboardsByUserId[row.UserId][row.DashboardId].Groups[row.GroupId] = &types.ValidatorDashboardGroup{ + // Name: row.GroupName, + // Validators: []uint64{}, + // } + // } + // validatorDashboardConfig.DashboardsByUserId[row.UserId][row.DashboardId].Groups[row.GroupId].Validators = append(validatorDashboardConfig.DashboardsByUserId[row.UserId][row.DashboardId].Groups[row.GroupId].Validators, uint64(row.ValidatorIndex)) + // } // TODO: pass the validatorDashboardConfig to the notification collection functions // The following functions will collect the notifications and add them to the @@ -526,7 +528,7 @@ func queueNotifications(notificationsByUserID types.NotificationsPerUserId, useD for state, subs := range stateToSub { subArray := make([]int64, 0) for subID := range subs { - subArray = append(subArray, int64(subID)) + subArray = append(subArray, int64(subID)) //nolint:gosec } _, err := db.FrontendWriterDB.Exec(`UPDATE users_subscriptions SET internal_state = $1 WHERE id = ANY($2)`, state, pq.Int64Array(subArray)) if err != nil { @@ -582,10 +584,7 @@ func getNetwork() string { } func queuePushNotification(notificationsByUserID types.NotificationsPerUserId, useDB *sqlx.DB) error { - userIDs := []types.UserId{} - for userID := range notificationsByUserID { - userIDs = append(userIDs, userID) - } + userIDs := slices.Collect(maps.Keys(notificationsByUserID)) tokensByUserID, err := GetUserPushTokenByIds(userIDs) if err != nil { @@ -690,10 +689,8 @@ func sendPushNotifications(useDB *sqlx.DB) error { } func queueEmailNotifications(notificationsByUserID types.NotificationsPerUserId, useDB *sqlx.DB) error { - userIDs := []types.UserId{} - for userID := range notificationsByUserID { - userIDs = append(userIDs, userID) - } + userIDs := slices.Collect(maps.Keys(notificationsByUserID)) + emailsByUserID, err := GetUserEmailsByIds(userIDs) if err != nil { metrics.Errors.WithLabelValues("notifications_get_user_mail_by_id").Inc() @@ -1009,7 +1006,7 @@ func sendWebhookNotifications(useDB *sqlx.DB) error { go func(n types.TransitWebhook) { if n.Content.Webhook.Retries > 0 { - time.Sleep(time.Duration(n.Content.Webhook.Retries) * time.Second) + time.Sleep(time.Duration(n.Content.Webhook.Retries) * time.Second) //nolint:gosec } resp, err := client.Post(n.Content.Webhook.Url, "application/json", reqBody) if err != nil { @@ -1127,7 +1124,7 @@ func sendDiscordNotifications(useDB *sqlx.DB) error { break // stop } // sleep between retries - time.Sleep(time.Duration(webhook.Retries) * time.Second) + time.Sleep(time.Duration(webhook.Retries) * time.Second) //nolint:gosec reqBody := new(bytes.Buffer) err := json.NewEncoder(reqBody).Encode(reqs[i].Content.DiscordRequest) From 3a148a7b108216aa7abf63aee42c5f84e62821bc Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Mon, 9 Sep 2024 09:58:32 +0000 Subject: [PATCH 18/94] chore(notifications): please linter --- backend/pkg/notification/notifications.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/pkg/notification/notifications.go b/backend/pkg/notification/notifications.go index 71ec9e0eb..4b5512f86 100644 --- a/backend/pkg/notification/notifications.go +++ b/backend/pkg/notification/notifications.go @@ -528,7 +528,7 @@ func queueNotifications(notificationsByUserID types.NotificationsPerUserId, useD for state, subs := range stateToSub { subArray := make([]int64, 0) for subID := range subs { - subArray = append(subArray, int64(subID)) //nolint:gosec + subArray = append(subArray, int64(subID)) } _, err := db.FrontendWriterDB.Exec(`UPDATE users_subscriptions SET internal_state = $1 WHERE id = ANY($2)`, state, pq.Int64Array(subArray)) if err != nil { @@ -1006,7 +1006,7 @@ func sendWebhookNotifications(useDB *sqlx.DB) error { go func(n types.TransitWebhook) { if n.Content.Webhook.Retries > 0 { - time.Sleep(time.Duration(n.Content.Webhook.Retries) * time.Second) //nolint:gosec + time.Sleep(time.Duration(n.Content.Webhook.Retries) * time.Second) } resp, err := client.Post(n.Content.Webhook.Url, "application/json", reqBody) if err != nil { @@ -1124,7 +1124,7 @@ func sendDiscordNotifications(useDB *sqlx.DB) error { break // stop } // sleep between retries - time.Sleep(time.Duration(webhook.Retries) * time.Second) //nolint:gosec + time.Sleep(time.Duration(webhook.Retries) * time.Second) reqBody := new(bytes.Buffer) err := json.NewEncoder(reqBody).Encode(reqs[i].Content.DiscordRequest) From f8265c5851a59abf5a0ff668c08e522e4b4fcfd1 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Mon, 9 Sep 2024 09:59:24 +0000 Subject: [PATCH 19/94] chore(notifications): disable loading dashboard configurations --- backend/pkg/notification/notifications.go | 42 +++++++++++------------ 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/backend/pkg/notification/notifications.go b/backend/pkg/notification/notifications.go index 4b5512f86..59399f3b7 100644 --- a/backend/pkg/notification/notifications.go +++ b/backend/pkg/notification/notifications.go @@ -263,27 +263,27 @@ func collectNotifications(epoch uint64) (types.NotificationsPerUserId, error) { ValidatorIndex types.ValidatorIndex `db:"validator_index"` } - log.Infof("retrieving dashboard definitions") - // Retrieve all dashboard definitions to be able to retrieve validators included in - // the group notification subscriptions - // TODO: add a filter to retrieve only groups that have notifications enabled - // Needs a new field in the db - var dashboardDefinitions []dashboardDefinitionRow - err = db.AlloyWriter.Select(&dashboardDefinitions, ` - select - users_val_dashboards.id as dashboard_id, - users_val_dashboards.name as dashboard_name, - users_val_dashboards.user_id, - users_val_dashboards_groups.id as group_id, - users_val_dashboards_groups.name as group_name, - users_val_dashboards_validators.validator_index - from users_val_dashboards - left join users_val_dashboards_groups on users_val_dashboards_groups.dashboard_id = users_val_dashboards.id - left join users_val_dashboards_validators on users_val_dashboards_validators.dashboard_id = users_val_dashboards_groups.dashboard_id AND users_val_dashboards_validators.group_id = users_val_dashboards_groups.id; - `) - if err != nil { - return nil, fmt.Errorf("error getting dashboard definitions: %v", err) - } + // log.Infof("retrieving dashboard definitions") + // // Retrieve all dashboard definitions to be able to retrieve validators included in + // // the group notification subscriptions + // // TODO: add a filter to retrieve only groups that have notifications enabled + // // Needs a new field in the db + // var dashboardDefinitions []dashboardDefinitionRow + // err = db.AlloyWriter.Select(&dashboardDefinitions, ` + // select + // users_val_dashboards.id as dashboard_id, + // users_val_dashboards.name as dashboard_name, + // users_val_dashboards.user_id, + // users_val_dashboards_groups.id as group_id, + // users_val_dashboards_groups.name as group_name, + // users_val_dashboards_validators.validator_index + // from users_val_dashboards + // left join users_val_dashboards_groups on users_val_dashboards_groups.dashboard_id = users_val_dashboards.id + // left join users_val_dashboards_validators on users_val_dashboards_validators.dashboard_id = users_val_dashboards_groups.dashboard_id AND users_val_dashboards_validators.group_id = users_val_dashboards_groups.id; + // `) + // if err != nil { + // return nil, fmt.Errorf("error getting dashboard definitions: %v", err) + // } // // Now initialize the validator dashboard configuration map // validatorDashboardConfig := &types.ValidatorDashboardConfig{ From 23ed6e285f69603811e1085f7bfaa7485428f2aa Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Mon, 9 Sep 2024 09:59:46 +0000 Subject: [PATCH 20/94] chore(notifications): remove unused code for dashboard configurations --- backend/pkg/notification/notifications.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/backend/pkg/notification/notifications.go b/backend/pkg/notification/notifications.go index 59399f3b7..a8a6209eb 100644 --- a/backend/pkg/notification/notifications.go +++ b/backend/pkg/notification/notifications.go @@ -254,14 +254,14 @@ func collectNotifications(epoch uint64) (types.NotificationsPerUserId, error) { log.Infof("started collecting notifications") - type dashboardDefinitionRow struct { - DashboardId types.DashboardId `db:"dashboard_id"` - DashboardName string `db:"dashboard_name"` - UserId types.UserId `db:"user_id"` - GroupId types.DashboardGroupId `db:"group_id"` - GroupName string `db:"group_name"` - ValidatorIndex types.ValidatorIndex `db:"validator_index"` - } + // type dashboardDefinitionRow struct { + // DashboardId types.DashboardId `db:"dashboard_id"` + // DashboardName string `db:"dashboard_name"` + // UserId types.UserId `db:"user_id"` + // GroupId types.DashboardGroupId `db:"group_id"` + // GroupName string `db:"group_name"` + // ValidatorIndex types.ValidatorIndex `db:"validator_index"` + // } // log.Infof("retrieving dashboard definitions") // // Retrieve all dashboard definitions to be able to retrieve validators included in From c8d0e0f993ebc902d9d433c1a9f7e7c68feb83be Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Mon, 9 Sep 2024 12:14:42 +0000 Subject: [PATCH 21/94] feat(notifications): store queued notifications in network db --- backend/pkg/commons/db/subscriptions.go | 4 +- backend/pkg/notification/notifications.go | 89 +++++++++++------------ 2 files changed, 46 insertions(+), 47 deletions(-) diff --git a/backend/pkg/commons/db/subscriptions.go b/backend/pkg/commons/db/subscriptions.go index 3edbc3d5f..639f70c8b 100644 --- a/backend/pkg/commons/db/subscriptions.go +++ b/backend/pkg/commons/db/subscriptions.go @@ -261,8 +261,8 @@ func GetSubscriptions(filter GetSubscriptionsFilter) ([]*types.Subscription, err } // UpdateSubscriptionsLastSent updates `last_sent_ts` column of the `users_subscriptions` table. -func UpdateSubscriptionsLastSent(subscriptionIDs []uint64, sent time.Time, epoch uint64, useDB *sqlx.DB) error { - _, err := useDB.Exec(` +func UpdateSubscriptionsLastSent(subscriptionIDs []uint64, sent time.Time, epoch uint64) error { + _, err := FrontendWriterDB.Exec(` UPDATE users_subscriptions SET last_sent_ts = TO_TIMESTAMP($1), last_sent_epoch = $2 WHERE id = ANY($3)`, sent.Unix(), epoch, pq.Array(subscriptionIDs)) diff --git a/backend/pkg/notification/notifications.go b/backend/pkg/notification/notifications.go index a8a6209eb..7b148d0d5 100644 --- a/backend/pkg/notification/notifications.go +++ b/backend/pkg/notification/notifications.go @@ -34,7 +34,6 @@ import ( "github.com/gobitfly/beaconchain/pkg/commons/types" "github.com/gobitfly/beaconchain/pkg/commons/utils" - "github.com/jmoiron/sqlx" "github.com/lib/pq" "github.com/rocket-pool/rocketpool-go/utils/eth" "golang.org/x/text/cases" @@ -126,7 +125,7 @@ func notificationCollector() { break } - queueNotifications(notifications, db.FrontendWriterDB) // this caused the collected notifications to be queued and sent + queueNotifications(notifications) // this caused the collected notifications to be queued and sent // Network DB Notifications (user related, must only run on one instance ever!!!!) if utils.Config.Notifications.UserDBNotifications { @@ -139,7 +138,7 @@ func notificationCollector() { continue } - queueNotifications(userNotifications, db.FrontendWriterDB) + queueNotifications(userNotifications) } log.InfoWithFields(log.Fields{"notifications": len(notifications), "duration": time.Since(start), "epoch": epoch}, "notifications completed") @@ -177,12 +176,12 @@ func notificationSender() { } log.Infof("lock obtained") - err = dispatchNotifications(db.FrontendWriterDB) + err = dispatchNotifications() if err != nil { log.Error(err, "error dispatching notifications", 0) } - err = garbageCollectNotificationQueue(db.FrontendWriterDB) + err = garbageCollectNotificationQueue() if err != nil { log.Error(err, "error garbage collecting notification queue", 0) } @@ -464,20 +463,20 @@ func collectUserDbNotifications(epoch uint64) (types.NotificationsPerUserId, err return notificationsByUserID, nil } -func queueNotifications(notificationsByUserID types.NotificationsPerUserId, useDB *sqlx.DB) { +func queueNotifications(notificationsByUserID types.NotificationsPerUserId) { subByEpoch := map[uint64][]uint64{} - err := queueEmailNotifications(notificationsByUserID, useDB) + err := queueEmailNotifications(notificationsByUserID) if err != nil { log.Error(err, "error queuing email notifications", 0) } - err = queuePushNotification(notificationsByUserID, useDB) + err = queuePushNotification(notificationsByUserID) if err != nil { log.Error(err, "error queuing push notifications", 0) } - err = queueWebhookNotifications(notificationsByUserID, useDB) + err = queueWebhookNotifications(notificationsByUserID) if err != nil { log.Error(err, "error queuing webhook notifications", 0) } @@ -498,7 +497,7 @@ func queueNotifications(notificationsByUserID types.NotificationsPerUserId, useD // obsolete as notifications are anyway sent on a per-epoch basis for epoch, subIDs := range subByEpoch { // update that we've queued the subscription (last sent rather means last queued) - err := db.UpdateSubscriptionsLastSent(subIDs, time.Now(), epoch, useDB) + err := db.UpdateSubscriptionsLastSent(subIDs, time.Now(), epoch) if err != nil { log.Error(err, "error updating sent-time of sent notifications", 0) metrics.Errors.WithLabelValues("notifications_updating_sent_time").Inc() @@ -537,23 +536,23 @@ func queueNotifications(notificationsByUserID types.NotificationsPerUserId, useD } } -func dispatchNotifications(useDB *sqlx.DB) error { - err := sendEmailNotifications(useDB) +func dispatchNotifications() error { + err := sendEmailNotifications() if err != nil { return fmt.Errorf("error sending email notifications, err: %w", err) } - err = sendPushNotifications(useDB) + err = sendPushNotifications() if err != nil { return fmt.Errorf("error sending push notifications, err: %w", err) } - err = sendWebhookNotifications(useDB) + err = sendWebhookNotifications() if err != nil { return fmt.Errorf("error sending webhook notifications, err: %w", err) } - err = sendDiscordNotifications(useDB) + err = sendDiscordNotifications() if err != nil { return fmt.Errorf("error sending webhook discord notifications, err: %w", err) } @@ -562,8 +561,8 @@ func dispatchNotifications(useDB *sqlx.DB) error { } // garbageCollectNotificationQueue deletes entries from the notification queue that have been processed -func garbageCollectNotificationQueue(useDB *sqlx.DB) error { - rows, err := useDB.Exec(`DELETE FROM notification_queue WHERE (sent < now() - INTERVAL '30 minutes') OR (created < now() - INTERVAL '1 hour')`) +func garbageCollectNotificationQueue() error { + rows, err := db.WriterDb.Exec(`DELETE FROM notification_queue WHERE (sent < now() - INTERVAL '30 minutes') OR (created < now() - INTERVAL '1 hour')`) if err != nil { return fmt.Errorf("error deleting from notification_queue %w", err) } @@ -583,7 +582,7 @@ func getNetwork() string { return "" } -func queuePushNotification(notificationsByUserID types.NotificationsPerUserId, useDB *sqlx.DB) error { +func queuePushNotification(notificationsByUserID types.NotificationsPerUserId) error { userIDs := slices.Collect(maps.Keys(notificationsByUserID)) tokensByUserID, err := GetUserPushTokenByIds(userIDs) @@ -636,7 +635,7 @@ func queuePushNotification(notificationsByUserID types.NotificationsPerUserId, u Messages: batch, } - _, err = useDB.Exec(`INSERT INTO notification_queue (created, channel, content) VALUES ($1, 'push', $2)`, time.Now(), transitPushContent) + _, err = db.WriterDb.Exec(`INSERT INTO notification_queue (created, channel, content) VALUES ($1, 'push', $2)`, time.Now(), transitPushContent) if err != nil { log.Error(err, "error writing transit push notification to db", 0) return @@ -646,10 +645,10 @@ func queuePushNotification(notificationsByUserID types.NotificationsPerUserId, u return nil } -func sendPushNotifications(useDB *sqlx.DB) error { +func sendPushNotifications() error { var notificationQueueItem []types.TransitPush - err := useDB.Select(¬ificationQueueItem, `SELECT + err := db.WriterDb.Select(¬ificationQueueItem, `SELECT id, created, sent, @@ -679,7 +678,7 @@ func sendPushNotifications(useDB *sqlx.DB) error { metrics.NotificationsSent.WithLabelValues("push", "200").Add(float64(len(n.Content.Messages))) } - _, err = useDB.Exec(`UPDATE notification_queue SET sent = now() WHERE id = $1`, n.Id) + _, err = db.WriterDb.Exec(`UPDATE notification_queue SET sent = now() WHERE id = $1`, n.Id) if err != nil { return fmt.Errorf("error updating sent status for push notification with id: %v, err: %w", n.Id, err) } @@ -688,7 +687,7 @@ func sendPushNotifications(useDB *sqlx.DB) error { return nil } -func queueEmailNotifications(notificationsByUserID types.NotificationsPerUserId, useDB *sqlx.DB) error { +func queueEmailNotifications(notificationsByUserID types.NotificationsPerUserId) error { userIDs := slices.Collect(maps.Keys(notificationsByUserID)) emailsByUserID, err := GetUserEmailsByIds(userIDs) @@ -777,7 +776,7 @@ func queueEmailNotifications(notificationsByUserID types.NotificationsPerUserId, Attachments: attachments, } - _, err = useDB.Exec(`INSERT INTO notification_queue (created, channel, content) VALUES ($1, 'email', $2)`, time.Now(), transitEmailContent) + _, err = db.WriterDb.Exec(`INSERT INTO notification_queue (created, channel, content) VALUES ($1, 'email', $2)`, time.Now(), transitEmailContent) if err != nil { log.Error(err, "error writing transit email to db", 0) } @@ -786,10 +785,10 @@ func queueEmailNotifications(notificationsByUserID types.NotificationsPerUserId, return nil } -func sendEmailNotifications(useDb *sqlx.DB) error { +func sendEmailNotifications() error { var notificationQueueItem []types.TransitEmail - err := useDb.Select(¬ificationQueueItem, `SELECT + err := db.WriterDb.Select(¬ificationQueueItem, `SELECT id, created, sent, @@ -812,7 +811,7 @@ func sendEmailNotifications(useDb *sqlx.DB) error { metrics.NotificationsSent.WithLabelValues("email", "200").Inc() } } - _, err = useDb.Exec(`UPDATE notification_queue set sent = now() where id = $1`, n.Id) + _, err = db.WriterDb.Exec(`UPDATE notification_queue set sent = now() where id = $1`, n.Id) if err != nil { return fmt.Errorf("error updating sent status for email notification with id: %v, err: %w", n.Id, err) } @@ -820,10 +819,10 @@ func sendEmailNotifications(useDb *sqlx.DB) error { return nil } -func queueWebhookNotifications(notificationsByUserID types.NotificationsPerUserId, useDB *sqlx.DB) error { +func queueWebhookNotifications(notificationsByUserID types.NotificationsPerUserId) error { for userID, userNotifications := range notificationsByUserID { var webhooks []types.UserWebhook - err := useDB.Select(&webhooks, ` + err := db.FrontendWriterDB.Select(&webhooks, ` SELECT id, user_id, @@ -861,7 +860,7 @@ func queueWebhookNotifications(notificationsByUserID types.NotificationsPerUserI if len(notifications) > 0 { // reset Retries if w.Retries > 5 && w.LastSent.Valid && w.LastSent.Time.Add(time.Hour).Before(time.Now()) { - _, err = useDB.Exec(`UPDATE users_webhooks SET retries = 0 WHERE id = $1;`, w.ID) + _, err = db.FrontendWriterDB.Exec(`UPDATE users_webhooks SET retries = 0 WHERE id = $1;`, w.ID) if err != nil { log.Error(err, "error updating users_webhooks table; setting retries to zero", 0) continue @@ -938,7 +937,7 @@ func queueWebhookNotifications(notificationsByUserID types.NotificationsPerUserI } // process notifs for _, n := range notifs { - _, err = useDB.Exec(`INSERT INTO notification_queue (created, channel, content) VALUES (now(), $1, $2);`, n.Channel, n.Content) + _, err = db.WriterDb.Exec(`INSERT INTO notification_queue (created, channel, content) VALUES (now(), $1, $2);`, n.Channel, n.Content) if err != nil { log.Error(err, "error inserting into webhooks_queue", 0) } else { @@ -948,7 +947,7 @@ func queueWebhookNotifications(notificationsByUserID types.NotificationsPerUserI // process discord notifs for _, dNotifs := range discordNotifMap { for _, n := range dNotifs { - _, err = useDB.Exec(`INSERT INTO notification_queue (created, channel, content) VALUES (now(), 'webhook_discord', $1);`, n) + _, err = db.WriterDb.Exec(`INSERT INTO notification_queue (created, channel, content) VALUES (now(), 'webhook_discord', $1);`, n) if err != nil { log.Error(err, "error inserting into webhooks_queue (discord)", 0) continue @@ -961,10 +960,10 @@ func queueWebhookNotifications(notificationsByUserID types.NotificationsPerUserI return nil } -func sendWebhookNotifications(useDB *sqlx.DB) error { +func sendWebhookNotifications() error { var notificationQueueItem []types.TransitWebhook - err := useDB.Select(¬ificationQueueItem, `SELECT + err := db.WriterDb.Select(¬ificationQueueItem, `SELECT id, created, sent, @@ -981,7 +980,7 @@ func sendWebhookNotifications(useDB *sqlx.DB) error { for _, n := range notificationQueueItem { // do not retry after 5 attempts if n.Content.Webhook.Retries > 5 { - _, err := db.FrontendWriterDB.Exec(`DELETE FROM notification_queue WHERE id = $1`, n.Id) + _, err := db.WriterDb.Exec(`DELETE FROM notification_queue WHERE id = $1`, n.Id) if err != nil { return fmt.Errorf("error deleting from notification queue: %w", err) } @@ -997,7 +996,7 @@ func sendWebhookNotifications(useDB *sqlx.DB) error { _, err = url.Parse(n.Content.Webhook.Url) if err != nil { - _, err := db.FrontendWriterDB.Exec(`DELETE FROM notification_queue WHERE id = $1`, n.Id) + _, err := db.WriterDb.Exec(`DELETE FROM notification_queue WHERE id = $1`, n.Id) if err != nil { return fmt.Errorf("error deleting from notification queue: %w", err) } @@ -1017,14 +1016,14 @@ func sendWebhookNotifications(useDB *sqlx.DB) error { } defer resp.Body.Close() - _, err = useDB.Exec(`UPDATE notification_queue SET sent = now() WHERE id = $1`, n.Id) + _, err = db.WriterDb.Exec(`UPDATE notification_queue SET sent = now() WHERE id = $1`, n.Id) if err != nil { log.Error(err, "error updating notification_queue table", 0) return } if resp != nil && resp.StatusCode < 400 { - _, err = useDB.Exec(`UPDATE users_webhooks SET retries = 0, last_sent = now() WHERE id = $1;`, n.Content.Webhook.ID) + _, err = db.FrontendWriterDB.Exec(`UPDATE users_webhooks SET retries = 0, last_sent = now() WHERE id = $1;`, n.Content.Webhook.ID) if err != nil { log.Error(err, "error updating users_webhooks table", 0) return @@ -1042,7 +1041,7 @@ func sendWebhookNotifications(useDB *sqlx.DB) error { errResp.Body = string(b) } - _, err = useDB.Exec(`UPDATE users_webhooks SET retries = retries + 1, last_sent = now(), request = $2, response = $3 WHERE id = $1;`, n.Content.Webhook.ID, n.Content, errResp) + _, err = db.FrontendWriterDB.Exec(`UPDATE users_webhooks SET retries = retries + 1, last_sent = now(), request = $2, response = $3 WHERE id = $1;`, n.Content.Webhook.ID, n.Content, errResp) if err != nil { log.Error(err, "error updating users_webhooks table", 0) return @@ -1053,10 +1052,10 @@ func sendWebhookNotifications(useDB *sqlx.DB) error { return nil } -func sendDiscordNotifications(useDB *sqlx.DB) error { +func sendDiscordNotifications() error { var notificationQueueItem []types.TransitDiscord - err := useDB.Select(¬ificationQueueItem, `SELECT + err := db.WriterDb.Select(¬ificationQueueItem, `SELECT id, created, sent, @@ -1077,7 +1076,7 @@ func sendDiscordNotifications(useDB *sqlx.DB) error { for _, n := range notificationQueueItem { // purge the event from existence if the retry counter is over 5 if n.Content.Webhook.Retries > 5 { - _, err = db.FrontendWriterDB.Exec(`DELETE FROM notification_queue where id = $1`, n.Id) + _, err = db.WriterDb.Exec(`DELETE FROM notification_queue where id = $1`, n.Id) if err != nil { log.Warnf("failed to delete notification from queue: %v", err) } @@ -1097,7 +1096,7 @@ func sendDiscordNotifications(useDB *sqlx.DB) error { go func(webhook types.UserWebhook, reqs []types.TransitDiscord) { defer func() { // update retries counters in db based on end result - _, err = useDB.Exec(`UPDATE users_webhooks SET retries = $1, last_sent = now() WHERE id = $2;`, webhook.Retries, webhook.ID) + _, err = db.FrontendWriterDB.Exec(`UPDATE users_webhooks SET retries = $1, last_sent = now() WHERE id = $2;`, webhook.Retries, webhook.ID) if err != nil { log.Warnf("failed to update retries counter to %v for webhook %v: %v", webhook.Retries, webhook.ID, err) } @@ -1107,7 +1106,7 @@ func sendDiscordNotifications(useDB *sqlx.DB) error { for _, req := range reqs { ids = append(ids, req.Id) } - _, err = db.FrontendWriterDB.Exec(`UPDATE notification_queue SET sent = now() where id = ANY($1)`, pq.Array(ids)) + _, err = db.WriterDb.Exec(`UPDATE notification_queue SET sent = now() where id = ANY($1)`, pq.Array(ids)) if err != nil { log.Warnf("failed to update sent for notifcations in queue: %v", err) } @@ -1161,7 +1160,7 @@ func sendDiscordNotifications(useDB *sqlx.DB) error { } else { log.Error(nil, "error pushing discord webhook", 0, map[string]interface{}{"errResp.Body": errResp.Body, "webhook.Url": webhook.Url}) } - _, err = useDB.Exec(`UPDATE users_webhooks SET request = $2, response = $3 WHERE id = $1;`, webhook.ID, reqs[i].Content.DiscordRequest, errResp) + _, err = db.FrontendWriterDB.Exec(`UPDATE users_webhooks SET request = $2, response = $3 WHERE id = $1;`, webhook.ID, reqs[i].Content.DiscordRequest, errResp) if err != nil { log.Error(err, "error storing failure data in users_webhooks table", 0) } From 86da74c466e8e61e90e1b3e1bf7cc7744cc2963a Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Tue, 10 Sep 2024 10:27:46 +0000 Subject: [PATCH 22/94] fix(notifications): expand args in sql query --- backend/pkg/notification/db.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/pkg/notification/db.go b/backend/pkg/notification/db.go index 4363c12f7..6e19acf3b 100644 --- a/backend/pkg/notification/db.go +++ b/backend/pkg/notification/db.go @@ -58,7 +58,7 @@ func GetSubsForEventFilter(eventName types.EventName, lastSentFilter string, las } subMap := make(map[string][]types.Subscription, 0) - err = db.FrontendWriterDB.Select(&subs, query, args) + err = db.FrontendWriterDB.Select(&subs, query, args...) if err != nil { return nil, err } @@ -69,6 +69,7 @@ func GetSubsForEventFilter(eventName types.EventName, lastSentFilter string, las } subMap[sub.EventFilter] = append(subMap[sub.EventFilter], sub) } + return subMap, nil } From 454534a2df918840e113c46b120e8f38a8e6c9d0 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Tue, 10 Sep 2024 11:09:27 +0000 Subject: [PATCH 23/94] chore(notifications): add some util funcs --- backend/cmd/misc/main.go | 60 ++++++++++++++--------- backend/pkg/commons/types/frontend.go | 4 +- backend/pkg/notification/db.go | 6 ++- backend/pkg/notification/notifications.go | 16 ++++-- 4 files changed, 57 insertions(+), 29 deletions(-) diff --git a/backend/cmd/misc/main.go b/backend/cmd/misc/main.go index d7b9b77a1..8af9a3499 100644 --- a/backend/cmd/misc/main.go +++ b/backend/cmd/misc/main.go @@ -32,6 +32,7 @@ import ( edb "github.com/gobitfly/beaconchain/pkg/exporter/db" "github.com/gobitfly/beaconchain/pkg/exporter/modules" "github.com/gobitfly/beaconchain/pkg/exporter/services" + "github.com/gobitfly/beaconchain/pkg/notification" _ "github.com/jackc/pgx/v5/stdlib" "github.com/pkg/errors" utilMath "github.com/protolambda/zrnt/eth2/util/math" @@ -75,7 +76,7 @@ func Run() { } configPath := fs.String("config", "config/default.config.yml", "Path to the config file") - fs.StringVar(&opts.Command, "command", "", "command to run, available: updateAPIKey, applyDbSchema, initBigtableSchema, epoch-export, debug-rewards, debug-blocks, clear-bigtable, index-old-eth1-blocks, update-aggregation-bits, historic-prices-export, index-missing-blocks, export-epoch-missed-slots, migrate-last-attestation-slot-bigtable, export-genesis-validators, update-block-finalization-sequentially, nameValidatorsByRanges, export-stats-totals, export-sync-committee-periods, export-sync-committee-validator-stats, partition-validator-stats, migrate-app-purchases") + fs.StringVar(&opts.Command, "command", "", "command to run, available: updateAPIKey, applyDbSchema, initBigtableSchema, epoch-export, debug-rewards, debug-blocks, clear-bigtable, index-old-eth1-blocks, update-aggregation-bits, historic-prices-export, index-missing-blocks, export-epoch-missed-slots, migrate-last-attestation-slot-bigtable, export-genesis-validators, update-block-finalization-sequentially, nameValidatorsByRanges, export-stats-totals, export-sync-committee-periods, export-sync-committee-validator-stats, partition-validator-stats, migrate-app-purchases, collect-notifications") fs.Uint64Var(&opts.StartEpoch, "start-epoch", 0, "start epoch") fs.Uint64Var(&opts.EndEpoch, "end-epoch", 0, "end epoch") fs.Uint64Var(&opts.User, "user", 0, "user id") @@ -181,27 +182,27 @@ func Run() { defer db.FrontendWriterDB.Close() // clickhouse - db.ClickHouseWriter, db.ClickHouseReader = db.MustInitDB(&types.DatabaseConfig{ - Username: cfg.ClickHouse.WriterDatabase.Username, - Password: cfg.ClickHouse.WriterDatabase.Password, - Name: cfg.ClickHouse.WriterDatabase.Name, - Host: cfg.ClickHouse.WriterDatabase.Host, - Port: cfg.ClickHouse.WriterDatabase.Port, - MaxOpenConns: cfg.ClickHouse.WriterDatabase.MaxOpenConns, - SSL: true, - MaxIdleConns: cfg.ClickHouse.WriterDatabase.MaxIdleConns, - }, &types.DatabaseConfig{ - Username: cfg.ClickHouse.ReaderDatabase.Username, - Password: cfg.ClickHouse.ReaderDatabase.Password, - Name: cfg.ClickHouse.ReaderDatabase.Name, - Host: cfg.ClickHouse.ReaderDatabase.Host, - Port: cfg.ClickHouse.ReaderDatabase.Port, - MaxOpenConns: cfg.ClickHouse.ReaderDatabase.MaxOpenConns, - SSL: true, - MaxIdleConns: cfg.ClickHouse.ReaderDatabase.MaxIdleConns, - }, "clickhouse", "clickhouse") - defer db.ClickHouseReader.Close() - defer db.ClickHouseWriter.Close() + // db.ClickHouseWriter, db.ClickHouseReader = db.MustInitDB(&types.DatabaseConfig{ + // Username: cfg.ClickHouse.WriterDatabase.Username, + // Password: cfg.ClickHouse.WriterDatabase.Password, + // Name: cfg.ClickHouse.WriterDatabase.Name, + // Host: cfg.ClickHouse.WriterDatabase.Host, + // Port: cfg.ClickHouse.WriterDatabase.Port, + // MaxOpenConns: cfg.ClickHouse.WriterDatabase.MaxOpenConns, + // SSL: true, + // MaxIdleConns: cfg.ClickHouse.WriterDatabase.MaxIdleConns, + // }, &types.DatabaseConfig{ + // Username: cfg.ClickHouse.ReaderDatabase.Username, + // Password: cfg.ClickHouse.ReaderDatabase.Password, + // Name: cfg.ClickHouse.ReaderDatabase.Name, + // Host: cfg.ClickHouse.ReaderDatabase.Host, + // Port: cfg.ClickHouse.ReaderDatabase.Port, + // MaxOpenConns: cfg.ClickHouse.ReaderDatabase.MaxOpenConns, + // SSL: true, + // MaxIdleConns: cfg.ClickHouse.ReaderDatabase.MaxIdleConns, + // }, "clickhouse", "clickhouse") + // defer db.ClickHouseReader.Close() + // defer db.ClickHouseWriter.Close() // Initialize the persistent redis client rdc := redis.NewClient(&redis.Options{ @@ -456,6 +457,8 @@ func Run() { err = fixEns(erigonClient) case "fix-ens-addresses": err = fixEnsAddresses(erigonClient) + case "collect-notifications": + err = collectNotifications(opts.StartEpoch, opts.EndEpoch) default: log.Fatal(nil, fmt.Sprintf("unknown command %s", opts.Command), 0) } @@ -467,6 +470,19 @@ func Run() { } } +func collectNotifications(startEpoch, endEpoch uint64) error { + epoch := startEpoch + + log.Infof("collecting notifications for epoch %v", epoch) + notifications, err := notification.GetNotificationsForEpoch(utils.Config.Notifications.PubkeyCachePath, epoch) + if err != nil { + return err + } + + log.Infof("found %v notifications for epoch %v", len(notifications), epoch) + return nil +} + func fixEns(erigonClient *rpc.ErigonClient) error { log.Infof("command: fix-ens") addrs := []struct { diff --git a/backend/pkg/commons/types/frontend.go b/backend/pkg/commons/types/frontend.go index 767aca38b..eb1c04472 100644 --- a/backend/pkg/commons/types/frontend.go +++ b/backend/pkg/commons/types/frontend.go @@ -28,10 +28,10 @@ type NotificationsPerUserId map[UserId]map[EventName]map[EventFilter]Notificatio func (npui NotificationsPerUserId) AddNotification(n Notification) { if n.GetUserId() == 0 { - log.Fatal(fmt.Errorf("Notification user id is 0"), fmt.Sprintf("Notification: %v", n), 0) + log.Fatal(fmt.Errorf("Notification user id is 0"), fmt.Sprintf("Notification: %v", n), 1) } if n.GetEventName() == "" { - log.Fatal(fmt.Errorf("Notification event name is empty"), fmt.Sprintf("Notification: %v", n), 0) + log.Fatal(fmt.Errorf("Notification event name is empty"), fmt.Sprintf("Notification: %v", n), 1) } // next check is disabled as there are events that do not require a filter (rocketpool, network events) // if n.GetEventFilter() == "" { diff --git a/backend/pkg/notification/db.go b/backend/pkg/notification/db.go index 6e19acf3b..1f999a411 100644 --- a/backend/pkg/notification/db.go +++ b/backend/pkg/notification/db.go @@ -3,6 +3,7 @@ package notification import ( "github.com/doug-martin/goqu/v9" "github.com/gobitfly/beaconchain/pkg/commons/db" + "github.com/gobitfly/beaconchain/pkg/commons/log" "github.com/gobitfly/beaconchain/pkg/commons/types" "github.com/gobitfly/beaconchain/pkg/commons/utils" "github.com/lib/pq" @@ -39,7 +40,8 @@ func GetSubsForEventFilter(eventName types.EventName, lastSentFilter string, las goqu.C("last_sent_epoch"), goqu.C("created_epoch"), goqu.C("event_threshold"), - ).Where(goqu.C("event_name").Eq(utils.GetNetwork() + ":" + string(eventName))) + goqu.C("event_name"), + ).Where(goqu.L("(event_name = ? AND user_id <> 0)", utils.GetNetwork()+":"+string(eventName))) if lastSentFilter != "" { if len(lastSentFilterArgs) > 0 { @@ -63,6 +65,8 @@ func GetSubsForEventFilter(eventName types.EventName, lastSentFilter string, las return nil, err } + log.Infof("Found %d subscriptions for event %s", len(subs), eventName) + for _, sub := range subs { if _, ok := subMap[sub.EventFilter]; !ok { subMap[sub.EventFilter] = make([]types.Subscription, 0) diff --git a/backend/pkg/notification/notifications.go b/backend/pkg/notification/notifications.go index 7b148d0d5..d7cb84e97 100644 --- a/backend/pkg/notification/notifications.go +++ b/backend/pkg/notification/notifications.go @@ -45,6 +45,14 @@ func InitNotificationSender() { go notificationSender() } +func GetNotificationsForEpoch(pubkeyCachePath string, epoch uint64) (types.NotificationsPerUserId, error) { + err := initPubkeyCache(pubkeyCachePath) + if err != nil { + log.Fatal(err, "error initializing pubkey cache path for notifications", 0) + } + return collectNotifications(epoch) +} + func InitNotificationCollector(pubkeyCachePath string) { err := initPubkeyCache(pubkeyCachePath) if err != nil { @@ -1739,8 +1747,8 @@ func collectValidatorGotSlashedNotifications(notificationsByUserID types.Notific for _, subs := range subscribedUsers { for _, sub := range subs { event := pubkeyToSlashingInfoMap[sub.EventFilter] - if event == nil { - log.Error(fmt.Errorf("error retrieving slashing info for public key %s", sub.EventFilter), "", 0) + if event == nil { // pubkey has not been slashed + //log.Error(fmt.Errorf("error retrieving slashing info for public key %s", sub.EventFilter), "", 0) continue } log.Infof("creating %v notification for validator %v in epoch %v", event.Reason, sub.EventFilter, epoch) @@ -2386,7 +2394,7 @@ func collectNetworkNotifications(notificationsByUserID types.NotificationsPerUse dbResult, err := GetSubsForEventFilter( types.NetworkLivenessIncreasedEventName, - "us.last_sent_ts <= NOW() - INTERVAL '1 hour' OR us.last_sent_ts IS NULL", + "(last_sent_ts <= NOW() - INTERVAL '1 hour' OR last_sent_ts IS NULL)", nil, nil, ) @@ -2760,7 +2768,7 @@ func collectSyncCommittee(notificationsByUserID types.NotificationsPerUserId, ep pubKeys = append(pubKeys, val.PubKey) } - dbResult, err := GetSubsForEventFilter(types.SyncCommitteeSoon, "us.last_sent_ts <= NOW() - INTERVAL '26 hours' OR us.last_sent_ts IS NULL", nil, pubKeys) + dbResult, err := GetSubsForEventFilter(types.SyncCommitteeSoon, "(last_sent_ts <= NOW() - INTERVAL '26 hours' OR last_sent_ts IS NULL)", nil, pubKeys) // err = db.FrontendWriterDB.Select(&dbResult, ` // SELECT us.id, us.user_id, us.event_filter, ENCODE(us.unsubscribe_hash, 'hex') as unsubscribe_hash // FROM users_subscriptions AS us From d8282b1130aaf0251e5bab103767e2066a3df80a Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Tue, 10 Sep 2024 11:13:44 +0000 Subject: [PATCH 24/94] chore(notifications): please linter --- backend/cmd/misc/main.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/cmd/misc/main.go b/backend/cmd/misc/main.go index 8af9a3499..fd77d8217 100644 --- a/backend/cmd/misc/main.go +++ b/backend/cmd/misc/main.go @@ -458,7 +458,7 @@ func Run() { case "fix-ens-addresses": err = fixEnsAddresses(erigonClient) case "collect-notifications": - err = collectNotifications(opts.StartEpoch, opts.EndEpoch) + err = collectNotifications(opts.StartEpoch) default: log.Fatal(nil, fmt.Sprintf("unknown command %s", opts.Command), 0) } @@ -470,7 +470,7 @@ func Run() { } } -func collectNotifications(startEpoch, endEpoch uint64) error { +func collectNotifications(startEpoch uint64) error { epoch := startEpoch log.Infof("collecting notifications for epoch %v", epoch) From 4467399f158602a575230cd39f294448c9beda37 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Tue, 10 Sep 2024 11:31:13 +0000 Subject: [PATCH 25/94] chore(notification): update firebase sdk --- backend/go.mod | 49 +++++----- backend/go.sum | 109 ++++++++++++---------- backend/pkg/notification/firebase.go | 30 +++--- backend/pkg/notification/notifications.go | 2 +- 4 files changed, 104 insertions(+), 86 deletions(-) diff --git a/backend/go.mod b/backend/go.mod index b53006df9..25788bb93 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -6,6 +6,7 @@ require ( cloud.google.com/go/bigtable v1.21.0 cloud.google.com/go/secretmanager v1.11.5 firebase.google.com/go v3.13.0+incompatible + firebase.google.com/go/v4 v4.14.1 github.com/ClickHouse/clickhouse-go/v2 v2.17.1 github.com/Gurpartap/storekit-go v0.0.0-20201205024111-36b6cd5c6a21 github.com/alexedwards/scs/redisstore v0.0.0-20240316134038-7e11d57e8885 @@ -29,7 +30,7 @@ require ( github.com/gobitfly/eth.store v0.0.0-20240312111708-b43f13990280 github.com/golang-jwt/jwt v3.2.2+incompatible github.com/golang-jwt/jwt/v4 v4.5.0 - github.com/golang/protobuf v1.5.3 + github.com/golang/protobuf v1.5.4 github.com/gomodule/redigo v1.9.2 github.com/google/uuid v1.6.0 github.com/gorilla/csrf v1.7.2 @@ -69,26 +70,27 @@ require ( github.com/wealdtech/go-eth2-types/v2 v2.8.2 github.com/wealdtech/go-eth2-util v1.8.0 github.com/xeipuuv/gojsonschema v1.2.0 - golang.org/x/crypto v0.19.0 + golang.org/x/crypto v0.21.0 golang.org/x/exp v0.0.0-20240213143201-ec583247a57a golang.org/x/sync v0.6.0 golang.org/x/text v0.14.0 golang.org/x/time v0.5.0 golang.org/x/tools v0.18.0 - google.golang.org/api v0.164.0 - google.golang.org/protobuf v1.32.0 + google.golang.org/api v0.170.0 + google.golang.org/protobuf v1.33.0 gopkg.in/yaml.v2 v2.4.0 ) require ( - cloud.google.com/go v0.112.0 // indirect - cloud.google.com/go/compute v1.23.3 // indirect + cloud.google.com/go v0.112.1 // indirect + cloud.google.com/go/compute v1.24.0 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/firestore v1.14.0 // indirect - cloud.google.com/go/iam v1.1.5 // indirect - cloud.google.com/go/longrunning v0.5.4 // indirect - cloud.google.com/go/storage v1.36.0 // indirect + cloud.google.com/go/firestore v1.15.0 // indirect + cloud.google.com/go/iam v1.1.7 // indirect + cloud.google.com/go/longrunning v0.5.5 // indirect + cloud.google.com/go/storage v1.40.0 // indirect github.com/ClickHouse/ch-go v0.58.2 // indirect + github.com/MicahParks/keyfunc v1.9.0 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/TylerBrock/colorjson v0.0.0-20200706003622-8a50f05110d2 // indirect github.com/ajg/form v1.5.1 // indirect @@ -149,7 +151,7 @@ require ( github.com/google/go-querystring v1.1.0 // indirect github.com/google/s2a-go v0.1.7 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect - github.com/googleapis/gax-go/v2 v2.12.0 // indirect + github.com/googleapis/gax-go/v2 v2.12.3 // indirect github.com/gorilla/securecookie v1.1.2 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 // indirect github.com/herumi/bls-eth-go-binary v1.31.0 // indirect @@ -241,24 +243,25 @@ require ( github.com/yudai/golcs v0.0.0-20170316035057-ecda9a501e82 // indirect github.com/yusufpapurcu/wmi v1.2.3 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 // indirect - go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 // indirect - go.opentelemetry.io/otel v1.23.0 // indirect - go.opentelemetry.io/otel/metric v1.23.0 // indirect - go.opentelemetry.io/otel/trace v1.23.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect + go.opentelemetry.io/otel v1.24.0 // indirect + go.opentelemetry.io/otel/metric v1.24.0 // indirect + go.opentelemetry.io/otel/trace v1.24.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.27.0 // indirect golang.org/x/mod v0.15.0 // indirect - golang.org/x/net v0.21.0 // indirect - golang.org/x/oauth2 v0.17.0 // indirect - golang.org/x/sys v0.17.0 // indirect + golang.org/x/net v0.23.0 // indirect + golang.org/x/oauth2 v0.18.0 // indirect + golang.org/x/sys v0.18.0 // indirect golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect google.golang.org/appengine v1.6.8 // indirect - google.golang.org/genproto v0.0.0-20240125205218-1f4bbc51befe // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240125205218-1f4bbc51befe // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240205150955-31a09d347014 // indirect - google.golang.org/grpc v1.62.0 // indirect + google.golang.org/appengine/v2 v2.0.2 // indirect + google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240311132316-a219d84964c2 // indirect + google.golang.org/grpc v1.62.1 // indirect gopkg.in/cenkalti/backoff.v1 v1.1.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.2.1 // indirect diff --git a/backend/go.sum b/backend/go.sum index 7d41b5693..a15806cb0 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -1,26 +1,28 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= -cloud.google.com/go v0.112.0 h1:tpFCD7hpHFlQ8yPwT3x+QeXqc2T6+n6T+hmABHfDUSM= -cloud.google.com/go v0.112.0/go.mod h1:3jEEVwZ/MHU4djK5t5RHuKOA/GbLddgTdVubX1qnPD4= +cloud.google.com/go v0.112.1 h1:uJSeirPke5UNZHIb4SxfZklVSiWWVqW4oXlETwZziwM= +cloud.google.com/go v0.112.1/go.mod h1:+Vbu+Y1UU+I1rjmzeMOb/8RfkKJK2Gyxi1X6jJCZLo4= cloud.google.com/go/bigtable v1.21.0 h1:BFN4jhkA9ULYYV2Ug7AeOtetVLnN2jKuIq5TcRc5C38= cloud.google.com/go/bigtable v1.21.0/go.mod h1:V0sYNRtk0dgAKjyRr/MyBpHpSXqh+9P39euf820EZ74= -cloud.google.com/go/compute v1.23.3 h1:6sVlXXBmbd7jNX0Ipq0trII3e4n1/MsADLK6a+aiVlk= -cloud.google.com/go/compute v1.23.3/go.mod h1:VCgBUoMnIVIR0CscqQiPJLAG25E3ZRZMzcFZeQ+h8CI= +cloud.google.com/go/compute v1.24.0 h1:phWcR2eWzRJaL/kOiJwfFsPs4BaKq1j6vnpZrc1YlVg= +cloud.google.com/go/compute v1.24.0/go.mod h1:kw1/T+h/+tK2LJK0wiPPx1intgdAM3j/g3hFDlscY40= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= -cloud.google.com/go/firestore v1.14.0 h1:8aLcKnMPoldYU3YHgu4t2exrKhLQkqaXAGqT0ljrFVw= -cloud.google.com/go/firestore v1.14.0/go.mod h1:96MVaHLsEhbvkBEdZgfN+AS/GIkco1LRpH9Xp9YZfzQ= -cloud.google.com/go/iam v1.1.5 h1:1jTsCu4bcsNsE4iiqNT5SHwrDRCfRmIaaaVFhRveTJI= -cloud.google.com/go/iam v1.1.5/go.mod h1:rB6P/Ic3mykPbFio+vo7403drjlgvoWfYpJhMXEbzv8= -cloud.google.com/go/longrunning v0.5.4 h1:w8xEcbZodnA2BbW6sVirkkoC+1gP8wS57EUUgGS0GVg= -cloud.google.com/go/longrunning v0.5.4/go.mod h1:zqNVncI0BOP8ST6XQD1+VcvuShMmq7+xFSzOL++V0dI= +cloud.google.com/go/firestore v1.15.0 h1:/k8ppuWOtNuDHt2tsRV42yI21uaGnKDEQnRFeBpbFF8= +cloud.google.com/go/firestore v1.15.0/go.mod h1:GWOxFXcv8GZUtYpWHw/w6IuYNux/BtmeVTMmjrm4yhk= +cloud.google.com/go/iam v1.1.7 h1:z4VHOhwKLF/+UYXAJDFwGtNF0b6gjsW1Pk9Ml0U/IoM= +cloud.google.com/go/iam v1.1.7/go.mod h1:J4PMPg8TtyurAUvSmPj8FF3EDgY1SPRZxcUGrn7WXGA= +cloud.google.com/go/longrunning v0.5.5 h1:GOE6pZFdSrTb4KAiKnXsJBtlE6mEyaW44oKyMILWnOg= +cloud.google.com/go/longrunning v0.5.5/go.mod h1:WV2LAxD8/rg5Z1cNW6FJ/ZpX4E4VnDnoTk0yawPBB7s= cloud.google.com/go/secretmanager v1.11.5 h1:82fpF5vBBvu9XW4qj0FU2C6qVMtj1RM/XHwKXUEAfYY= cloud.google.com/go/secretmanager v1.11.5/go.mod h1:eAGv+DaCHkeVyQi0BeXgAHOU0RdrMeZIASKc+S7VqH4= -cloud.google.com/go/storage v1.36.0 h1:P0mOkAcaJxhCTvAkMhxMfrTKiNcub4YmmPBtlhAyTr8= -cloud.google.com/go/storage v1.36.0/go.mod h1:M6M/3V/D3KpzMTJyPOR/HU6n2Si5QdaXYEsng2xgOs8= +cloud.google.com/go/storage v1.40.0 h1:VEpDQV5CJxFmJ6ueWNsKxcr1QAYOXEgxDa+sBbJahPw= +cloud.google.com/go/storage v1.40.0/go.mod h1:Rrj7/hKlG87BLqDJYtwR0fbPld8uJPbQ2ucUMY7Ir0g= contrib.go.opencensus.io/exporter/jaeger v0.2.1 h1:yGBYzYMewVL0yO9qqJv3Z5+IRhPdU7e9o/2oKpX4YvI= contrib.go.opencensus.io/exporter/jaeger v0.2.1/go.mod h1:Y8IsLgdxqh1QxYxPC5IgXVmBaeLUeQFfBeBi9PbeZd0= firebase.google.com/go v3.13.0+incompatible h1:3TdYC3DDi6aHn20qoRkxwGqNgdjtblwVAyRLQwGn/+4= firebase.google.com/go v3.13.0+incompatible/go.mod h1:xlah6XbEyW6tbfSklcfe5FHJIwjt8toICdV5Wh9ptHs= +firebase.google.com/go/v4 v4.14.1 h1:4qiUETaFRWoFGE1XP5VbcEdtPX93Qs+8B/7KvP2825g= +firebase.google.com/go/v4 v4.14.1/go.mod h1:fgk2XshgNDEKaioKco+AouiegSI9oTWVqRaBdTTGBoM= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -37,6 +39,8 @@ github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t github.com/Gurpartap/storekit-go v0.0.0-20201205024111-36b6cd5c6a21 h1:HcdvlzaQ4CJfH7xbfJZ3ZHN//BTEpId46iKEMuP3wHE= github.com/Gurpartap/storekit-go v0.0.0-20201205024111-36b6cd5c6a21/go.mod h1:7PODFS++oNZ6khojmPBvkrDeFO/hrc3jmvWvQAOXorw= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= +github.com/MicahParks/keyfunc v1.9.0 h1:lhKd5xrFHLNOWrDc4Tyb/Q1AJ4LCzQ48GVJyVIID3+o= +github.com/MicahParks/keyfunc v1.9.0/go.mod h1:IdnCilugA0O/99dW+/MkvlyrsX8+L8+x95xuVNtM5jw= github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw= @@ -323,6 +327,7 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= +github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0= @@ -334,6 +339,7 @@ github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= @@ -345,8 +351,8 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= -github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= -github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -390,8 +396,8 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= -github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= -github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= +github.com/googleapis/gax-go/v2 v2.12.3 h1:5/zPPDvw8Q1SuXjrqrZslrqT7dL/uJT2CQii/cLCKqA= +github.com/googleapis/gax-go/v2 v2.12.3/go.mod h1:AKloxT6GtNbaLm8QTNSidHUVsHYcBHwWRvkNFJUQcS4= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c h1:7lF+Vz0LqiRidnzC1Oq86fpX1q/iEv2KJdrCtttYjT4= github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= @@ -979,18 +985,18 @@ github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxt go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 h1:UNQQKPfTDe1J81ViolILjTKPr9WetKW6uei2hFgJmFs= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0/go.mod h1:r9vWsPS/3AQItv3OSlEJ/E4mbrhUbbw18meOjArPtKQ= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0 h1:sv9kVfal0MK0wBMCOGr+HeJm9v803BkJxGrk2au7j08= -go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.47.0/go.mod h1:SK2UL73Zy1quvRPonmOmRDiWk1KBV3LyIeeIxcEApWw= -go.opentelemetry.io/otel v1.23.0 h1:Df0pqjqExIywbMCMTxkAwzjLZtRf+bBKLbUcpxO2C9E= -go.opentelemetry.io/otel v1.23.0/go.mod h1:YCycw9ZeKhcJFrb34iVSkyT0iczq/zYDtZYFufObyB0= -go.opentelemetry.io/otel/metric v1.23.0 h1:pazkx7ss4LFVVYSxYew7L5I6qvLXHA0Ap2pwV+9Cnpo= -go.opentelemetry.io/otel/metric v1.23.0/go.mod h1:MqUW2X2a6Q8RN96E2/nqNoT+z9BSms20Jb7Bbp+HiTo= -go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8= -go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E= -go.opentelemetry.io/otel/trace v1.23.0 h1:37Ik5Ib7xfYVb4V1UtnT97T1jI+AoIYkJyPkuL4iJgI= -go.opentelemetry.io/otel/trace v1.23.0/go.mod h1:GSGTbIClEsuZrGIzoEHqsVfxgn5UkggkflQwDScNUsk= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 h1:4Pp6oUg3+e/6M4C0A/3kJ2VYa++dsWVTtGgLVj5xtHg= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0/go.mod h1:Mjt1i1INqiaoZOMGR1RIUJN+i3ChKoFRqzrRQhlkbs0= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= +go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= +go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= +go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= +go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= +go.opentelemetry.io/otel/sdk v1.22.0 h1:6coWHw9xw7EfClIC/+O31R8IY3/+EiRFHevmHafB2Gw= +go.opentelemetry.io/otel/sdk v1.22.0/go.mod h1:iu7luyVGYovrRpe2fmj3CVKouQNdTOkxtLzPvPz1DOc= +go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= +go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -1032,8 +1038,8 @@ golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.19.0 h1:ENy+Az/9Y1vSrlrvBSyna3PITt4tiZLf7sgCjZBX7Wo= -golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20240213143201-ec583247a57a h1:HinSgX1tJRX3KsL//Gxynpw5CTOAIPhgL4W8PNiIpVE= golang.org/x/exp v0.0.0-20240213143201-ec583247a57a/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= @@ -1070,12 +1076,13 @@ golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220708220712-1185a9018129/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= -golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= +golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.17.0 h1:6m3ZPmLEFdVxKKWnKq4VqZ60gutO35zm+zrAHVmHyDQ= -golang.org/x/oauth2 v0.17.0/go.mod h1:OzPDGQiuQMguemayvdylqddI7qcD9lnSDb+1FiwQ5HA= +golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= +golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1127,13 +1134,13 @@ golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= -golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U= -golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.18.0 h1:FcHjZXDMxI8mM3nwhX9HlKop4C0YQvCVCdwYl2wOtE8= +golang.org/x/term v0.18.0/go.mod h1:ILwASektA3OnRv7amZ1xhE/KTR+u50pbXfZ03+6Nx58= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -1175,28 +1182,30 @@ golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= -google.golang.org/api v0.164.0 h1:of5G3oE2WRMVb2yoWKME4ZP8y8zpUKC6bMhxDr8ifyk= -google.golang.org/api v0.164.0/go.mod h1:2OatzO7ZDQsoS7IFf3rvsE17/TldiU3F/zxFHeqUB5o= +google.golang.org/api v0.170.0 h1:zMaruDePM88zxZBG+NG8+reALO2rfLhe/JShitLyT48= +google.golang.org/api v0.170.0/go.mod h1:/xql9M2btF85xac/VAm4PsLMTLVGUOpq4BE9R8jyNy8= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= +google.golang.org/appengine/v2 v2.0.2 h1:MSqyWy2shDLwG7chbwBJ5uMyw6SNqJzhJHNDwYB0Akk= +google.golang.org/appengine/v2 v2.0.2/go.mod h1:PkgRUWz4o1XOvbqtWTkBtCitEJ5Tp4HoVEdMMYQR/8E= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= -google.golang.org/genproto v0.0.0-20240125205218-1f4bbc51befe h1:USL2DhxfgRchafRvt/wYyyQNzwgL7ZiURcozOE/Pkvo= -google.golang.org/genproto v0.0.0-20240125205218-1f4bbc51befe/go.mod h1:cc8bqMqtv9gMOr0zHg2Vzff5ULhhL2IXP4sbcn32Dro= -google.golang.org/genproto/googleapis/api v0.0.0-20240125205218-1f4bbc51befe h1:0poefMBYvYbs7g5UkjS6HcxBPaTRAmznle9jnxYoAI8= -google.golang.org/genproto/googleapis/api v0.0.0-20240125205218-1f4bbc51befe/go.mod h1:4jWUdICTdgc3Ibxmr8nAJiiLHwQBY0UI0XZcEMaFKaA= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240205150955-31a09d347014 h1:FSL3lRCkhaPFxqi0s9o+V4UI2WTzAVOvkgbd4kVV4Wg= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240205150955-31a09d347014/go.mod h1:SaPjaZGWb0lPqs6Ittu0spdfrOArqji4ZdeP5IC/9N4= +google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 h1:9+tzLLstTlPTRyJTh+ah5wIMsBW5c4tQwGTN3thOW9Y= +google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9/go.mod h1:mqHbVIp48Muh7Ywss/AD6I5kNVKZMmAa/QEW58Gxp2s= +google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c h1:kaI7oewGK5YnVwj+Y+EJBO/YN1ht8iTL9XkFHtVZLsc= +google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c/go.mod h1:VQW3tUculP/D4B+xVCo+VgSq8As6wA9ZjHl//pmk+6s= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240311132316-a219d84964c2 h1:9IZDv+/GcI6u+a4jRFRLxQs0RUCfavGfoOgEW6jpkI0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240311132316-a219d84964c2/go.mod h1:UCOku4NytXMJuLQE5VuqA5lX3PcHCBo8pxNyvkf4xBs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.62.0 h1:HQKZ/fa1bXkX1oFOvSjmZEUL8wLSaZTjCcLAlmZRtdk= -google.golang.org/grpc v1.62.0/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= +google.golang.org/grpc v1.62.1 h1:B4n+nfKzOICUXMgyrNd19h/I9oH0L1pizfk1d4zSgTk= +google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1209,8 +1218,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= -google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/cenkalti/backoff.v1 v1.1.0 h1:Arh75ttbsvlpVA7WtVpH4u9h6Zl46xuptxqLxPiSo4Y= gopkg.in/cenkalti/backoff.v1 v1.1.0/go.mod h1:J6Vskwqd+OMVJl8C33mmtxTBs2gyzfv7UDAkHu8BrjI= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/backend/pkg/notification/firebase.go b/backend/pkg/notification/firebase.go index 91291da54..9e42cc49a 100644 --- a/backend/pkg/notification/firebase.go +++ b/backend/pkg/notification/firebase.go @@ -2,12 +2,13 @@ package notification import ( "context" + "fmt" "strings" "time" - firebase "firebase.google.com/go" "firebase.google.com/go/messaging" + firebase "firebase.google.com/go/v4" "github.com/gobitfly/beaconchain/pkg/commons/log" "github.com/gobitfly/beaconchain/pkg/commons/utils" "google.golang.org/api/option" @@ -24,10 +25,10 @@ func isRelevantError(response *messaging.SendResponse) bool { return false } -func SendPushBatch(messages []*messaging.Message) error { +func SendPushBatch(messages []*messaging.Message, dryRun bool) error { credentialsPath := utils.Config.Notifications.FirebaseCredentialsPath if credentialsPath == "" { - log.Error(nil, "firebase credentials path not provided, disabling push notifications", 0) + log.Error(fmt.Errorf("firebase credentials path not provided, disabling push notifications"), "error initializing SendPushBatch", 0) return nil } @@ -42,29 +43,32 @@ func SendPushBatch(messages []*messaging.Message) error { app, err := firebase.NewApp(context.Background(), nil, opt) if err != nil { - log.Error(nil, "error initializing app", 0) + log.Error(err, "error initializing app", 0) return err } client, err := app.Messaging(ctx) if err != nil { - log.Error(nil, "error initializing messaging", 0) + log.Error(err, "error initializing messaging", 0) return err } - var waitBeforeTryInSeconds = []time.Duration{0 * time.Second, 2 * time.Second, 4 * time.Second, 8 * time.Second, 16 * time.Second} + var waitBeforeTryInSeconds = []time.Duration{0, 2, 4, 8, 16} var resultSuccessCount, resultFailureCount int = 0, 0 var result *messaging.BatchResponse currentMessages := messages tries := 0 for _, s := range waitBeforeTryInSeconds { - time.Sleep(s) + time.Sleep(s * time.Second) tries++ - - result, err = client.SendAll(context.Background(), currentMessages) + if dryRun { + result, err = client.SendEachDryRun(context.Background(), currentMessages) + } else { + result, err = client.SendEach(context.Background(), currentMessages) + } if err != nil { - log.Error(nil, "error sending push notifications", 0) + log.Error(err, "error sending push notifications", 0) return err } @@ -74,7 +78,9 @@ func SendPushBatch(messages []*messaging.Message) error { newMessages := make([]*messaging.Message, 0, result.FailureCount) if result.FailureCount > 0 { for i, response := range result.Responses { + logger.Info(response) if isRelevantError(response) { + logger.Infof("retrying message %d", i) newMessages = append(newMessages, currentMessages[i]) resultFailureCount-- } @@ -90,12 +96,12 @@ func SendPushBatch(messages []*messaging.Message) error { if len(currentMessages) > 0 { for _, response := range result.Responses { if isRelevantError(response) { - log.Error(nil, "firebase error", 0, log.Fields{"MessageID": response.MessageID, "response": response.Error}) + logger.WithError(response.Error).WithField("MessageID", response.MessageID).Errorf("firebase error") resultFailureCount++ } } } - log.Infof("sent %d firebase notifications in %d of %d tries. successful: %d | failed: %d", len(messages), tries, len(waitBeforeTryInSeconds), resultSuccessCount, resultFailureCount) + logger.Infof("sent %d firebase notifications in %d of %d tries. successful: %d | failed: %d", len(messages), tries, len(waitBeforeTryInSeconds), resultSuccessCount, resultFailureCount) return nil } diff --git a/backend/pkg/notification/notifications.go b/backend/pkg/notification/notifications.go index d7cb84e97..944e2aa99 100644 --- a/backend/pkg/notification/notifications.go +++ b/backend/pkg/notification/notifications.go @@ -678,7 +678,7 @@ func sendPushNotifications() error { end = len(n.Content.Messages) } - err = SendPushBatch(n.Content.Messages[start:end]) + err = SendPushBatch(n.Content.Messages[start:end], false) if err != nil { metrics.Errors.WithLabelValues("notifications_send_push_batch").Inc() log.Error(err, "error sending firebase batch job", 0) From 13780b3bd18e55c854f469d4bd0f65b906531fec Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Tue, 10 Sep 2024 11:37:15 +0000 Subject: [PATCH 26/94] chore(notifications): fix build errors --- backend/pkg/commons/types/frontend.go | 2 +- backend/pkg/notification/firebase.go | 10 +++++----- backend/pkg/notification/notifications.go | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/backend/pkg/commons/types/frontend.go b/backend/pkg/commons/types/frontend.go index eb1c04472..a7b637d8f 100644 --- a/backend/pkg/commons/types/frontend.go +++ b/backend/pkg/commons/types/frontend.go @@ -10,7 +10,7 @@ import ( "strings" "time" - "firebase.google.com/go/messaging" + "firebase.google.com/go/v4/messaging" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/gobitfly/beaconchain/pkg/commons/log" diff --git a/backend/pkg/notification/firebase.go b/backend/pkg/notification/firebase.go index 9e42cc49a..acdf303f4 100644 --- a/backend/pkg/notification/firebase.go +++ b/backend/pkg/notification/firebase.go @@ -7,8 +7,8 @@ import ( "strings" "time" - "firebase.google.com/go/messaging" firebase "firebase.google.com/go/v4" + "firebase.google.com/go/v4/messaging" "github.com/gobitfly/beaconchain/pkg/commons/log" "github.com/gobitfly/beaconchain/pkg/commons/utils" "google.golang.org/api/option" @@ -78,9 +78,9 @@ func SendPushBatch(messages []*messaging.Message, dryRun bool) error { newMessages := make([]*messaging.Message, 0, result.FailureCount) if result.FailureCount > 0 { for i, response := range result.Responses { - logger.Info(response) + log.Info(response) if isRelevantError(response) { - logger.Infof("retrying message %d", i) + log.Infof("retrying message %d", i) newMessages = append(newMessages, currentMessages[i]) resultFailureCount-- } @@ -96,12 +96,12 @@ func SendPushBatch(messages []*messaging.Message, dryRun bool) error { if len(currentMessages) > 0 { for _, response := range result.Responses { if isRelevantError(response) { - logger.WithError(response.Error).WithField("MessageID", response.MessageID).Errorf("firebase error") + log.Error(fmt.Errorf("firebase error, message id: %d, error: %s", response.MessageID, response.Error), "error sending push notifications", 0) resultFailureCount++ } } } - logger.Infof("sent %d firebase notifications in %d of %d tries. successful: %d | failed: %d", len(messages), tries, len(waitBeforeTryInSeconds), resultSuccessCount, resultFailureCount) + log.Infof("sent %d firebase notifications in %d of %d tries. successful: %d | failed: %d", len(messages), tries, len(waitBeforeTryInSeconds), resultSuccessCount, resultFailureCount) return nil } diff --git a/backend/pkg/notification/notifications.go b/backend/pkg/notification/notifications.go index 944e2aa99..352b71ca9 100644 --- a/backend/pkg/notification/notifications.go +++ b/backend/pkg/notification/notifications.go @@ -22,7 +22,7 @@ import ( "time" gcp_bigtable "cloud.google.com/go/bigtable" - "firebase.google.com/go/messaging" + "firebase.google.com/go/v4/messaging" "github.com/ethereum/go-ethereum/common" "github.com/gobitfly/beaconchain/pkg/commons/cache" "github.com/gobitfly/beaconchain/pkg/commons/db" From 98f7a6303facfeb26e6cb97f22625f787ba326ba Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Tue, 10 Sep 2024 11:39:46 +0000 Subject: [PATCH 27/94] chore(notifications): fix relevant error messages --- backend/pkg/notification/firebase.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/pkg/notification/firebase.go b/backend/pkg/notification/firebase.go index acdf303f4..cd1788da5 100644 --- a/backend/pkg/notification/firebase.go +++ b/backend/pkg/notification/firebase.go @@ -18,7 +18,9 @@ func isRelevantError(response *messaging.SendResponse) bool { if !response.Success && response.Error != nil { // Ignore https://stackoverflow.com/questions/58308835/using-firebase-for-notifications-getting-app-instance-has-been-unregistered // Errors since they indicate that the user token is expired - if !strings.Contains(response.Error.Error(), "registration-token-not-registered") { + if !strings.Contains(response.Error.Error(), "registration-token-not-registered") && + !strings.Contains(response.Error.Error(), "Requested entity was not found.") && + !strings.Contains(response.Error.Error(), "Request contains an invalid argument.") { return true } } From 4f4367f9eff5fd24053436a29b38c697198e6f38 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Tue, 10 Sep 2024 11:41:30 +0000 Subject: [PATCH 28/94] chore(notifications): disable spammy log output --- backend/pkg/notification/firebase.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/pkg/notification/firebase.go b/backend/pkg/notification/firebase.go index cd1788da5..44551b12c 100644 --- a/backend/pkg/notification/firebase.go +++ b/backend/pkg/notification/firebase.go @@ -80,7 +80,7 @@ func SendPushBatch(messages []*messaging.Message, dryRun bool) error { newMessages := make([]*messaging.Message, 0, result.FailureCount) if result.FailureCount > 0 { for i, response := range result.Responses { - log.Info(response) + //log.Info(response) if isRelevantError(response) { log.Infof("retrying message %d", i) newMessages = append(newMessages, currentMessages[i]) From d3c3a1a44b61b6a87add9d383fa3ef5a9b8cd668 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Tue, 10 Sep 2024 11:44:11 +0000 Subject: [PATCH 29/94] chore(notifications): please linter --- backend/pkg/notification/firebase.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/pkg/notification/firebase.go b/backend/pkg/notification/firebase.go index 44551b12c..3b5264c38 100644 --- a/backend/pkg/notification/firebase.go +++ b/backend/pkg/notification/firebase.go @@ -98,7 +98,7 @@ func SendPushBatch(messages []*messaging.Message, dryRun bool) error { if len(currentMessages) > 0 { for _, response := range result.Responses { if isRelevantError(response) { - log.Error(fmt.Errorf("firebase error, message id: %d, error: %s", response.MessageID, response.Error), "error sending push notifications", 0) + log.Error(fmt.Errorf("firebase error, message id: %s, error: %s", response.MessageID, response.Error), "error sending push notifications", 0) resultFailureCount++ } } From 8ed5ca29394cf519809fedec4f1c5d9e9bb7746a Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Tue, 10 Sep 2024 11:47:31 +0000 Subject: [PATCH 30/94] chore(notifications): please linter --- backend/pkg/notification/firebase.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/pkg/notification/firebase.go b/backend/pkg/notification/firebase.go index 3b5264c38..b4a75a3d6 100644 --- a/backend/pkg/notification/firebase.go +++ b/backend/pkg/notification/firebase.go @@ -55,14 +55,14 @@ func SendPushBatch(messages []*messaging.Message, dryRun bool) error { return err } - var waitBeforeTryInSeconds = []time.Duration{0, 2, 4, 8, 16} + var waitBeforeTryInSeconds = []int{0, 2, 4, 8, 16} var resultSuccessCount, resultFailureCount int = 0, 0 var result *messaging.BatchResponse currentMessages := messages tries := 0 for _, s := range waitBeforeTryInSeconds { - time.Sleep(s * time.Second) + time.Sleep(time.Duration(s) * time.Second) tries++ if dryRun { result, err = client.SendEachDryRun(context.Background(), currentMessages) From 688c3997c1ed9f1ad2f7d2d67052ceedfe95b919 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Mon, 16 Sep 2024 08:48:27 +0000 Subject: [PATCH 31/94] fix(notifications): correct wrong sql syntax --- backend/pkg/notification/db.go | 2 ++ backend/pkg/notification/notifications.go | 10 +++++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/backend/pkg/notification/db.go b/backend/pkg/notification/db.go index 1f999a411..d065f4e87 100644 --- a/backend/pkg/notification/db.go +++ b/backend/pkg/notification/db.go @@ -59,6 +59,8 @@ func GetSubsForEventFilter(eventName types.EventName, lastSentFilter string, las return nil, err } + log.Info(query) + subMap := make(map[string][]types.Subscription, 0) err = db.FrontendWriterDB.Select(&subs, query, args...) if err != nil { diff --git a/backend/pkg/notification/notifications.go b/backend/pkg/notification/notifications.go index 352b71ca9..03b27ef44 100644 --- a/backend/pkg/notification/notifications.go +++ b/backend/pkg/notification/notifications.go @@ -1952,7 +1952,7 @@ func collectEthClientNotifications(notificationsByUserID types.NotificationsPerU dbResult, err := GetSubsForEventFilter( types.EthClientUpdateEventName, - "(us.last_sent_ts <= NOW() - INTERVAL '2 DAY' AND TO_TIMESTAMP(?) > us.last_sent_ts) OR us.last_sent_ts IS NULL", + "((last_sent_ts <= NOW() - INTERVAL '2 DAY' AND TO_TIMESTAMP(?) > last_sent_ts) OR last_sent_ts IS NULL)", []interface{}{client.Date.Unix()}, []string{strings.ToLower(client.Name)}) if err != nil { @@ -2074,7 +2074,7 @@ func collectMonitoringMachine( dbResult, err := GetSubsForEventFilter( eventName, - "us.created_epoch <= ? AND (us.last_sent_epoch < (? - ?) OR us.last_sent_epoch IS NULL)", + "(created_epoch <= ? AND (last_sent_epoch < (? - ?) OR last_sent_epoch IS NULL))", []interface{}{epoch, epoch, epochWaitInBetween}, nil, ) @@ -2325,7 +2325,7 @@ func collectTaxReportNotificationNotifications(notificationsByUserID types.Notif dbResults, err := GetSubsForEventFilter( types.TaxReportEventName, - "us.last_sent_ts < ? OR (us.last_sent_ts IS NULL AND us.created_ts < ?)", + "(last_sent_ts < ? OR (last_sent_ts IS NULL AND created_ts < ?))", []interface{}{firstDayOfMonth, firstDayOfMonth}, nil, ) @@ -2487,7 +2487,7 @@ func collectRocketpoolComissionNotifications(notificationsByUserID types.Notific dbResult, err := GetSubsForEventFilter( types.RocketpoolCommissionThresholdEventName, - "(us.last_sent_ts <= NOW() - INTERVAL '8 hours' OR us.last_sent_ts IS NULL) AND (us.event_threshold <= ? OR (us.event_threshold < 0 AND us.event_threshold * -1 >= ?)", + "(last_sent_ts <= NOW() - INTERVAL '8 hours' OR last_sent_ts IS NULL) AND (event_threshold <= ? OR (event_threshold < 0 AND event_threshold * -1 >= ?))", []interface{}{fee, fee}, nil, ) @@ -2539,7 +2539,7 @@ func collectRocketpoolRewardClaimRoundNotifications(notificationsByUserID types. dbResult, err := GetSubsForEventFilter( types.RocketpoolNewClaimRoundStartedEventName, - "us.last_sent_ts <= NOW() - INTERVAL '5 hours' OR us.last_sent_ts IS NULL", + "(last_sent_ts <= NOW() - INTERVAL '5 hours' OR last_sent_ts IS NULL)", nil, nil, ) From c141b4c79e7ee6ffac857ede972a48be72ef82b6 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Tue, 17 Sep 2024 09:43:56 +0000 Subject: [PATCH 32/94] fix(notifications): add last sent field to webhook data retrieval query --- backend/pkg/notification/notifications.go | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/pkg/notification/notifications.go b/backend/pkg/notification/notifications.go index 03b27ef44..357c81a35 100644 --- a/backend/pkg/notification/notifications.go +++ b/backend/pkg/notification/notifications.go @@ -837,6 +837,7 @@ func queueWebhookNotifications(notificationsByUserID types.NotificationsPerUserI url, retries, event_names, + last_sent, destination FROM users_webhooks From 3ff4be3781d99f302204338d2727a3235154e73e Mon Sep 17 00:00:00 2001 From: Patrick Pfeiffer Date: Tue, 17 Sep 2024 13:41:50 +0200 Subject: [PATCH 33/94] chore(notifications): better check for webhook-ratelimit --- backend/pkg/notification/notifications.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/pkg/notification/notifications.go b/backend/pkg/notification/notifications.go index 357c81a35..89ec6c858 100644 --- a/backend/pkg/notification/notifications.go +++ b/backend/pkg/notification/notifications.go @@ -1164,7 +1164,7 @@ func sendDiscordNotifications() error { resp.Body.Close() } - if strings.Contains(errResp.Body, "You are being rate limited") { + if resp.StatusCode == http.StatusTooManyRequests { log.Warnf("could not push to discord webhook due to rate limit. %v url: %v", errResp.Body, webhook.Url) } else { log.Error(nil, "error pushing discord webhook", 0, map[string]interface{}{"errResp.Body": errResp.Body, "webhook.Url": webhook.Url}) From f9aecaf7bdd9f5cd63f20c6bf096a92bac6326bd Mon Sep 17 00:00:00 2001 From: LUCCA DUKIC <109136188+LuccaBitfly@users.noreply.github.com> Date: Thu, 19 Sep 2024 10:58:52 +0200 Subject: [PATCH 34/94] (BEDS-452) move clients in notification settings --- backend/pkg/api/types/notifications.go | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/backend/pkg/api/types/notifications.go b/backend/pkg/api/types/notifications.go index 27bf585be..5d80efc89 100644 --- a/backend/pkg/api/types/notifications.go +++ b/backend/pkg/api/types/notifications.go @@ -162,6 +162,12 @@ type NotificationPairedDevice struct { } type InternalPutUserNotificationSettingsPairedDevicesResponse ApiDataResponse[NotificationPairedDevice] +type NotificationSettingsClients struct { + Id uint64 `json:"id"` + Name string `json:"name"` + Category string `json:"category"` + IsSubscribed bool `json:"is_subscribed"` +} type NotificationSettingsGeneral struct { DoNotDisturbTimestamp int64 `json:"do_not_disturb_timestamp"` // notifications are disabled until this timestamp IsEmailNotificationsEnabled bool `json:"is_email_notifications_enabled"` @@ -175,18 +181,18 @@ type NotificationSettingsGeneral struct { IsMachineMemoryUsageSubscribed bool `json:"is_machine_memory_usage_subscribed"` MachineMemoryUsageThreshold float64 `json:"machine_memory_usage_threshold" faker:"boundary_start=0, boundary_end=1"` - SubscribedClients []string `json:"subscribed_clients"` - IsRocketPoolNewRewardRoundSubscribed bool `json:"is_rocket_pool_new_reward_round_subscribed"` - IsRocketPoolMaxCollateralSubscribed bool `json:"is_rocket_pool_max_collateral_subscribed"` - RocketPoolMaxCollateralThreshold float64 `json:"rocket_pool_max_collateral_threshold" faker:"boundary_start=0, boundary_end=1"` - IsRocketPoolMinCollateralSubscribed bool `json:"is_rocket_pool_min_collateral_subscribed"` - RocketPoolMinCollateralThreshold float64 `json:"rocket_pool_min_collateral_threshold" faker:"boundary_start=0, boundary_end=1"` + IsRocketPoolNewRewardRoundSubscribed bool `json:"is_rocket_pool_new_reward_round_subscribed"` + IsRocketPoolMaxCollateralSubscribed bool `json:"is_rocket_pool_max_collateral_subscribed"` + RocketPoolMaxCollateralThreshold float64 `json:"rocket_pool_max_collateral_threshold" faker:"boundary_start=0, boundary_end=1"` + IsRocketPoolMinCollateralSubscribed bool `json:"is_rocket_pool_min_collateral_subscribed"` + RocketPoolMinCollateralThreshold float64 `json:"rocket_pool_min_collateral_threshold" faker:"boundary_start=0, boundary_end=1"` } type InternalPutUserNotificationSettingsGeneralResponse ApiDataResponse[NotificationSettingsGeneral] type NotificationSettings struct { - GeneralSettings NotificationSettingsGeneral `json:"general_settings"` - Networks []NotificationNetwork `json:"networks"` - PairedDevices []NotificationPairedDevice `json:"paired_devices"` + GeneralSettings NotificationSettingsGeneral `json:"general_settings"` + Networks []NotificationNetwork `json:"networks"` + PairedDevices []NotificationPairedDevice `json:"paired_devices"` + Clients []NotificationSettingsClients `json:"clients"` } type InternalGetUserNotificationSettingsResponse ApiDataResponse[NotificationSettings] From d0a011c6dc740ce3673464078e94cfc3afedb0dc Mon Sep 17 00:00:00 2001 From: LUCCA DUKIC <109136188+LuccaBitfly@users.noreply.github.com> Date: Thu, 19 Sep 2024 12:29:45 +0200 Subject: [PATCH 35/94] (BEDS-452) add new client notification settings endpoint --- backend/pkg/api/data_access/dummy.go | 5 + backend/pkg/api/data_access/notifications.go | 4 + backend/pkg/api/handlers/internal.go | 4 + backend/pkg/api/handlers/public.go | 122 +++++++++++++------ backend/pkg/api/router.go | 1 + backend/pkg/api/types/notifications.go | 13 +- 6 files changed, 104 insertions(+), 45 deletions(-) diff --git a/backend/pkg/api/data_access/dummy.go b/backend/pkg/api/data_access/dummy.go index 7bee9a38c..38fb693ab 100644 --- a/backend/pkg/api/data_access/dummy.go +++ b/backend/pkg/api/data_access/dummy.go @@ -488,6 +488,11 @@ func (d *DummyService) UpdateNotificationSettingsPairedDevice(ctx context.Contex func (d *DummyService) DeleteNotificationSettingsPairedDevice(ctx context.Context, userId uint64, pairedDeviceId string) error { return nil } + +func (d *DummyService) UpdateNotificationSettingsClients(ctx context.Context, userId uint64, clientId uint64, IsSubscribed bool) (*t.NotificationSettingsClient, error) { + return getDummyStruct[t.NotificationSettingsClient]() +} + func (d *DummyService) GetNotificationSettingsDashboards(ctx context.Context, userId uint64, cursor string, colSort t.Sort[enums.NotificationSettingsDashboardColumn], search string, limit uint64) ([]t.NotificationSettingsDashboardsTableRow, *t.Paging, error) { r, p, err := getDummyWithPaging[t.NotificationSettingsDashboardsTableRow]() for i, n := range r { diff --git a/backend/pkg/api/data_access/notifications.go b/backend/pkg/api/data_access/notifications.go index 5389847db..44f7ca4a5 100644 --- a/backend/pkg/api/data_access/notifications.go +++ b/backend/pkg/api/data_access/notifications.go @@ -25,6 +25,7 @@ type NotificationsRepository interface { UpdateNotificationSettingsNetworks(ctx context.Context, userId uint64, chainId uint64, settings t.NotificationSettingsNetwork) error UpdateNotificationSettingsPairedDevice(ctx context.Context, userId uint64, pairedDeviceId string, name string, IsNotificationsEnabled bool) error DeleteNotificationSettingsPairedDevice(ctx context.Context, userId uint64, pairedDeviceId string) error + UpdateNotificationSettingsClients(ctx context.Context, userId uint64, clientId uint64, IsSubscribed bool) (*t.NotificationSettingsClient, error) GetNotificationSettingsDashboards(ctx context.Context, userId uint64, cursor string, colSort t.Sort[enums.NotificationSettingsDashboardColumn], search string, limit uint64) ([]t.NotificationSettingsDashboardsTableRow, *t.Paging, error) UpdateNotificationSettingsValidatorDashboard(ctx context.Context, dashboardId t.VDBIdPrimary, groupId uint64, settings t.NotificationSettingsValidatorDashboard) error UpdateNotificationSettingsAccountDashboard(ctx context.Context, dashboardId t.VDBIdPrimary, groupId uint64, settings t.NotificationSettingsAccountDashboard) error @@ -72,6 +73,9 @@ func (d *DataAccessService) UpdateNotificationSettingsPairedDevice(ctx context.C func (d *DataAccessService) DeleteNotificationSettingsPairedDevice(ctx context.Context, userId uint64, pairedDeviceId string) error { return d.dummy.DeleteNotificationSettingsPairedDevice(ctx, userId, pairedDeviceId) } +func (d *DataAccessService) UpdateNotificationSettingsClients(ctx context.Context, userId uint64, clientId uint64, IsSubscribed bool) (*t.NotificationSettingsClient, error) { + return d.dummy.UpdateNotificationSettingsClients(ctx, userId, clientId, IsSubscribed) +} func (d *DataAccessService) GetNotificationSettingsDashboards(ctx context.Context, userId uint64, cursor string, colSort t.Sort[enums.NotificationSettingsDashboardColumn], search string, limit uint64) ([]t.NotificationSettingsDashboardsTableRow, *t.Paging, error) { return d.dummy.GetNotificationSettingsDashboards(ctx, userId, cursor, colSort, search, limit) } diff --git a/backend/pkg/api/handlers/internal.go b/backend/pkg/api/handlers/internal.go index 78d998856..bf3e347fe 100644 --- a/backend/pkg/api/handlers/internal.go +++ b/backend/pkg/api/handlers/internal.go @@ -609,6 +609,10 @@ func (h *HandlerService) InternalDeleteUserNotificationSettingsPairedDevices(w h h.PublicDeleteUserNotificationSettingsPairedDevices(w, r) } +func (h *HandlerService) InternalPutUserNotificationSettingsClient(w http.ResponseWriter, r *http.Request) { + h.PublicPutUserNotificationSettingsClient(w, r) +} + func (h *HandlerService) InternalGetUserNotificationSettingsDashboards(w http.ResponseWriter, r *http.Request) { h.PublicGetUserNotificationSettingsDashboards(w, r) } diff --git a/backend/pkg/api/handlers/public.go b/backend/pkg/api/handlers/public.go index e0e560b8c..d160582df 100644 --- a/backend/pkg/api/handlers/public.go +++ b/backend/pkg/api/handlers/public.go @@ -271,7 +271,7 @@ func (h *HandlerService) PublicGetValidatorDashboard(w http.ResponseWriter, r *h // @Security ApiKeyInHeader || ApiKeyInQuery // @Tags Validator Dashboard Management // @Produce json -// @Param dashboard_id path string true "The ID of the dashboard." +// @Param dashboard_id path integer true "The ID of the dashboard." // @Success 204 "Dashboard deleted successfully." // @Failure 400 {object} types.ApiErrorResponse "Bad Request" // @Router /validator-dashboards/{dashboard_id} [delete] @@ -297,7 +297,7 @@ func (h *HandlerService) PublicDeleteValidatorDashboard(w http.ResponseWriter, r // @Tags Validator Dashboard Management // @Accept json // @Produce json -// @Param dashboard_id path string true "The ID of the dashboard." +// @Param dashboard_id path integer true "The ID of the dashboard." // @Param request body handlers.PublicPutValidatorDashboardName.request true "request" // @Success 200 {object} types.ApiDataResponse[types.VDBPostReturnData] // @Failure 400 {object} types.ApiErrorResponse @@ -336,7 +336,7 @@ func (h *HandlerService) PublicPutValidatorDashboardName(w http.ResponseWriter, // @Tags Validator Dashboard Management // @Accept json // @Produce json -// @Param dashboard_id path string true "The ID of the dashboard." +// @Param dashboard_id path integer true "The ID of the dashboard." // @Param request body handlers.PublicPostValidatorDashboardGroups.request true "request" // @Success 201 {object} types.ApiDataResponse[types.VDBPostCreateGroupData] // @Failure 400 {object} types.ApiErrorResponse @@ -400,8 +400,8 @@ func (h *HandlerService) PublicPostValidatorDashboardGroups(w http.ResponseWrite // @Tags Validator Dashboard Management // @Accept json // @Produce json -// @Param dashboard_id path string true "The ID of the dashboard." -// @Param group_id path string true "The ID of the group." +// @Param dashboard_id path integer true "The ID of the dashboard." +// @Param group_id path integer true "The ID of the group." // @Param request body handlers.PublicPutValidatorDashboardGroups.request true "request" // @Success 200 {object} types.ApiDataResponse[types.VDBPostCreateGroupData] // @Failure 400 {object} types.ApiErrorResponse @@ -453,8 +453,8 @@ func (h *HandlerService) PublicPutValidatorDashboardGroups(w http.ResponseWriter // @Security ApiKeyInHeader || ApiKeyInQuery // @Accept json // @Produce json -// @Param dashboard_id path string true "The ID of the dashboard." -// @Param group_id path string true "The ID of the group." +// @Param dashboard_id path integer true "The ID of the dashboard." +// @Param group_id path integer true "The ID of the group." // @Success 204 "Group deleted successfully." // @Failure 400 {object} types.ApiErrorResponse // @Router /validator-dashboards/{dashboard_id}/groups/{group_id} [delete] @@ -496,7 +496,7 @@ func (h *HandlerService) PublicDeleteValidatorDashboardGroup(w http.ResponseWrit // @Tags Validator Dashboard Management // @Accept json // @Produce json -// @Param dashboard_id path string true "The ID of the dashboard." +// @Param dashboard_id path integer true "The ID of the dashboard." // @Param request body handlers.PublicPostValidatorDashboardValidators.request true "`group_id`: (optional) Provide a single group id, to which all validators get added to. If omitted, the default group will be used.

To add validators, only one of the following fields can be set:" // @Success 201 {object} types.ApiDataResponse[[]types.VDBPostValidatorsData] "Returns a list of added validators." // @Failure 400 {object} types.ApiErrorResponse @@ -631,7 +631,7 @@ func (h *HandlerService) PublicPostValidatorDashboardValidators(w http.ResponseW // @Tags Validator Dashboard // @Produce json // @Param dashboard_id path string true "The ID of the dashboard." -// @Param group_id query string false "The ID of the group." +// @Param group_id query integer false "The ID of the group." // @Param limit query string false "The maximum number of results that may be returned." // @Param sort query string false "The field you want to sort by. Append with `:desc` for descending order." Enums(index, public_key, balance, status, withdrawal_credentials) // @Param search query string false "Search for Address, ENS." @@ -672,7 +672,7 @@ func (h *HandlerService) PublicGetValidatorDashboardValidators(w http.ResponseWr // @Tags Validator Dashboard Management // @Accept json // @Produce json -// @Param dashboard_id path string true "The ID of the dashboard." +// @Param dashboard_id path integer true "The ID of the dashboard." // @Param request body handlers.PublicDeleteValidatorDashboardValidators.request true "`validators`: Provide an array of validator indices or public keys that should get removed from the dashboard." // @Success 204 "Validators removed successfully." // @Failure 400 {object} types.ApiErrorResponse @@ -714,7 +714,7 @@ func (h *HandlerService) PublicDeleteValidatorDashboardValidators(w http.Respons // @Tags Validator Dashboard Management // @Accept json // @Produce json -// @Param dashboard_id path string true "The ID of the dashboard." +// @Param dashboard_id path integer true "The ID of the dashboard." // @Param request body handlers.PublicPostValidatorDashboardPublicIds.request true "`name`: Provide a public name for the dashboard
`share_settings`:" // @Success 201 {object} types.ApiDataResponse[types.VDBPublicId] // @Failure 400 {object} types.ApiErrorResponse @@ -768,7 +768,7 @@ func (h *HandlerService) PublicPostValidatorDashboardPublicIds(w http.ResponseWr // @Tags Validator Dashboard Management // @Accept json // @Produce json -// @Param dashboard_id path string true "The ID of the dashboard." +// @Param dashboard_id path integer true "The ID of the dashboard." // @Param public_id path string true "The ID of the public ID." // @Param request body handlers.PublicPutValidatorDashboardPublicId.request true "`name`: Provide a public name for the dashboard
`share_settings`:" // @Success 200 {object} types.ApiDataResponse[types.VDBPublicId] @@ -823,7 +823,7 @@ func (h *HandlerService) PublicPutValidatorDashboardPublicId(w http.ResponseWrit // @Security ApiKeyInHeader || ApiKeyInQuery // @Tags Validator Dashboard Management // @Produce json -// @Param dashboard_id path string true "The ID of the dashboard." +// @Param dashboard_id path integer true "The ID of the dashboard." // @Param public_id path string true "The ID of the public ID." // @Success 204 "Public ID deleted successfully." // @Failure 400 {object} types.ApiErrorResponse @@ -863,8 +863,8 @@ func (h *HandlerService) PublicDeleteValidatorDashboardPublicId(w http.ResponseW // @Tags Validator Dashboard Management // @Accept json // @Produce json -// @Param dashboard_id path string true "The ID of the dashboard." -// @Param request body handlers.PublicPutValidatorDashboardArchiving.request true "request" +// @Param dashboard_id path integer true "The ID of the dashboard." +// @Param request body handlers.PublicPutValidatorDashboardArchiving.request true "`is_archived`: Set to `true` to archive the dashboard, or `false` to unarchive it." // @Success 200 {object} types.ApiDataResponse[types.VDBPostArchivingReturnData] // @Failure 400 {object} types.ApiErrorResponse // @Failure 409 {object} types.ApiErrorResponse "Conflict. The request could not be performed by the server because the authenticated user has already reached their subscription limit." @@ -1043,7 +1043,7 @@ func (h *HandlerService) PublicGetValidatorDashboardSummary(w http.ResponseWrite // @Tags Validator Dashboard // @Produce json // @Param dashboard_id path string true "The ID of the dashboard." -// @Param group_id path string true "The ID of the group." +// @Param group_id path integer true "The ID of the group." // @Param period query string true "Time period to get data for." Enums(all_time, last_30d, last_7d, last_24h, last_1h) // @Param modes query string false "Provide a comma separated list of protocol modes which should be respected for validator calculations. Possible values are `rocket_pool``." // @Success 200 {object} types.GetValidatorDashboardGroupSummaryResponse @@ -1142,7 +1142,7 @@ func (h *HandlerService) PublicGetValidatorDashboardSummaryChart(w http.Response // @Tags Validator Dashboard // @Produce json // @Param dashboard_id path string true "The ID of the dashboard." -// @Param group_id query string false "The ID of the group." +// @Param group_id query integer false "The ID of the group." // @Param duty query string false "Validator duty to get data for." Enums(none, sync, slashed, proposal) Default(none) // @Param period query string true "Time period to get data for." Enums(all_time, last_30d, last_7d, last_24h, last_1h) // @Success 200 {object} types.GetValidatorDashboardSummaryValidatorsResponse @@ -1246,8 +1246,8 @@ func (h *HandlerService) PublicGetValidatorDashboardRewards(w http.ResponseWrite // @Tags Validator Dashboard // @Produce json // @Param dashboard_id path string true "The ID of the dashboard." -// @Param group_id path string true "The ID of the group." -// @Param epoch path string true "The epoch to get data for." +// @Param group_id path integer true "The ID of the group." +// @Param epoch path integer true "The epoch to get data for." // @Param modes query string false "Provide a comma separated list of protocol modes which should be respected for validator calculations. Possible values are `rocket_pool``." // @Success 200 {object} types.GetValidatorDashboardGroupRewardsResponse // @Failure 400 {object} types.ApiErrorResponse @@ -1322,8 +1322,8 @@ func (h *HandlerService) PublicGetValidatorDashboardRewardsChart(w http.Response // @Tags Validator Dashboard // @Produce json // @Param dashboard_id path string true "The ID of the dashboard." -// @Param epoch path string true "The epoch to get data for." -// @Param group_id query string false "The ID of the group." +// @Param epoch path integer true "The epoch to get data for." +// @Param group_id query integer false "The ID of the group." // @Param cursor query string false "Return data for the given cursor value. Pass the `paging.next_cursor`` value of the previous response to navigate to forward, or pass the `paging.prev_cursor`` value of the previous response to navigate to backward." // @Param limit query string false "The maximum number of results that may be returned." // @Param sort query string false "The field you want to sort by. Append with `:desc` for descending order." Enums(validator, reward) @@ -1460,8 +1460,8 @@ func (h *HandlerService) PublicGetValidatorDashboardHeatmap(w http.ResponseWrite // @Tags Validator Dashboard // @Produce json // @Param dashboard_id path string true "The ID of the dashboard." -// @Param group_id path string true "The ID of the group." -// @Param timestamp path string true "The timestamp to get data for." +// @Param group_id path integer true "The ID of the group." +// @Param timestamp path integer true "The timestamp to get data for." // @Param modes query string false "Provide a comma separated list of protocol modes which should be respected for validator calculations. Possible values are `rocket_pool``." // @Param aggregation query string false "Aggregation type to get data for." Enums(epoch, hourly, daily, weekly) Default(hourly) // @Success 200 {object} types.GetValidatorDashboardGroupHeatmapResponse @@ -1903,7 +1903,7 @@ func (h *HandlerService) PublicGetUserNotifications(w http.ResponseWriter, r *ht // @Produce json // @Param network query string false "If set, results will be filtered to only include networks given. Provide a comma separated list." // @Param cursor query string false "Return data for the given cursor value. Pass the `paging.next_cursor`` value of the previous response to navigate to forward, or pass the `paging.prev_cursor`` value of the previous response to navigate to backward." -// @Param limit query string false "The maximum number of results that may be returned." +// @Param limit query integer false "The maximum number of results that may be returned." // @Param sort query string false "The field you want to sort by. Append with `:desc` for descending order." " Enums(chain_id, timestamp, dashboard_id) // @Param search query string false "Search for Dashboard, Group" // @Success 200 {object} types.InternalGetUserNotificationDashboardsResponse @@ -1931,7 +1931,7 @@ func (h *HandlerService) PublicGetUserNotificationDashboards(w http.ResponseWrit } response := types.InternalGetUserNotificationDashboardsResponse{ Data: data, - Paging: *paging, + Paging: *paging, // @Param epoch path strings } returnOk(w, r, response) } @@ -1943,8 +1943,8 @@ func (h *HandlerService) PublicGetUserNotificationDashboards(w http.ResponseWrit // @Tags Notifications // @Produce json // @Param dashboard_id path string true "The ID of the dashboard." -// @Param group_id path string true "The ID of the group." -// @Param epoch path string true "The epoch of the notification." +// @Param group_id path integer true "The ID of the group." +// @Param epoch path integer true "The epoch of the notification." // @Success 200 {object} types.InternalGetUserNotificationsValidatorDashboardResponse // @Failure 400 {object} types.ApiErrorResponse // @Router /users/me/notifications/validator-dashboards/{dashboard_id}/groups/{group_id}/epochs/{epoch} [get] @@ -1976,8 +1976,8 @@ func (h *HandlerService) PublicGetUserNotificationsValidatorDashboard(w http.Res // @Tags Notifications // @Produce json // @Param dashboard_id path string true "The ID of the dashboard." -// @Param group_id path string true "The ID of the group." -// @Param epoch path string true "The epoch of the notification." +// @Param group_id path integer true "The ID of the group." +// @Param epoch path integer true "The epoch of the notification." // @Success 200 {object} types.InternalGetUserNotificationsAccountDashboardResponse // @Failure 400 {object} types.ApiErrorResponse // @Router /users/me/notifications/account-dashboards/{dashboard_id}/groups/{group_id}/epochs/{epoch} [get] @@ -2009,7 +2009,7 @@ func (h *HandlerService) PublicGetUserNotificationsAccountDashboard(w http.Respo // @Tags Notifications // @Produce json // @Param cursor query string false "Return data for the given cursor value. Pass the `paging.next_cursor`` value of the previous response to navigate to forward, or pass the `paging.prev_cursor`` value of the previous response to navigate to backward." -// @Param limit query string false "The maximum number of results that may be returned." +// @Param limit query integer false "The maximum number of results that may be returned." // @Param sort query string false "The field you want to sort by. Append with `:desc` for descending order." Enums(machine_name, threshold, event_type, timestamp) // @Param search query string false "Search for Machine" // @Success 200 {object} types.InternalGetUserNotificationMachinesResponse @@ -2048,7 +2048,7 @@ func (h *HandlerService) PublicGetUserNotificationMachines(w http.ResponseWriter // @Tags Notifications // @Produce json // @Param cursor query string false "Return data for the given cursor value. Pass the `paging.next_cursor`` value of the previous response to navigate to forward, or pass the `paging.prev_cursor`` value of the previous response to navigate to backward." -// @Param limit query string false "The maximum number of results that may be returned." +// @Param limit query integer false "The maximum number of results that may be returned." // @Param sort query string false "The field you want to sort by. Append with `:desc` for descending order." Enums(client_name, timestamp) // @Param search query string false "Search for Client" // @Success 200 {object} types.InternalGetUserNotificationClientsResponse @@ -2087,7 +2087,7 @@ func (h *HandlerService) PublicGetUserNotificationClients(w http.ResponseWriter, // @Tags Notifications // @Produce json // @Param cursor query string false "Return data for the given cursor value. Pass the `paging.next_cursor`` value of the previous response to navigate to forward, or pass the `paging.prev_cursor`` value of the previous response to navigate to backward." -// @Param limit query string false "The maximum number of results that may be returned." +// @Param limit query integer false "The maximum number of results that may be returned." // @Param sort query string false "The field you want to sort by. Append with `:desc` for descending order." Enums(timestamp, event_type, node_address) // @Param search query string false "Search for TODO" // @Success 200 {object} types.InternalGetUserNotificationRocketPoolResponse @@ -2126,7 +2126,7 @@ func (h *HandlerService) PublicGetUserNotificationRocketPool(w http.ResponseWrit // @Tags Notifications // @Produce json // @Param cursor query string false "Return data for the given cursor value. Pass the `paging.next_cursor`` value of the previous response to navigate to forward, or pass the `paging.prev_cursor`` value of the previous response to navigate to backward." -// @Param limit query string false "The maximum number of results that may be returned." +// @Param limit query integer false "The maximum number of results that may be returned." // @Param sort query string false "The field you want to sort by. Append with `:desc` for descending order." Enums(timestamp, event_type) // @Param search query string false "Search for TODO" // @Success 200 {object} types.InternalGetUserNotificationNetworksResponse @@ -2190,7 +2190,7 @@ func (h *HandlerService) PublicGetUserNotificationSettings(w http.ResponseWriter // @Tags Notification Settings // @Accept json // @Produce json -// @Param request body types.NotificationSettingsGeneral true "Notification settings" +// @Param request body types.NotificationSettingsGeneral true "Description TODO" // @Success 200 {object} types.InternalPutUserNotificationSettingsGeneralResponse // @Failure 400 {object} types.ApiErrorResponse // @Router /users/me/notifications/settings/general [put] @@ -2211,7 +2211,6 @@ func (h *HandlerService) PublicPutUserNotificationSettingsGeneral(w http.Respons checkMinMax(&v, req.MachineMemoryUsageThreshold, 0, 1, "machine_memory_usage_threshold") checkMinMax(&v, req.RocketPoolMaxCollateralThreshold, 0, 1, "rocket_pool_max_collateral_threshold") checkMinMax(&v, req.RocketPoolMinCollateralThreshold, 0, 1, "rocket_pool_min_collateral_threshold") - // TODO: check validity of clients if v.hasErrors() { handleErr(w, r, v) return @@ -2235,7 +2234,7 @@ func (h *HandlerService) PublicPutUserNotificationSettingsGeneral(w http.Respons // @Accept json // @Produce json // @Param network path string true "The networks name or chain ID." -// @Param request body types.NotificationSettingsNetwork true "Notification settings" +// @Param request body types.NotificationSettingsNetwork true "Description Todo" // @Success 200 {object} types.InternalPutUserNotificationSettingsNetworksResponse // @Failure 400 {object} types.ApiErrorResponse // @Router /users/me/notifications/settings/networks/{network} [put] @@ -2280,7 +2279,7 @@ func (h *HandlerService) PublicPutUserNotificationSettingsNetworks(w http.Respon // @Accept json // @Produce json // @Param paired_device_id path string true "The paired device ID." -// @Param request body handlers.PublicPutUserNotificationSettingsPairedDevices.request true "Notification settings" +// @Param request body handlers.PublicPutUserNotificationSettingsPairedDevices.request true "Description TODO" // @Success 200 {object} types.InternalPutUserNotificationSettingsPairedDevicesResponse // @Failure 400 {object} types.ApiErrorResponse // @Router /users/me/notifications/settings/paired-devices/{paired_device_id} [put] @@ -2355,6 +2354,49 @@ func (h *HandlerService) PublicDeleteUserNotificationSettingsPairedDevices(w htt returnNoContent(w, r) } +// PublicPutUserNotificationSettingsClient godoc +// +// @Description Update client notification settings for the authenticated user. When a client is subscribed, notifications will be sent when a new version is available. +// @Security ApiKeyInHeader || ApiKeyInQuery +// @Tags Notification Settings +// @Accept json +// @Produce json +// @Param client_id path integer true "The ID of the client." +// @Param request body handlers.PublicPutUserNotificationSettingsClient.request true "`is_subscribed`: Set to `true` to subscribe to notifications; set to `false` to unsubscribe." +// @Success 200 {object} types.InternalPutUserNotificationSettingsClientResponse +// @Failure 400 {object} types.ApiErrorResponse +// @Router /users/me/notifications/settings/clients/{client_id} [put] +func (h *HandlerService) PublicPutUserNotificationSettingsClient(w http.ResponseWriter, r *http.Request) { + var v validationError + userId, err := GetUserIdByContext(r) + if err != nil { + handleErr(w, r, err) + return + } + type request struct { + IsSubscribed bool `json:"is_subscribed"` + } + var req request + if err := v.checkBody(&req, r); err != nil { + handleErr(w, r, err) + return + } + clientId := v.checkUint(mux.Vars(r)["client_id"], "client_id") + if v.hasErrors() { + handleErr(w, r, v) + return + } + data, err := h.dai.UpdateNotificationSettingsClients(r.Context(), userId, clientId, req.IsSubscribed) + if err != nil { + handleErr(w, r, err) + return + } + response := types.InternalPutUserNotificationSettingsClientResponse{ + Data: *data, + } + returnOk(w, r, response) +} + // PublicGetUserNotificationSettingsDashboards godoc // // @Description Get a list of notification settings for the dashboards of the authenticated user. @@ -2362,7 +2404,7 @@ func (h *HandlerService) PublicDeleteUserNotificationSettingsPairedDevices(w htt // @Tags Notification Settings // @Produce json // @Param cursor query string false "Return data for the given cursor value. Pass the `paging.next_cursor`` value of the previous response to navigate to forward, or pass the `paging.prev_cursor`` value of the previous response to navigate to backward." -// @Param limit query string false "The maximum number of results that may be returned." +// @Param limit query integer false "The maximum number of results that may be returned." // @Param sort query string false "The field you want to sort by. Append with `:desc` for descending order." Enums (dashboard_id, group_name) // @Param search query string false "Search for Dashboard, Group" // @Success 200 {object} types.InternalGetUserNotificationSettingsDashboardsResponse @@ -2402,7 +2444,7 @@ func (h *HandlerService) PublicGetUserNotificationSettingsDashboards(w http.Resp // @Accept json // @Produce json // @Param dashboard_id path string true "The ID of the dashboard." -// @Param group_id path string true "The ID of the group." +// @Param group_id path integer true "The ID of the group." // @Param request body types.NotificationSettingsValidatorDashboard true "Notification settings" // @Success 200 {object} types.InternalPutUserNotificationSettingsValidatorDashboardResponse // @Failure 400 {object} types.ApiErrorResponse @@ -2441,7 +2483,7 @@ func (h *HandlerService) PublicPutUserNotificationSettingsValidatorDashboard(w h // @Accept json // @Produce json // @Param dashboard_id path string true "The ID of the dashboard." -// @Param group_id path string true "The ID of the group." +// @Param group_id path integer true "The ID of the group." // @Param request body handlers.PublicPutUserNotificationSettingsAccountDashboard.request true "Notification settings" // @Success 200 {object} types.InternalPutUserNotificationSettingsAccountDashboardResponse // @Failure 400 {object} types.ApiErrorResponse diff --git a/backend/pkg/api/router.go b/backend/pkg/api/router.go index 16ab4c0b2..864cb69d8 100644 --- a/backend/pkg/api/router.go +++ b/backend/pkg/api/router.go @@ -329,6 +329,7 @@ func addNotificationRoutes(hs *handlers.HandlerService, publicRouter, internalRo {http.MethodPut, "/settings/networks/{network}", hs.PublicPutUserNotificationSettingsNetworks, hs.InternalPutUserNotificationSettingsNetworks}, {http.MethodPut, "/settings/paired-devices/{paired_device_id}", hs.PublicPutUserNotificationSettingsPairedDevices, hs.InternalPutUserNotificationSettingsPairedDevices}, {http.MethodDelete, "/settings/paired-devices/{paired_device_id}", hs.PublicDeleteUserNotificationSettingsPairedDevices, hs.InternalDeleteUserNotificationSettingsPairedDevices}, + {http.MethodPut, "/settings/clients/{client_id}", hs.PublicPutUserNotificationSettingsClient, hs.InternalPutUserNotificationSettingsClient}, {http.MethodGet, "/settings/dashboards", hs.PublicGetUserNotificationSettingsDashboards, hs.InternalGetUserNotificationSettingsDashboards}, {http.MethodPost, "/test-email", hs.PublicPostUserNotificationsTestEmail, hs.InternalPostUserNotificationsTestEmail}, {http.MethodPost, "/test-push", hs.PublicPostUserNotificationsTestPush, hs.InternalPostUserNotificationsTestPush}, diff --git a/backend/pkg/api/types/notifications.go b/backend/pkg/api/types/notifications.go index 5d80efc89..4d512ff5a 100644 --- a/backend/pkg/api/types/notifications.go +++ b/backend/pkg/api/types/notifications.go @@ -162,12 +162,15 @@ type NotificationPairedDevice struct { } type InternalPutUserNotificationSettingsPairedDevicesResponse ApiDataResponse[NotificationPairedDevice] -type NotificationSettingsClients struct { +type NotificationSettingsClient struct { Id uint64 `json:"id"` Name string `json:"name"` Category string `json:"category"` IsSubscribed bool `json:"is_subscribed"` } + +type InternalPutUserNotificationSettingsClientResponse ApiDataResponse[NotificationSettingsClient] + type NotificationSettingsGeneral struct { DoNotDisturbTimestamp int64 `json:"do_not_disturb_timestamp"` // notifications are disabled until this timestamp IsEmailNotificationsEnabled bool `json:"is_email_notifications_enabled"` @@ -189,10 +192,10 @@ type NotificationSettingsGeneral struct { } type InternalPutUserNotificationSettingsGeneralResponse ApiDataResponse[NotificationSettingsGeneral] type NotificationSettings struct { - GeneralSettings NotificationSettingsGeneral `json:"general_settings"` - Networks []NotificationNetwork `json:"networks"` - PairedDevices []NotificationPairedDevice `json:"paired_devices"` - Clients []NotificationSettingsClients `json:"clients"` + GeneralSettings NotificationSettingsGeneral `json:"general_settings"` + Networks []NotificationNetwork `json:"networks"` + PairedDevices []NotificationPairedDevice `json:"paired_devices"` + Clients []NotificationSettingsClient `json:"clients"` } type InternalGetUserNotificationSettingsResponse ApiDataResponse[NotificationSettings] From 1b7d591b99887e569067d4080006ffb59ed28022 Mon Sep 17 00:00:00 2001 From: LUCCA DUKIC <109136188+LuccaBitfly@users.noreply.github.com> Date: Thu, 19 Sep 2024 12:30:19 +0200 Subject: [PATCH 36/94] (BEDS-452) typescript conversion --- frontend/types/api/notifications.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/frontend/types/api/notifications.ts b/frontend/types/api/notifications.ts index 0d9f53898..57a2900d4 100644 --- a/frontend/types/api/notifications.ts +++ b/frontend/types/api/notifications.ts @@ -156,6 +156,13 @@ export interface NotificationPairedDevice { is_notifications_enabled: boolean; } export type InternalPutUserNotificationSettingsPairedDevicesResponse = ApiDataResponse; +export interface NotificationSettingsClient { + id: number /* uint64 */; + name: string; + category: string; + is_subscribed: boolean; +} +export type InternalPutUserNotificationSettingsClientResponse = ApiDataResponse; export interface NotificationSettingsGeneral { do_not_disturb_timestamp: number /* int64 */; // notifications are disabled until this timestamp is_email_notifications_enabled: boolean; @@ -167,7 +174,6 @@ export interface NotificationSettingsGeneral { machine_cpu_usage_threshold: number /* float64 */; is_machine_memory_usage_subscribed: boolean; machine_memory_usage_threshold: number /* float64 */; - subscribed_clients: string[]; is_rocket_pool_new_reward_round_subscribed: boolean; is_rocket_pool_max_collateral_subscribed: boolean; rocket_pool_max_collateral_threshold: number /* float64 */; @@ -179,6 +185,7 @@ export interface NotificationSettings { general_settings: NotificationSettingsGeneral; networks: NotificationNetwork[]; paired_devices: NotificationPairedDevice[]; + clients: NotificationSettingsClient[]; } export type InternalGetUserNotificationSettingsResponse = ApiDataResponse; export interface NotificationSettingsValidatorDashboard { From f29cd091cd21b671a4ba63d7b94b83e10e5322ad Mon Sep 17 00:00:00 2001 From: LUCCA DUKIC <109136188+LuccaBitfly@users.noreply.github.com> Date: Thu, 19 Sep 2024 12:48:14 +0200 Subject: [PATCH 37/94] (BEDS-452) defeat frontend typecheck --- frontend/utils/mock.ts | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/frontend/utils/mock.ts b/frontend/utils/mock.ts index f942d8fbb..0f5e8bc6e 100644 --- a/frontend/utils/mock.ts +++ b/frontend/utils/mock.ts @@ -443,6 +443,32 @@ export function simulateAPIresponseAboutNetworkList(): ApiDataResponse< export function mockManageNotificationsGeneral(): InternalGetUserNotificationSettingsResponse { return { data: { + clients: [ + { + category: 'EL', + id: 1, + is_subscribed: true, + name: 'EL Client 1', + }, + { + category: 'CL', + id: 2, + is_subscribed: false, + name: 'CL Client 1', + }, + { + category: 'other', + id: 3, + is_subscribed: true, + name: 'Other Clien 1', + }, + { + category: 'other', + id: 4, + is_subscribed: false, + name: 'Other Clien 2', + }, + ], general_settings: { do_not_disturb_timestamp: 9000, is_email_notifications_enabled: false, @@ -459,7 +485,6 @@ export function mockManageNotificationsGeneral(): InternalGetUserNotificationSet machine_storage_usage_threshold: 80, rocket_pool_max_collateral_threshold: 29823, rocket_pool_min_collateral_threshold: 123, - subscribed_clients: [], }, networks: [], paired_devices: [ From 17918e6e95e069471977143416185e184619abe0 Mon Sep 17 00:00:00 2001 From: Lucca <109136188+LuccaBitfly@users.noreply.github.com> Date: Thu, 19 Sep 2024 13:12:31 +0200 Subject: [PATCH 38/94] (BEDS-477) add url to clients notification table (#872) --- backend/pkg/api/types/notifications.go | 1 + frontend/types/api/notifications.ts | 1 + 2 files changed, 2 insertions(+) diff --git a/backend/pkg/api/types/notifications.go b/backend/pkg/api/types/notifications.go index 27bf585be..af31fd56a 100644 --- a/backend/pkg/api/types/notifications.go +++ b/backend/pkg/api/types/notifications.go @@ -111,6 +111,7 @@ type InternalGetUserNotificationMachinesResponse ApiPagingResponse[NotificationM type NotificationClientsTableRow struct { ClientName string `json:"client_name"` Version string `json:"version"` + Url string `json:"url"` Timestamp int64 `json:"timestamp"` } diff --git a/frontend/types/api/notifications.ts b/frontend/types/api/notifications.ts index 0d9f53898..1ff93b79b 100644 --- a/frontend/types/api/notifications.ts +++ b/frontend/types/api/notifications.ts @@ -107,6 +107,7 @@ export type InternalGetUserNotificationMachinesResponse = ApiPagingResponse; From e0f0a76c6a5c0c6ec2f3eafe0e5c3bff79c069a0 Mon Sep 17 00:00:00 2001 From: benji-bitfly Date: Fri, 20 Sep 2024 08:48:40 +0200 Subject: [PATCH 39/94] feat: add `notifications overview` See: BEDS-335 --- frontend/components/BcFeatureFlag.vue | 35 +++ frontend/components/bc/BcButtonText.vue | 19 ++ .../notifications/NotificationsOverview.vue | 247 ++++++++++++++++++ frontend/locales/en.json | 30 ++- frontend/pages/notifications.vue | 4 +- .../useNotificationsOverviewStore.ts | 35 +++ frontend/types/customFetch.ts | 6 + 7 files changed, 374 insertions(+), 2 deletions(-) create mode 100644 frontend/components/BcFeatureFlag.vue create mode 100644 frontend/components/bc/BcButtonText.vue create mode 100644 frontend/components/notifications/NotificationsOverview.vue create mode 100644 frontend/stores/notifications/useNotificationsOverviewStore.ts diff --git a/frontend/components/BcFeatureFlag.vue b/frontend/components/BcFeatureFlag.vue new file mode 100644 index 000000000..53c6c6673 --- /dev/null +++ b/frontend/components/BcFeatureFlag.vue @@ -0,0 +1,35 @@ + + + + + diff --git a/frontend/components/bc/BcButtonText.vue b/frontend/components/bc/BcButtonText.vue new file mode 100644 index 000000000..60eb5acc1 --- /dev/null +++ b/frontend/components/bc/BcButtonText.vue @@ -0,0 +1,19 @@ + + + + + diff --git a/frontend/components/notifications/NotificationsOverview.vue b/frontend/components/notifications/NotificationsOverview.vue new file mode 100644 index 000000000..7eb583cd4 --- /dev/null +++ b/frontend/components/notifications/NotificationsOverview.vue @@ -0,0 +1,247 @@ + + + + + diff --git a/frontend/locales/en.json b/frontend/locales/en.json index 302cfa091..9a989b149 100644 --- a/frontend/locales/en.json +++ b/frontend/locales/en.json @@ -20,6 +20,7 @@ }, "common": { "account": "Account | Accounts", + "active": "Active", "address": "Address | Addresses", "age": "Age", "and_more": "and {count} more", @@ -34,6 +35,7 @@ "deselected": "Not selected", "earned": "Earned", "edit": "Edit", + "email": "Email", "ens_address": "ENS address | ENS addresses", "epoch": "Epoch | Epochs", "erc20token": "ERC-20 Token | ERC-20 Tokens", @@ -45,6 +47,7 @@ "graffiti": "Graffiti | Graffitis", "id": "ID", "in_day": "In one day | In {count} days", + "inactive": "Inactive", "index": "Index", "last_x": "Last {duration}", "live": "Live", @@ -68,8 +71,12 @@ "true": "True", "tx_batch": "Tx Batch | Tx Batches", "unavailable": "Unavailable", + "units": { + "per_day": "per day" + }, "upcoming": "Upcoming", - "validator": "Validator | Validators" + "validator": "Validator | Validators", + "webhook": "Webhook" }, "cookies": { "accept_all": "Accept All", @@ -692,6 +699,27 @@ "subscriptions": "Network ({count} Subscription) | Validators ({count} Subscriptions)" } }, + "overview": { + "email_activate": "Click here to activate Email notifications", + "email_tooltip": "Your current limit is { limit } emails per day. Your email limit resets in { hours } hours. Upgrade to premium for more.", + "headers": { + "account_groups": "Most notified account groups", + "email_notifications": "Email Notifications", + "most_notifications_24h": "Most Notifications in 24h", + "most_notifications_30d": "Most Notifications in 30d", + "push_notifications": "Push Notifications", + "validator_groups": "Most notified validator groups" + }, + "notifications_activate_premium": { + "_link": "here", + "template": "Click {_link} to activate" + }, + "notifications_download_app": { + "_link": "mobile app", + "template": "Download the {_link} to activate" + }, + "push": "Push" + }, "subscriptions": { "accounts": { "all": { diff --git a/frontend/pages/notifications.vue b/frontend/pages/notifications.vue index 5d591e793..d1701d51d 100644 --- a/frontend/pages/notifications.vue +++ b/frontend/pages/notifications.vue @@ -84,7 +84,9 @@ const openManageNotifications = () => { { + const data = ref() + return { data } +}) + +export function useNotificationsDashboardOverviewStore() { + const { fetch } = useCustomFetch() + const { data: overview } = storeToRefs(notificationsOverviewStore()) + + async function refreshOverview() { + try { + const res = await fetch( + API_PATH.NOTIFICATIONS_OVERVIEW, + ) + overview.value = res.data + + return overview.value + } + catch (e) { + overview.value = undefined + throw e + } + } + + return { + overview, + refreshOverview, + } +} diff --git a/frontend/types/customFetch.ts b/frontend/types/customFetch.ts index 18b2d3ea1..b7cf641b4 100644 --- a/frontend/types/customFetch.ts +++ b/frontend/types/customFetch.ts @@ -45,6 +45,7 @@ export enum API_PATH { NOTIFICATIONS_MACHINE = '/notifications/machines', NOTIFICATIONS_MANAGEMENT_GENERAL = '/notifications/managementGeneral', NOTIFICATIONS_NETWORK = '/notifications/networks', + NOTIFICATIONS_OVERVIEW = '/notifications', NOTIFICATIONS_TEST_EMAIL = '/notifications/test_email', NOTIFICATIONS_TEST_PUSH = '/notifications/test_push', NOTIFICATIONS_TEST_WEBHOOK = '/users/me/notifications/test-webhook', @@ -298,6 +299,11 @@ export const mapping: Record = { [API_PATH.NOTIFICATIONS_NETWORK]: { path: '/users/me/notifications/networks', }, + [API_PATH.NOTIFICATIONS_OVERVIEW]: { + method: 'GET', + mock: false, + path: '/users/me/notifications', + }, [API_PATH.NOTIFICATIONS_TEST_EMAIL]: { method: 'POST', mock: true, From cdffa1ef8f5c23a19200c6ea1e2108494f1d2ff5 Mon Sep 17 00:00:00 2001 From: MarcelBitfly <174338434+marcel-bitfly@users.noreply.github.com> Date: Mon, 9 Sep 2024 18:08:28 +0200 Subject: [PATCH 40/94] chore(eslint): add rule to sort `conventional commit scopes` --- frontend/eslint.config.mjs | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/frontend/eslint.config.mjs b/frontend/eslint.config.mjs index 08474dd9e..ec277885d 100644 --- a/frontend/eslint.config.mjs +++ b/frontend/eslint.config.mjs @@ -114,6 +114,16 @@ export default withNuxt({ ...eslintPluginJsonc.configs['flat/recommended-with-json'], { rules: { + 'jsonc/sort-array-values': [ + 'warn', + { + order: { + natural: true, + type: 'asc', + }, + pathPattern: 'conventionalCommits.scopes', + }, + ], 'jsonc/sort-keys': [ 'error', 'asc', From 9c38c2f91db699e344f13e9c89efc8dd6172511d Mon Sep 17 00:00:00 2001 From: MarcelBitfly <174338434+marcel-bitfly@users.noreply.github.com> Date: Fri, 20 Sep 2024 12:47:10 +0200 Subject: [PATCH 41/94] refactor(feature-flags): manage via `composable` With a `composable` `feature flags` can be used in nearly every `nuxt function`. --- frontend/.vscode/settings.json | 10 ++++---- frontend/components/BcFeatureFlag.vue | 29 ++++------------------ frontend/composables/useFeatureFlag.ts | 33 ++++++++++++++++++++++++++ frontend/types/feature-flags.ts | 6 +++++ 4 files changed, 49 insertions(+), 29 deletions(-) create mode 100644 frontend/composables/useFeatureFlag.ts create mode 100644 frontend/types/feature-flags.ts diff --git a/frontend/.vscode/settings.json b/frontend/.vscode/settings.json index a97dbcf2a..e5ba8997f 100644 --- a/frontend/.vscode/settings.json +++ b/frontend/.vscode/settings.json @@ -1,17 +1,19 @@ { "conventionalCommits.scopes": [ + "DashboardChartSummaryChartFilter", + "DashboardGroupManagementModal", + "DashboardValidatorManagmentModal", "checkout", "ci", "customFetch", - "DashboardChartSummaryChartFilter", - "DashboardGroupManagementModal", "eslint", + "feature-flags", "git", "i18n", "mainHeader", + "notifications", "qrCode", - "vscode", - "DashboardValidatorManagmentModal" + "vscode" ], "editor.codeActionsOnSave": { "source.fixAll.eslint": "always" diff --git a/frontend/components/BcFeatureFlag.vue b/frontend/components/BcFeatureFlag.vue index 53c6c6673..5d69eb435 100644 --- a/frontend/components/BcFeatureFlag.vue +++ b/frontend/components/BcFeatureFlag.vue @@ -1,35 +1,14 @@ diff --git a/frontend/composables/useFeatureFlag.ts b/frontend/composables/useFeatureFlag.ts new file mode 100644 index 000000000..6307dfaf2 --- /dev/null +++ b/frontend/composables/useFeatureFlag.ts @@ -0,0 +1,33 @@ +import { warn } from 'vue' +import type { FeatureFlag } from '~/types/feature-flags' + +export const useFeatureFlag = () => { + type Environment = 'development' | 'production' | 'staging' + + const currentEnvironment = useRuntimeConfig().public.deploymentType as Environment + if (!currentEnvironment) { + warn('Environment variable `deploymentType` is not set.') + } + + const staging: FeatureFlag[] = [ 'feature-notifications' ] + const development: FeatureFlag[] + = [ + ...staging, + 'feature-account_dashboards', + 'feature-user_settings', + ] + const featureCatalog: Record = { + development, + production: [], + staging, + } + + const activeFeatures = featureCatalog[currentEnvironment] + + const has = (feature: FeatureFlag) => activeFeatures.includes(feature) + + return { + activeFeatures, + has, + } +} diff --git a/frontend/types/feature-flags.ts b/frontend/types/feature-flags.ts new file mode 100644 index 000000000..90c2d8d4f --- /dev/null +++ b/frontend/types/feature-flags.ts @@ -0,0 +1,6 @@ +export type FeatureFlag = (typeof FEATURE_FLAGS)[number] +const FEATURE_FLAGS = [ + 'feature-account_dashboards', + 'feature-notifications', + 'feature-user_settings', +] as const From 789fecc2951c8a95fda984d65640d3f5836ad34f Mon Sep 17 00:00:00 2001 From: MarcelBitfly <174338434+marcel-bitfly@users.noreply.github.com> Date: Fri, 20 Sep 2024 13:01:00 +0200 Subject: [PATCH 42/94] feat(notifications): unlock `route` for `staging` --- frontend/components/bc/header/MegaMenu.vue | 1677 ++++++++--------- .../components/dashboard/DashboardHeader.vue | 11 +- frontend/composables/useFeatureFlag.ts | 3 +- frontend/middleware/redirect.global.ts | 11 +- 4 files changed, 820 insertions(+), 882 deletions(-) diff --git a/frontend/components/bc/header/MegaMenu.vue b/frontend/components/bc/header/MegaMenu.vue index abb846647..9af0b4b09 100644 --- a/frontend/components/bc/header/MegaMenu.vue +++ b/frontend/components/bc/header/MegaMenu.vue @@ -1,61 +1,10 @@ +