merge staging->main

backend/go.mod

@@ -193,6 +193,7 @@ require (
 	github.com/jbenet/goprocess v0.1.4 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/k3a/html2text v1.2.1 // indirect
 	github.com/klauspost/cpuid/v2 v2.2.7 // indirect
 	github.com/libp2p/go-buffer-pool v0.1.0 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect

backend/go.sum

@@ -597,6 +597,8 @@ github.com/juliangruber/go-intersect v1.1.0/go.mod h1:WMau+1kAmnlQnKiikekNJbtGtf
 github.com/jung-kurt/gofpdf v1.0.0/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
 github.com/jung-kurt/gofpdf v1.16.2 h1:jgbatWHfRlPYiK85qgevsZTHviWXKwB1TTiKdz5PtRc=
 github.com/jung-kurt/gofpdf v1.16.2/go.mod h1:1hl7y57EsiPAkLbOwzpzqgx1A30nQCk/YmFV8S2vmK0=
+github.com/k3a/html2text v1.2.1 h1:nvnKgBvBR/myqrwfLuiqecUtaK1lB9hGziIJKatNFVY=
+github.com/k3a/html2text v1.2.1/go.mod h1:ieEXykM67iT8lTvEWBh6fhpH4B23kB9OMKPdIBmgUqA=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8=
 github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8=

backend/pkg/api/data_access/dummy.go

@@ -258,6 +258,10 @@ func (d *DummyService) RemoveValidatorDashboardGroup(ctx context.Context, dashbo
 	return nil
 }
 
+func (d *DummyService) RemoveValidatorDashboardGroupValidators(ctx context.Context, dashboardId t.VDBIdPrimary, groupId uint64) error {
+	return nil
+}
+
 func (d *DummyService) GetValidatorDashboardGroupExists(ctx context.Context, dashboardId t.VDBIdPrimary, groupId uint64) (bool, error) {
 	return true, nil
 }

backend/pkg/api/data_access/general.go

@@ -2,6 +2,7 @@ package dataaccess
 
 import (
 	"context"
+	"fmt"
 
 	"github.com/doug-martin/goqu/v9"
 	"github.com/doug-martin/goqu/v9/exp"
@@ -52,7 +53,7 @@ func (d *DataAccessService) GetNamesAndEnsForAddresses(ctx context.Context, addr
 // helper function to sort and apply pagination to a query
 // 1st param is the list of all columns necessary to sort the table deterministically; it defines their precedence and sort direction
 // 2nd param is the requested sort column; it may or may not be part of the default columns (if it is, you don't have to specify the cursor limit again)
-func applySortAndPagination(defaultColumns []types.SortColumn, primary types.SortColumn, cursor types.GenericCursor) ([]exp.OrderedExpression, exp.Expression) {
+func applySortAndPagination(defaultColumns []types.SortColumn, primary types.SortColumn, cursor types.GenericCursor) ([]exp.OrderedExpression, exp.Expression, error) {
 	// prepare ordering columns; always need all columns to ensure consistent ordering
 	queryOrderColumns := make([]types.SortColumn, 0, len(defaultColumns))
 	queryOrderColumns = append(queryOrderColumns, primary)
@@ -90,22 +91,43 @@ func applySortAndPagination(defaultColumns []types.SortColumn, primary types.Sor
 			var colWhere exp.Expression
 
 			// current convention is opposite of the psql default (ASC: nulls first, DESC: nulls last)
-			colWhere = goqu.Or(column.Column.Lt(column.Offset), column.Column.IsNull())
-			if !column.Desc {
-				colWhere = column.Column.Gt(column.Offset)
-				if column.Offset == nil {
-					colWhere = goqu.Or(colWhere, column.Column.IsNull())
+			if column.Desc {
+				if column.Offset == nil && queryWhere == nil {
+					continue
 				}
-			}
 
-			if queryWhere == nil {
-				queryWhere = colWhere
+				colWhere = goqu.Or(column.Column.Lt(column.Offset), column.Column.IsNull())
+
+				if queryWhere == nil {
+					queryWhere = colWhere
+				} else {
+					if column.Offset == nil {
+						queryWhere = goqu.And(column.Column.IsNull(), queryWhere)
+					} else {
+						queryWhere = goqu.And(column.Column.Eq(column.Offset), queryWhere)
+						queryWhere = goqu.Or(colWhere, queryWhere)
+					}
+				}
 			} else {
-				queryWhere = goqu.And(column.Column.Eq(column.Offset), queryWhere)
-				queryWhere = goqu.Or(colWhere, queryWhere)
+				if column.Offset == nil {
+					colWhere = column.Column.IsNotNull()
+				} else {
+					colWhere = column.Column.Gt(column.Offset)
+				}
+
+				if queryWhere == nil {
+					queryWhere = colWhere
+				} else {
+					queryWhere = goqu.And(column.Column.Eq(column.Offset), queryWhere)
+					queryWhere = goqu.Or(colWhere, queryWhere)
+				}
 			}
 		}
+
+		if queryWhere == nil {
+			return nil, nil, fmt.Errorf("cursor given for descending order but all offset are nil meaning no data after it")
+		}
 	}
 
-	return queryOrder, queryWhere
+	return queryOrder, queryWhere, nil
 }

backend/pkg/api/data_access/notifications.go

@@ -353,7 +353,10 @@ func (d *DataAccessService) GetDashboardNotifications(ctx context.Context, userI
 		{Column: enums.NotificationsDashboardsColumns.GroupId.ToExpr(), Desc: false, Offset: currentCursor.GroupId},
 		{Column: enums.NotificationsDashboardsColumns.ChainId.ToExpr(), Desc: true, Offset: currentCursor.ChainId},
 	}
-	order, directions := applySortAndPagination(defaultColumns, t.SortColumn{Column: colSort.Column.ToExpr(), Desc: colSort.Desc}, currentCursor.GenericCursor)
+	order, directions, err := applySortAndPagination(defaultColumns, t.SortColumn{Column: colSort.Column.ToExpr(), Desc: colSort.Desc}, currentCursor.GenericCursor)
+	if err != nil {
+		return nil, nil, err
+	}
 	unionQuery = unionQuery.Order(order...)
 	if directions != nil {
 		unionQuery = unionQuery.Where(directions)
@@ -751,7 +754,10 @@ func (d *DataAccessService) GetMachineNotifications(ctx context.Context, userId
 		offset = currentCursor.EventThreshold
 	}
 
-	order, directions := applySortAndPagination(defaultColumns, t.SortColumn{Column: colSort.Column.ToExpr(), Desc: colSort.Desc, Offset: offset}, currentCursor.GenericCursor)
+	order, directions, err := applySortAndPagination(defaultColumns, t.SortColumn{Column: colSort.Column.ToExpr(), Desc: colSort.Desc, Offset: offset}, currentCursor.GenericCursor)
+	if err != nil {
+		return nil, nil, err
+	}
 	ds = ds.Order(order...)
 	if directions != nil {
 		ds = ds.Where(directions)
@@ -863,7 +869,10 @@ func (d *DataAccessService) GetClientNotifications(ctx context.Context, userId u
 		{Column: enums.NotificationsClientsColumns.Timestamp.ToExpr(), Desc: true, Offset: currentCursor.Ts},
 		{Column: enums.NotificationsClientsColumns.ClientName.ToExpr(), Desc: false, Offset: currentCursor.Client},
 	}
-	order, directions := applySortAndPagination(defaultColumns, t.SortColumn{Column: colSort.Column.ToExpr(), Desc: colSort.Desc}, currentCursor.GenericCursor)
+	order, directions, err := applySortAndPagination(defaultColumns, t.SortColumn{Column: colSort.Column.ToExpr(), Desc: colSort.Desc}, currentCursor.GenericCursor)
+	if err != nil {
+		return nil, nil, err
+	}
 	ds = ds.Order(order...)
 	if directions != nil {
 		ds = ds.Where(directions)
@@ -961,7 +970,10 @@ func (d *DataAccessService) GetNetworkNotifications(ctx context.Context, userId
 		{Column: enums.NotificationNetworksColumns.Network.ToExpr(), Desc: false, Offset: currentCursor.Network},
 		{Column: enums.NotificationNetworksColumns.EventType.ToExpr(), Desc: false, Offset: currentCursor.EventType},
 	}
-	order, directions := applySortAndPagination(defaultColumns, t.SortColumn{Column: colSort.Column.ToExpr(), Desc: colSort.Desc}, currentCursor.GenericCursor)
+	order, directions, err := applySortAndPagination(defaultColumns, t.SortColumn{Column: colSort.Column.ToExpr(), Desc: colSort.Desc}, currentCursor.GenericCursor)
+	if err != nil {
+		return nil, nil, err
+	}
 	ds = ds.Order(order...)
 	if directions != nil {
 		ds = ds.Where(directions)
@@ -1149,7 +1161,7 @@ func (d *DataAccessService) GetNotificationSettings(ctx context.Context, userId
 			device_name,
 			COALESCE(notify_enabled, false) AS notify_enabled
 		FROM users_devices
-		WHERE user_id = $1`, userId)
+		WHERE user_id = $1 AND notification_token IS NOT NULL AND LENGTH(notification_token) > 0`, userId)
 		if err != nil {
 			return fmt.Errorf(`error retrieving data for notifications paired devices: %w`, err)
 		}

backend/pkg/api/data_access/vdb.go

@@ -25,6 +25,7 @@ type ValidatorDashboardRepository interface {
 	CreateValidatorDashboardGroup(ctx context.Context, dashboardId t.VDBIdPrimary, name string) (*t.VDBPostCreateGroupData, error)
 	UpdateValidatorDashboardGroup(ctx context.Context, dashboardId t.VDBIdPrimary, groupId uint64, name string) (*t.VDBPostCreateGroupData, error)
 	RemoveValidatorDashboardGroup(ctx context.Context, dashboardId t.VDBIdPrimary, groupId uint64) error
+	RemoveValidatorDashboardGroupValidators(ctx context.Context, dashboardId t.VDBIdPrimary, groupId uint64) error
 	GetValidatorDashboardGroupCount(ctx context.Context, dashboardId t.VDBIdPrimary) (uint64, error)
 	GetValidatorDashboardGroupExists(ctx context.Context, dashboardId t.VDBIdPrimary, groupId uint64) (bool, error)
 

backend/pkg/api/data_access/vdb_blocks.go

@@ -131,9 +131,6 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das
 		InnerJoin(blocks, goqu.On(
 			blocks.Col("proposer").Eq(validators.Col("validator_index")),
 		)).
-		LeftJoin(goqu.T("consensus_payloads").As("cp"), goqu.On(
-			blocks.Col("slot").Eq(goqu.I("cp.slot")),
-		)).
 		LeftJoin(goqu.T("execution_payloads").As("ep"), goqu.On(
 			blocks.Col("exec_block_hash").Eq(goqu.I("ep.block_hash")),
 		)).
@@ -145,6 +142,7 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das
 					goqu.I("relays_blocks.exec_block_hash"),
 					goqu.I("relays_blocks.proposer_fee_recipient"),
 					goqu.MAX(goqu.I("relays_blocks.value")).As("value")).
+				Where(goqu.I("relays_blocks.exec_block_hash").Eq(blocks.Col("exec_block_hash"))).
 				GroupBy(
 					"exec_block_hash",
 					"proposer_fee_recipient",
@@ -171,7 +169,6 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das
 			blocks.Col("graffiti_text"),
 			goqu.COALESCE(goqu.I("rb.proposer_fee_recipient"), blocks.Col("exec_fee_recipient")).As("fee_recipient"),
 			goqu.COALESCE(goqu.L("rb.value / 1e18"), goqu.I("ep.fee_recipient_reward")).As("el_reward"),
-			goqu.L("cp.cl_attestations_reward / 1e9 + cp.cl_sync_aggregate_reward / 1e9 + cp.cl_slashing_inclusion_reward / 1e9").As("cl_reward"),
 		)
 
 	// 3. Sorting and pagination
@@ -193,7 +190,10 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das
 		offset = currentCursor.Reward
 	}
 
-	order, directions := applySortAndPagination(defaultColumns, t.SortColumn{Column: colSort.Column.ToExpr(), Desc: colSort.Desc, Offset: offset}, currentCursor.GenericCursor)
+	order, directions, err := applySortAndPagination(defaultColumns, t.SortColumn{Column: colSort.Column.ToExpr(), Desc: colSort.Desc, Offset: offset}, currentCursor.GenericCursor)
+	if err != nil {
+		return nil, nil, err
+	}
 	blocksDs = goqu.Dialect("postgres").From(goqu.T("past_blocks_cte")).
 		With("past_blocks_cte", blocksDs). // encapsulate so we can use selected fields
 		Order(order...)
@@ -267,7 +267,6 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das
 					goqu.V(nil).As("graffiti_text"),
 					goqu.V(nil).As("fee_recipient"),
 					goqu.V(nil).As("el_reward"),
-					goqu.V(nil).As("cl_reward"),
 				).
 				As("scheduled_blocks")
 
@@ -314,6 +313,7 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das
 	if err != nil {
 		return nil, nil, err
 	}
+
 	err = d.alloyReader.SelectContext(ctx, &proposals, query, args...)
 	log.Debugf("=== getting past blocks took %s", time.Since(startTime))
 	if err != nil {
@@ -333,6 +333,53 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das
 		slices.Reverse(proposals)
 	}
 
+	slots := make([]uint64, len(proposals))
+	for i, proposal := range proposals {
+		slots[i] = proposal.Slot
+	}
+
+	// retrieve the cl rewards, source it from clickhouse for mainnet and from postgres for holsky
+	// TODO: harmonize this @invis
+	clRewardsData := []struct {
+		Slot     uint64              `db:"slot"`
+		ClReward decimal.NullDecimal `db:"cl_reward"`
+	}{}
+	if utils.Config.Chain.ClConfig.DepositChainID == 17000 {
+		clRewardsQuery := goqu.Dialect("postgres").
+			From(goqu.T("consensus_payloads")).
+			Select(
+				goqu.C("slot"),
+				goqu.L("cl_attestations_reward / 1e9 + cl_sync_aggregate_reward / 1e9 + cl_slashing_inclusion_reward / 1e9 AS cl_reward"),
+			).Where(goqu.C("slot").In(slots))
+		clRewardsQuerySql, args, err := clRewardsQuery.Prepared(true).ToSQL()
+		if err != nil {
+			return nil, nil, err
+		}
+		err = d.alloyReader.SelectContext(ctx, &clRewardsData, clRewardsQuerySql, args...)
+		if err != nil {
+			return nil, nil, err
+		}
+	} else {
+		clRewardsQuery := goqu.Dialect("postgres").
+			From(goqu.L("mainnet.validator_proposal_rewards_slot")).
+			Select(
+				goqu.C("slot"),
+				goqu.L("attestations_reward / 1e9 + sync_aggregate_reward / 1e9 + slasher_reward / 1e9 AS cl_reward"),
+			).Where(goqu.C("slot").In(slots))
+		clRewardsQuerySql, args, err := clRewardsQuery.Prepared(true).ToSQL()
+		if err != nil {
+			return nil, nil, err
+		}
+		err = d.clickhouseReader.SelectContext(ctx, &clRewardsData, clRewardsQuerySql, args...)
+		if err != nil {
+			return nil, nil, err
+		}
+	}
+	clRewards := make(map[uint64]decimal.NullDecimal)
+	for _, reward := range clRewardsData {
+		clRewards[reward.Slot] = reward.ClReward
+	}
+
 	data := make([]t.VDBBlocksTableRow, len(proposals))
 	addressMapping := make(map[string]*t.Address)
 	contractStatusRequests := make([]db.ContractInteractionAtRequest, 0, len(proposals))
@@ -385,8 +432,8 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das
 			})
 			reward.El = proposal.ElReward.Decimal.Mul(decimal.NewFromInt(1e18))
 		}
-		if proposal.ClReward.Valid {
-			reward.Cl = proposal.ClReward.Decimal.Mul(decimal.NewFromInt(1e18))
+		if clReward, ok := clRewards[proposal.Slot]; ok && clReward.Valid {
+			reward.Cl = clReward.Decimal.Mul(decimal.NewFromInt(1e18))
 		}
 		proposals[i].Reward = proposal.ElReward.Decimal.Add(proposal.ClReward.Decimal)
 		data[i].Reward = &reward

backend/pkg/api/data_access/vdb_management.go

@@ -606,6 +606,19 @@ func (d *DataAccessService) RemoveValidatorDashboardGroup(ctx context.Context, d
 	return err
 }
 
+func (d *DataAccessService) RemoveValidatorDashboardGroupValidators(ctx context.Context, dashboardId t.VDBIdPrimary, groupId uint64) error {
+	//Create the query to delete validators
+	deleteValidatorsQuery := `
+		DELETE FROM users_val_dashboards_validators
+		WHERE dashboard_id = $1 AND group_id = $2
+	`
+
+	// Delete the validators
+	_, err := d.alloyWriter.ExecContext(ctx, deleteValidatorsQuery, dashboardId, groupId)
+
+	return err
+}
+
 func (d *DataAccessService) GetValidatorDashboardGroupCount(ctx context.Context, dashboardId t.VDBIdPrimary) (uint64, error) {
 	var count uint64
 	err := d.alloyReader.GetContext(ctx, &count, `
@@ -970,7 +983,7 @@ func (d *DataAccessService) AddValidatorDashboardValidatorsByWithdrawalAddress(c
 		SELECT DISTINCT uvdv.validator_index
 		FROM validators v
 		JOIN users_val_dashboards_validators uvdv ON v.validatorindex = uvdv.validator_index
-		WHERE uvdv.dashboard_id = $1 AND v.withdrawalcredentials = $2 AND uvdv.dashboard_id = $2;
+		WHERE uvdv.dashboard_id = $1 AND v.withdrawalcredentials = $2;
 		`, dashboardId, addressParsed)
 	})
 

backend/pkg/api/handlers/handler_service.go

@@ -9,6 +9,7 @@ import (
 	"io"
 	"net/http"
 	"strconv"
+	"strings"
 
 	"github.com/gobitfly/beaconchain/pkg/commons/log"
 	"github.com/invopop/jsonschema"
@@ -579,6 +580,7 @@ func (v *intOrString) UnmarshalJSON(data []byte) error {
 	// If unmarshalling as uint64 fails, try to unmarshal as string
 	var strValue string
 	if err := json.Unmarshal(data, &strValue); err == nil {
+		strValue = strings.TrimSpace(strValue)
 		if parsedInt, err := strconv.ParseUint(strValue, 10, 64); err == nil {
 			v.intValue = &parsedInt
 		} else {

backend/pkg/api/handlers/input_validation.go

@@ -30,6 +30,7 @@ var (
 	reValidatorDashboardPublicId   = regexp.MustCompile(`^v-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$`)
 	reValidatorPublicKeyWithPrefix = regexp.MustCompile(`^0x[0-9a-fA-F]{96}$`)
 	reValidatorPublicKey           = regexp.MustCompile(`^(0x)?[0-9a-fA-F]{96}$`)
+	reValidatorList                = regexp.MustCompile(`^(0x[0-9a-fA-F]{96}|[0-9]+)(,\s*(0x[0-9a-fA-F]{96}|[0-9]+)\s*)+$`)
 	reEthereumAddress              = regexp.MustCompile(`^(0x)?[0-9a-fA-F]{40}$`)
 	reWithdrawalCredential         = regexp.MustCompile(`^(0x0[01])?[0-9a-fA-F]{62}$`)
 	reEnsName                      = regexp.MustCompile(`^.+\.eth$`)
@@ -442,12 +443,14 @@ func (v *validationError) checkValidatorList(validators string, allowEmpty bool)
 	var indexes []types.VDBValidator
 	var publicKeys []string
 	for _, validator := range validatorsSlice {
+		validator = strings.TrimSpace(validator)
 		if reInteger.MatchString(validator) {
 			indexes = append(indexes, v.checkUint(validator, "validators"))
 		} else if reValidatorPublicKeyWithPrefix.MatchString(validator) {
 			_, err := hexutil.Decode(validator)
 			if err != nil {
 				v.add("validators", fmt.Sprintf("invalid value '%s' in list of validators", v))
+				continue
 			}
 			publicKeys = append(publicKeys, validator)
 		} else {

backend/pkg/api/handlers/internal.go

@@ -360,6 +360,10 @@ func (h *HandlerService) InternalDeleteValidatorDashboardGroup(w http.ResponseWr
 	h.PublicDeleteValidatorDashboardGroup(w, r)
 }
 
+func (h *HandlerService) InternalDeleteValidatorDashboardGroupValidators(w http.ResponseWriter, r *http.Request) {
+	h.PublicDeleteValidatorDashboardGroupValidators(w, r)
+}
+
 func (h *HandlerService) InternalPostValidatorDashboardValidators(w http.ResponseWriter, r *http.Request) {
 	h.PublicPostValidatorDashboardValidators(w, r)
 }