From d82c5a99c6dae5df1c1a0483433ed557dff30cab Mon Sep 17 00:00:00 2001 From: Stefan Pletka <124689083+Eisei24@users.noreply.github.com> Date: Wed, 11 Sep 2024 17:28:45 +0200 Subject: [PATCH 01/19] Implemented helper functions and summary --- backend/pkg/api/data_access/vdb_helpers.go | 68 ++++ backend/pkg/api/data_access/vdb_management.go | 69 ++-- backend/pkg/api/data_access/vdb_summary.go | 300 ++++++++++++++---- backend/pkg/api/types/archiver.go | 14 +- 4 files changed, 331 insertions(+), 120 deletions(-) diff --git a/backend/pkg/api/data_access/vdb_helpers.go b/backend/pkg/api/data_access/vdb_helpers.go index 12bfe2c46..969f181c6 100644 --- a/backend/pkg/api/data_access/vdb_helpers.go +++ b/backend/pkg/api/data_access/vdb_helpers.go @@ -7,12 +7,14 @@ import ( "time" "github.com/doug-martin/goqu/v9" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/gobitfly/beaconchain/pkg/api/enums" t "github.com/gobitfly/beaconchain/pkg/api/types" "github.com/gobitfly/beaconchain/pkg/commons/cache" "github.com/gobitfly/beaconchain/pkg/commons/utils" "github.com/lib/pq" "github.com/pkg/errors" + "github.com/shopspring/decimal" ) //////////////////// Helper functions (must be used by more than one VDB endpoint!) @@ -152,3 +154,69 @@ func (d *DataAccessService) getTimeToNextWithdrawal(distance uint64) time.Time { return timeToWithdrawal } + +func (d *DataAccessService) getRocketPoolOperators(ctx context.Context, validatorIndices []t.VDBValidator) (map[t.VDBValidator]t.RpOperatorInfo, error) { + validatorMapping, err := d.services.GetCurrentValidatorMapping() + if err != nil { + return nil, err + } + + pubKeyList := make([][]byte, 0, len(validatorIndices)) + pubKeyToIndex := make(map[string]t.VDBValidator, len(validatorIndices)) + for _, validator := range validatorIndices { + publicKey := validatorMapping.ValidatorMetadata[validator].PublicKey + pubKeyList = append(pubKeyList, publicKey) + pubKeyToIndex[hexutil.Encode(publicKey)] = validator + } + + queryResult := []struct { + Pubkey []byte `db:"pubkey"` + NodeFee float64 `db:"node_fee"` + NodeDepositBalance decimal.Decimal `db:"node_deposit_balance"` + UserDepositBalance decimal.Decimal `db:"user_deposit_balance"` + SmoothingPoolOptIn bool `db:"smoothing_pool_opted_in"` + }{} + + query := ` + SELECT + rplm.pubkey, + rplm.node_fee, + rplm.node_deposit_balance, + rplm.user_deposit_balance, + COALESCE(rpln.smoothing_pool_opted_in, false) AS smoothing_pool_opted_in + FROM rocketpool_minipools rplm + LEFT JOIN rocketpool_nodes rpln ON rplm.node_address = rpln.address + WHERE pubkey = ANY($1) AND node_deposit_balance IS NOT NULL AND user_deposit_balance IS NOT NULL` + + err = d.alloyReader.SelectContext(ctx, &queryResult, query, pubKeyList) + if err != nil { + return nil, fmt.Errorf("error retrieving rocketpool validators data: %w", err) + } + + rpValidators := make(map[t.VDBValidator]t.RpOperatorInfo) + for _, res := range queryResult { + publicKey := hexutil.Encode(res.Pubkey) + rpValidators[pubKeyToIndex[publicKey]] = t.RpOperatorInfo{ + NodeFee: res.NodeFee, + NodeDepositBalance: res.NodeDepositBalance, + UserDepositBalance: res.UserDepositBalance, + SmoothingPoolOptIn: res.SmoothingPoolOptIn, + } + } + + return rpValidators, nil +} + +func (d *DataAccessService) getRocketPoolOperatorReward(operator t.RpOperatorInfo, reward, effBalance decimal.Decimal) decimal.Decimal { + if reward.GreaterThan(decimal.Zero) && effBalance.GreaterThanOrEqual(decimal.New(32, 18)) { + fullDeposit := operator.UserDepositBalance.Add(operator.NodeDepositBalance) + operatorShare := operator.NodeDepositBalance.Div(fullDeposit) + invOperatorShare := decimal.NewFromInt(1).Sub(operatorShare) + + commissionReward := reward.Mul(invOperatorShare).Mul(decimal.NewFromFloat(operator.NodeFee)) + rpReward := reward.Mul(operatorShare).Add(commissionReward) + + return rpReward + } + return reward +} diff --git a/backend/pkg/api/data_access/vdb_management.go b/backend/pkg/api/data_access/vdb_management.go index 9163a51ab..b2ed347d7 100644 --- a/backend/pkg/api/data_access/vdb_management.go +++ b/backend/pkg/api/data_access/vdb_management.go @@ -320,6 +320,20 @@ func (d *DataAccessService) GetValidatorDashboardOverview(ctx context.Context, d }) } + validators, err := d.getDashboardValidators(ctx, dashboardId, nil) + if err != nil { + return nil, fmt.Errorf("error retrieving validators from dashboard id: %v", err) + } + + if dashboardId.Validators != nil || dashboardId.AggregateGroups { + data.Groups = append(data.Groups, t.VDBOverviewGroup{Id: t.DefaultGroupId, Name: t.DefaultGroupName, Count: uint64(len(validators))}) + } + + rpValidators, err := d.getRocketPoolOperators(ctx, validators) + if err != nil { + return nil, fmt.Errorf("error retrieving rocketpool validators: %w", err) + } + // Validator status and balance eg.Go(func() error { validatorMapping, err := d.services.GetCurrentValidatorMapping() @@ -327,21 +341,13 @@ func (d *DataAccessService) GetValidatorDashboardOverview(ctx context.Context, d return err } - validators, err := d.getDashboardValidators(ctx, dashboardId, nil) - if err != nil { - return fmt.Errorf("error retrieving validators from dashboard id: %v", err) - } - - if dashboardId.Validators != nil || dashboardId.AggregateGroups { - data.Groups = append(data.Groups, t.VDBOverviewGroup{Id: t.DefaultGroupId, Name: t.DefaultGroupName, Count: uint64(len(validators))}) - } + // Create a new sub-dashboard to get the total cl deposits for non-rocketpool validators + var nonRpDashboardId t.VDBId - // Status - pubKeyList := make([][]byte, 0, len(validators)) for _, validator := range validators { metadata := validatorMapping.ValidatorMetadata[validator] - pubKeyList = append(pubKeyList, metadata.PublicKey) + // Status switch constypes.ValidatorDbStatus(metadata.Status) { case constypes.DbExitingOnline, constypes.DbSlashingOnline, constypes.DbActiveOnline: data.Validators.Online++ @@ -354,47 +360,12 @@ func (d *DataAccessService) GetValidatorDashboardOverview(ctx context.Context, d case constypes.DbExited: data.Validators.Exited++ } - } - // Find rocketpool validators - type RpOperatorInfo struct { - Pubkey []byte `db:"pubkey"` - NodeFee float64 `db:"node_fee"` - NodeDepositBalance decimal.Decimal `db:"node_deposit_balance"` - UserDepositBalance decimal.Decimal `db:"user_deposit_balance"` - } - var queryResult []RpOperatorInfo - query := ` - SELECT - pubkey, - node_fee, - node_deposit_balance, - user_deposit_balance - FROM rocketpool_minipools - WHERE pubkey = ANY($1) - AND node_deposit_balance is not null - AND user_deposit_balance is not null - ` - - err = d.alloyReader.SelectContext(ctx, &queryResult, query, pubKeyList) - if err != nil { - return fmt.Errorf("error retrieving rocketpool validators data: %w", err) - } - - rpValidators := make(map[string]RpOperatorInfo) - for _, res := range queryResult { - rpValidators[hexutil.Encode(res.Pubkey)] = res - } - - // Create a new sub-dashboard to get the total cl deposits for non-rocketpool validators - var nonRpDashboardId t.VDBId - - for _, validator := range validators { - metadata := validatorMapping.ValidatorMetadata[validator] + // Balance validatorBalance := utils.GWeiToWei(big.NewInt(int64(metadata.Balance))) effectiveBalance := utils.GWeiToWei(big.NewInt(int64(metadata.EffectiveBalance))) - if rpValidator, ok := rpValidators[hexutil.Encode(metadata.PublicKey)]; ok { + if rpValidator, ok := rpValidators[validator]; ok { if protocolModes.RocketPool { // Calculate the balance of the operator fullDeposit := rpValidator.UserDepositBalance.Add(rpValidator.NodeDepositBalance) @@ -435,7 +406,7 @@ func (d *DataAccessService) GetValidatorDashboardOverview(ctx context.Context, d retrieveRewardsAndEfficiency := func(table string, hours int, rewards *t.ClElValue[decimal.Decimal], apr *t.ClElValue[float64], efficiency *float64) { // Rewards + APR eg.Go(func() error { - (*rewards).El, (*apr).El, (*rewards).Cl, (*apr).Cl, err = d.internal_getElClAPR(ctx, dashboardId, -1, hours) + (*rewards).El, (*apr).El, (*rewards).Cl, (*apr).Cl, err = d.internal_getElClAPR(ctx, dashboardId, protocolModes, -1, rpValidators, hours) if err != nil { return err } diff --git a/backend/pkg/api/data_access/vdb_summary.go b/backend/pkg/api/data_access/vdb_summary.go index 310e9e2b4..99c9d51f2 100644 --- a/backend/pkg/api/data_access/vdb_summary.go +++ b/backend/pkg/api/data_access/vdb_summary.go @@ -4,8 +4,10 @@ import ( "context" "database/sql" "fmt" + "maps" "math" "math/big" + "slices" "sort" "strconv" "strings" @@ -40,6 +42,11 @@ func (d *DataAccessService) GetValidatorDashboardSummary(ctx context.Context, da return nil, nil, err } + validatorMapping, err := d.services.GetCurrentValidatorMapping() + if err != nil { + return nil, nil, err + } + // Searching for a group name is not supported when aggregating groups or for guest dashboards groupNameSearchEnabled := !dashboardId.AggregateGroups && dashboardId.Validators == nil @@ -98,7 +105,7 @@ func (d *DataAccessService) GetValidatorDashboardSummary(ctx context.Context, da var queryResult []struct { GroupId int64 `db:"result_group_id"` GroupName string `db:"group_name"` - ValidatorIndices []uint64 `db:"validator_indices"` + ValidatorIndex uint64 `db:"validator_index"` ClRewards int64 `db:"cl_rewards"` AttestationReward decimal.Decimal `db:"attestations_reward"` AttestationIdealReward decimal.Decimal `db:"attestations_ideal_reward"` @@ -108,27 +115,26 @@ func (d *DataAccessService) GetValidatorDashboardSummary(ctx context.Context, da BlocksScheduled uint64 `db:"blocks_scheduled"` SyncExecuted uint64 `db:"sync_executed"` SyncScheduled uint64 `db:"sync_scheduled"` - MinEpochStart int64 `db:"min_epoch_start"` - MaxEpochEnd int64 `db:"max_epoch_end"` + EpochStart int64 `db:"epoch_start"` + EpochEnd int64 `db:"epoch_end"` } ds := goqu.Dialect("postgres"). From(goqu.L(fmt.Sprintf(`%s AS r FINAL`, clickhouseTable))). With("validators", goqu.L("(SELECT dashboard_id, group_id, validator_index FROM users_val_dashboards_validators WHERE dashboard_id = ?)", dashboardId.Id)). Select( - goqu.L("ARRAY_AGG(r.validator_index) AS validator_indices"), - goqu.L("(SUM(COALESCE(r.balance_end,0)) + SUM(COALESCE(r.withdrawals_amount,0)) - SUM(COALESCE(r.deposits_amount,0)) - SUM(COALESCE(r.balance_start,0))) AS cl_rewards"), - goqu.L("COALESCE(SUM(r.attestations_reward)::decimal, 0) AS attestations_reward"), - goqu.L("COALESCE(SUM(r.attestations_ideal_reward)::decimal, 0) AS attestations_ideal_reward"), - goqu.L("COALESCE(SUM(r.attestations_executed), 0) AS attestations_executed"), - goqu.L("COALESCE(SUM(r.attestations_scheduled), 0) AS attestations_scheduled"), - goqu.L("COALESCE(SUM(r.blocks_proposed), 0) AS blocks_proposed"), - goqu.L("COALESCE(SUM(r.blocks_scheduled), 0) AS blocks_scheduled"), - goqu.L("COALESCE(SUM(r.sync_executed), 0) AS sync_executed"), - goqu.L("COALESCE(SUM(r.sync_scheduled), 0) AS sync_scheduled"), - goqu.L("COALESCE(MIN(r.epoch_start), 0) AS min_epoch_start"), - goqu.L("COALESCE(MAX(r.epoch_end), 0) AS max_epoch_end")). - GroupBy(goqu.L("result_group_id")) + goqu.L("r.validator_index"), + goqu.L("(r.balance_end + r.withdrawals_amount - r.deposits_amount - r.balance_start) AS cl_rewards"), + goqu.L("r.attestations_reward::decimal AS attestations_reward"), + goqu.L("r.attestations_ideal_reward::decimal AS attestations_ideal_reward"), + goqu.L("r.attestations_executed"), + goqu.L("r.attestations_scheduled"), + goqu.L("r.blocks_proposed"), + goqu.L("r.blocks_scheduled"), + goqu.L("r.sync_executed"), + goqu.L("r.sync_scheduled"), + goqu.L("r.epoch_start"), + goqu.L("r.epoch_end")) if len(validators) > 0 { ds = ds. @@ -151,8 +157,7 @@ func (d *DataAccessService) GetValidatorDashboardSummary(ctx context.Context, da // Get the group names since we can filter and/or sort for them ds = ds. SelectAppend(goqu.L("g.name AS group_name")). - InnerJoin(goqu.L("users_val_dashboards_groups g"), goqu.On(goqu.L("v.group_id = g.id AND v.dashboard_id = g.dashboard_id"))). - GroupByAppend(goqu.L("group_name")) + InnerJoin(goqu.L("users_val_dashboards_groups g"), goqu.On(goqu.L("v.group_id = g.id AND v.dashboard_id = g.dashboard_id"))) } } @@ -171,17 +176,76 @@ func (d *DataAccessService) GetValidatorDashboardSummary(ctx context.Context, da return result, &paging, nil } - epochMin := int64(math.MaxInt32) - epochMax := int64(0) + rpValidators := make(map[uint64]t.RpOperatorInfo) + if protocolModes.RocketPool { + validators := make([]uint64, 0, len(queryResult)) + for _, row := range queryResult { + validators = append(validators, row.ValidatorIndex) + } + + rpValidators, err = d.getRocketPoolOperators(ctx, validators) + if err != nil { + return nil, nil, err + } + } + + type QueryResultSum struct { + GroupId int64 + GroupName string + ValidatorIndices []uint64 + ClRewards decimal.Decimal + AttestationReward decimal.Decimal + AttestationIdealReward decimal.Decimal + AttestationsExecuted uint64 + AttestationsScheduled uint64 + BlocksProposed uint64 + BlocksScheduled uint64 + SyncExecuted uint64 + SyncScheduled uint64 + } + + queryResultSumMap := make(map[int64]QueryResultSum) + + epochStart := int64(math.MaxInt32) + epochEnd := int64(0) for _, row := range queryResult { - if row.MinEpochStart < epochMin { - epochMin = row.MinEpochStart + if row.EpochStart < epochStart { + epochStart = row.EpochStart + } + if row.EpochEnd > epochEnd { + epochEnd = row.EpochEnd } - if row.MaxEpochEnd > epochMax { - epochMax = row.MaxEpochEnd + + if _, ok := queryResultSumMap[row.GroupId]; !ok { + queryResultSumMap[row.GroupId] = QueryResultSum{ + GroupId: row.GroupId, + GroupName: row.GroupName, + } + } + + groupSum := queryResultSumMap[row.GroupId] + groupSum.ValidatorIndices = append(groupSum.ValidatorIndices, row.ValidatorIndex) + groupSum.AttestationReward = groupSum.AttestationReward.Add(row.AttestationReward) + groupSum.AttestationIdealReward = groupSum.AttestationIdealReward.Add(row.AttestationIdealReward) + groupSum.AttestationsExecuted += row.AttestationsExecuted + groupSum.AttestationsScheduled += row.AttestationsScheduled + groupSum.BlocksProposed += row.BlocksProposed + groupSum.BlocksScheduled += row.BlocksScheduled + groupSum.SyncExecuted += row.SyncExecuted + groupSum.SyncScheduled += row.SyncScheduled + + clRewardWei := utils.GWeiToWei(big.NewInt(row.ClRewards)) + if rpValidator, ok := rpValidators[row.ValidatorIndex]; ok && protocolModes.RocketPool { + metadata := validatorMapping.ValidatorMetadata[row.ValidatorIndex] + effectiveBalance := utils.GWeiToWei(big.NewInt(int64(metadata.EffectiveBalance))) + groupSum.ClRewards = groupSum.ClRewards.Add(d.getRocketPoolOperatorReward(rpValidator, clRewardWei, effectiveBalance)) + } else { + groupSum.ClRewards = groupSum.ClRewards.Add(clRewardWei) } } + queryResultSum := slices.Collect(maps.Values(queryResultSumMap)) + // ------------------------------------------------------------------------------------------------------------------ // Get the EL rewards elRewards := make(map[int64]decimal.Decimal) @@ -199,7 +263,7 @@ func (d *DataAccessService) GetValidatorDashboardSummary(ctx context.Context, da GroupBy("exec_block_hash").As("rb"), goqu.On(goqu.L("rb.exec_block_hash = b.exec_block_hash")), ). - Where(goqu.L("b.epoch >= ? AND b.epoch <= ? AND b.status = '1'", epochMin, epochMax)). + Where(goqu.L("b.epoch >= ? AND b.epoch <= ? AND b.status = '1'", epochStart, epochEnd)). GroupBy(goqu.L("result_group_id")) if len(validators) > 0 { @@ -258,11 +322,11 @@ func (d *DataAccessService) GetValidatorDashboardSummary(ctx context.Context, da // ------------------------------------------------------------------------------------------------------------------ // Sort by group name, after this the name is no longer relevant if groupNameSearchEnabled && colSort.Column == enums.VDBSummaryColumns.Group { - sort.Slice(queryResult, func(i, j int) bool { + sort.Slice(queryResultSum, func(i, j int) bool { if colSort.Desc { - return queryResult[i].GroupName > queryResult[j].GroupName + return queryResultSum[i].GroupName > queryResultSum[j].GroupName } else { - return queryResult[i].GroupName < queryResult[j].GroupName + return queryResultSum[i].GroupName < queryResultSum[j].GroupName } }) } @@ -286,7 +350,7 @@ func (d *DataAccessService) GetValidatorDashboardSummary(ctx context.Context, da GroupId: t.AllGroups, } - for _, queryEntry := range queryResult { + for _, queryEntry := range queryResultSum { resultEntry := t.VDBSummaryTableRow{ GroupId: queryEntry.GroupId, AverageNetworkEfficiency: averageNetworkEfficiency, @@ -344,7 +408,7 @@ func (d *DataAccessService) GetValidatorDashboardSummary(ctx context.Context, da resultEntry.Proposals.Failed = queryEntry.BlocksScheduled - queryEntry.BlocksProposed // Rewards - resultEntry.Reward.Cl = utils.GWeiToWei(big.NewInt(queryEntry.ClRewards)) + resultEntry.Reward.Cl = queryEntry.ClRewards if _, ok := elRewards[queryEntry.GroupId]; ok { resultEntry.Reward.El = elRewards[queryEntry.GroupId] } @@ -453,7 +517,7 @@ func (d *DataAccessService) GetValidatorDashboardSummary(ctx context.Context, da // ------------------------------------------------------------------------------------------------------------------ // Calculate the total - if len(queryResult) > 1 && len(result) > 0 { + if len(queryResultSum) > 1 && len(result) > 0 { // We have more than one group and at least one group remains after the filtering so we need to show the total row totalEntry := t.VDBSummaryTableRow{ GroupId: total.GroupId, @@ -672,13 +736,18 @@ func (d *DataAccessService) GetValidatorDashboardGroupSummary(ctx context.Contex } } - _, ret.Apr.El, _, ret.Apr.Cl, err = d.internal_getElClAPR(ctx, dashboardId, groupId, hours) + if len(validators) > 0 { + validatorArr = validators + } + + rpValidators, err := d.getRocketPoolOperators(ctx, validatorArr) if err != nil { - return nil, err + return nil, fmt.Errorf("error retrieving rocketpool validators: %w", err) } - if len(validators) > 0 { - validatorArr = validators + _, ret.Apr.El, _, ret.Apr.Cl, err = d.internal_getElClAPR(ctx, dashboardId, protocolModes, groupId, rpValidators, hours) + if err != nil { + return nil, err } pastSyncPeriodCutoff := utils.SyncPeriodOfEpoch(rows[0].EpochStart) @@ -731,7 +800,7 @@ func (d *DataAccessService) GetValidatorDashboardGroupSummary(ctx context.Contex return ret, nil } -func (d *DataAccessService) internal_getElClAPR(ctx context.Context, dashboardId t.VDBId, groupId int64, hours int) (elIncome decimal.Decimal, elAPR float64, clIncome decimal.Decimal, clAPR float64, err error) { +func (d *DataAccessService) internal_getElClAPR(ctx context.Context, dashboardId t.VDBId, protocolModes t.VDBProtocolModes, groupId int64, rpValidators map[t.VDBValidator]t.RpOperatorInfo, hours int) (elIncome decimal.Decimal, elAPR float64, clIncome decimal.Decimal, clAPR float64, err error) { table := "" switch hours { @@ -749,24 +818,36 @@ func (d *DataAccessService) internal_getElClAPR(ctx context.Context, dashboardId return decimal.Zero, 0, decimal.Zero, 0, fmt.Errorf("invalid hours value: %v", hours) } - type RewardsResult struct { - EpochStart uint64 `db:"epoch_start"` - EpochEnd uint64 `db:"epoch_end"` - ValidatorCount uint64 `db:"validator_count"` - Reward sql.NullInt64 `db:"reward"` + validatorMapping, err := d.services.GetCurrentValidatorMapping() + if err != nil { + return decimal.Zero, 0, decimal.Zero, 0, err } - var rewardsResultTable RewardsResult - var rewardsResultTotal RewardsResult + type ClRewardsResult struct { + ValidatorIndex uint64 `db:"validator_index"` + EpochStart uint64 `db:"epoch_start"` + EpochEnd uint64 `db:"epoch_end"` + Reward int64 `db:"reward"` + } + + type ElRewardsResult struct { + ValidatorIndex uint64 `db:"validator_index"` + Reward decimal.Decimal `db:"el_reward"` + } + + var rewardsResultTable []ClRewardsResult + var rewardsResultTotal []ClRewardsResult + + var elRewardsResult []ElRewardsResult rewardsDs := goqu.Dialect("postgres"). From(goqu.L(fmt.Sprintf("%s AS r FINAL", table))). With("validators", goqu.L("(SELECT group_id, validator_index FROM users_val_dashboards_validators WHERE dashboard_id = ?)", dashboardId.Id)). Select( - goqu.L("MIN(epoch_start) AS epoch_start"), - goqu.L("MAX(epoch_end) AS epoch_end"), - goqu.L("COUNT(*) AS validator_count"), - goqu.L("(SUM(COALESCE(r.balance_end,0)) + SUM(COALESCE(r.withdrawals_amount,0)) - SUM(COALESCE(r.deposits_amount,0)) - SUM(COALESCE(r.balance_start,0))) AS reward")) + goqu.L("validator_index"), + goqu.L("epoch_start"), + goqu.L("epoch_end"), + goqu.L("(COALESCE(r.balance_end,0) + COALESCE(r.withdrawals_amount,0) - COALESCE(r.deposits_amount,0) - COALESCE(r.balance_start,0)) AS reward")) if len(dashboardId.Validators) > 0 { rewardsDs = rewardsDs. @@ -787,26 +868,56 @@ func (d *DataAccessService) internal_getElClAPR(ctx context.Context, dashboardId return decimal.Zero, 0, decimal.Zero, 0, fmt.Errorf("error preparing query: %v", err) } - err = d.clickhouseReader.GetContext(ctx, &rewardsResultTable, query, args...) - if err != nil || !rewardsResultTable.Reward.Valid { + err = d.clickhouseReader.SelectContext(ctx, &rewardsResultTable, query, args...) + if err != nil || len(rewardsResultTable) == 0 { return decimal.Zero, 0, decimal.Zero, 0, err } - if rewardsResultTable.ValidatorCount == 0 { - return decimal.Zero, 0, decimal.Zero, 0, nil + epochStart := uint64(math.MaxInt32) + epochEnd := uint64(0) + epochStartTotal := uint64(math.MaxInt32) + epochEndTotal := uint64(0) + + aprRewards := decimal.Zero + rewards := decimal.Zero + deposits := decimal.Zero + + for _, row := range rewardsResultTable { + if row.EpochStart < epochStart { + epochStart = row.EpochStart + } + if row.EpochEnd > epochEnd { + epochEnd = row.EpochEnd + } + + metadata := validatorMapping.ValidatorMetadata[row.ValidatorIndex] + reward := utils.GWeiToWei(big.NewInt(row.Reward)) + + if rpValidator, ok := rpValidators[row.ValidatorIndex]; ok { + effectiveBalance := utils.GWeiToWei(big.NewInt(int64(metadata.EffectiveBalance))) + rpReward := d.getRocketPoolOperatorReward(rpValidator, reward, effectiveBalance) + aprRewards = aprRewards.Add(rpReward) + if protocolModes.RocketPool { + rewards = rewards.Add(rpReward) + } else { + rewards = rewards.Add(reward) + } + deposits = deposits.Add(rpValidator.NodeDepositBalance) + } else { + aprRewards = aprRewards.Add(reward) + rewards = rewards.Add(reward) + deposits = deposits.Add(decimal.New(32, 18)) + } } aprDivisor := hours if hours == -1 { // for all time APR aprDivisor = 90 * 24 } - clAPR = ((float64(rewardsResultTable.Reward.Int64) / float64(aprDivisor)) / (float64(32e9) * float64(rewardsResultTable.ValidatorCount))) * 24.0 * 365.0 * 100.0 - if math.IsNaN(clAPR) { - clAPR = 0 + if !deposits.IsZero() { + clAPR = aprRewards.Div(decimal.NewFromInt(int64(aprDivisor))).Div(deposits).Mul(decimal.NewFromInt(24 * 365 * 100)).InexactFloat64() } - clIncome = decimal.NewFromInt(rewardsResultTable.Reward.Int64).Mul(decimal.NewFromInt(1e9)) - if hours == -1 { rewardsDs = rewardsDs. From(goqu.L("validator_dashboard_data_rolling_total AS r FINAL")) @@ -816,16 +927,37 @@ func (d *DataAccessService) internal_getElClAPR(ctx context.Context, dashboardId return decimal.Zero, 0, decimal.Zero, 0, fmt.Errorf("error preparing query: %v", err) } - err = d.clickhouseReader.GetContext(ctx, &rewardsResultTotal, query, args...) - if err != nil || !rewardsResultTotal.Reward.Valid { + err = d.clickhouseReader.SelectContext(ctx, &rewardsResultTotal, query, args...) + if err != nil || len(rewardsResultTotal) == 0 { return decimal.Zero, 0, decimal.Zero, 0, err } - clIncome = decimal.NewFromInt(rewardsResultTotal.Reward.Int64).Mul(decimal.NewFromInt(1e9)) + rewards = decimal.Zero + for _, row := range rewardsResultTotal { + if row.EpochStart < epochStartTotal { + epochStartTotal = row.EpochStart + } + if row.EpochEnd > epochEndTotal { + epochEndTotal = row.EpochEnd + } + + metadata := validatorMapping.ValidatorMetadata[row.ValidatorIndex] + reward := utils.GWeiToWei(big.NewInt(row.Reward)) + + if rpValidator, ok := rpValidators[row.ValidatorIndex]; ok && protocolModes.RocketPool { + effectiveBalance := utils.GWeiToWei(big.NewInt(int64(metadata.EffectiveBalance))) + rewards = rewards.Add(d.getRocketPoolOperatorReward(rpValidator, reward, effectiveBalance)) + } else { + rewards = rewards.Add(reward) + } + } } + clIncome = rewards elDs := goqu.Dialect("postgres"). - Select(goqu.L("COALESCE(SUM(COALESCE(rb.value / 1e18, fee_recipient_reward)), 0) AS el_reward")). + Select( + goqu.L("b.proposer AS validator_index"), + goqu.L("COALESCE(SUM(COALESCE(rb.value, fee_recipient_reward * 1e18)), 0) AS el_reward")). From(goqu.L("blocks AS b")). LeftJoin(goqu.L("execution_payloads AS ep"), goqu.On(goqu.L("b.exec_block_hash = ep.block_hash"))). LeftJoin( @@ -837,7 +969,8 @@ func (d *DataAccessService) internal_getElClAPR(ctx context.Context, dashboardId GroupBy("exec_block_hash").As("rb"), goqu.On(goqu.L("rb.exec_block_hash = b.exec_block_hash")), ). - Where(goqu.L("b.status = '1'")) + Where(goqu.L("b.status = '1'")). + GroupBy("b.proposer") if len(dashboardId.Validators) > 0 { elDs = elDs. @@ -854,38 +987,65 @@ func (d *DataAccessService) internal_getElClAPR(ctx context.Context, dashboardId } elTableDs := elDs. - Where(goqu.L("b.epoch >= ? AND b.epoch <= ?", rewardsResultTable.EpochStart, rewardsResultTable.EpochEnd)) + Where(goqu.L("b.epoch >= ? AND b.epoch <= ?", epochStart, epochEnd)) query, args, err = elTableDs.Prepared(true).ToSQL() if err != nil { return decimal.Zero, 0, decimal.Zero, 0, fmt.Errorf("error preparing query: %v", err) } - err = d.alloyReader.GetContext(ctx, &elIncome, query, args...) + err = d.alloyReader.SelectContext(ctx, &elRewardsResult, query, args...) if err != nil { return decimal.Zero, 0, decimal.Zero, 0, err } - elIncomeFloat, _ := elIncome.Float64() - elAPR = ((elIncomeFloat / float64(aprDivisor)) / (float64(32e18) * float64(rewardsResultTable.ValidatorCount))) * 24.0 * 365.0 * 100.0 - if math.IsNaN(elAPR) { - elAPR = 0 + + elRewards := decimal.Zero + for _, row := range elRewardsResult { + reward := row.Reward + if rpValidator, ok := rpValidators[row.ValidatorIndex]; ok { + // TODO: This is not quite correct + for epochPeriodEnd, smoothingPoolReward := range rpValidator.SmoothingPoolReward { + if epochPeriodEnd >= epochStart && epochPeriodEnd <= epochEnd { + reward = reward.Add(smoothingPoolReward) + } + } + } + elRewards = elIncome.Add(reward) + } + + if !deposits.IsZero() { + elAPR = elRewards.Div(decimal.NewFromInt(int64(aprDivisor))).Div(deposits).Mul(decimal.NewFromInt(24 * 365 * 100)).InexactFloat64() } if hours == -1 { elTotalDs := elDs. - Where(goqu.L("b.epoch >= ? AND b.epoch <= ?", rewardsResultTotal.EpochStart, rewardsResultTotal.EpochEnd)) + Where(goqu.L("b.epoch >= ? AND b.epoch <= ?", epochStartTotal, epochEndTotal)) query, args, err = elTotalDs.Prepared(true).ToSQL() if err != nil { return decimal.Zero, 0, decimal.Zero, 0, fmt.Errorf("error preparing query: %v", err) } - err = d.alloyReader.GetContext(ctx, &elIncome, query, args...) + err = d.alloyReader.SelectContext(ctx, &elRewardsResult, query, args...) if err != nil { return decimal.Zero, 0, decimal.Zero, 0, err } + + elRewards = decimal.Zero + for _, row := range elRewardsResult { + reward := row.Reward + if rpValidator, ok := rpValidators[row.ValidatorIndex]; ok { + // TODO: This is not quite correct + for epochPeriodEnd, smoothingPoolReward := range rpValidator.SmoothingPoolReward { + if epochPeriodEnd >= epochStartTotal && epochPeriodEnd <= epochEndTotal { + reward = reward.Add(smoothingPoolReward) + } + } + } + elRewards = elIncome.Add(reward) + } } - elIncome = elIncome.Mul(decimal.NewFromInt(1e18)) + elIncome = elRewards return elIncome, elAPR, clIncome, clAPR, nil } diff --git a/backend/pkg/api/types/archiver.go b/backend/pkg/api/types/archiver.go index ac1666e4a..b6c2267d4 100644 --- a/backend/pkg/api/types/archiver.go +++ b/backend/pkg/api/types/archiver.go @@ -1,6 +1,9 @@ package types -import "github.com/gobitfly/beaconchain/pkg/api/enums" +import ( + "github.com/gobitfly/beaconchain/pkg/api/enums" + "github.com/shopspring/decimal" +) type ArchiverDashboard struct { DashboardId uint64 @@ -13,3 +16,12 @@ type ArchiverDashboardArchiveReason struct { DashboardId uint64 ArchivedReason enums.VDBArchivedReason } + +// TODO: Find a good place for this +type RpOperatorInfo struct { + NodeFee float64 + NodeDepositBalance decimal.Decimal + UserDepositBalance decimal.Decimal + SmoothingPoolOptIn bool + SmoothingPoolReward map[uint64]decimal.Decimal +} From 4bdf19bc49ad2880d5ceaa968f451f5d854c1e86 Mon Sep 17 00:00:00 2001 From: Stefan Pletka <124689083+Eisei24@users.noreply.github.com> Date: Fri, 13 Sep 2024 14:25:28 +0200 Subject: [PATCH 02/19] Implemented cl rewards --- backend/pkg/api/data_access/vdb_blocks.go | 17 + backend/pkg/api/data_access/vdb_helpers.go | 37 +- backend/pkg/api/data_access/vdb_management.go | 2 +- backend/pkg/api/data_access/vdb_rewards.go | 460 ++++++++++++------ backend/pkg/api/data_access/vdb_summary.go | 89 ++-- backend/pkg/api/types/archiver.go | 3 +- 6 files changed, 390 insertions(+), 218 deletions(-) diff --git a/backend/pkg/api/data_access/vdb_blocks.go b/backend/pkg/api/data_access/vdb_blocks.go index 3780599b7..e8c2aaaab 100644 --- a/backend/pkg/api/data_access/vdb_blocks.go +++ b/backend/pkg/api/data_access/vdb_blocks.go @@ -344,6 +344,20 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das slices.Reverse(proposals) } + // Get the rocketpool minipool infos + rpValidators := make(map[uint64]t.RpMinipoolInfo) + if protocolModes.RocketPool { + var proposers []uint64 + for _, row := range proposals { + proposers = append(proposers, row.Proposer) + } + + rpValidators, err = d.getRocketPoolMinipoolInfos(ctx, proposers) + if err != nil { + return nil, nil, err + } + } + data := make([]t.VDBBlocksTableRow, len(proposals)) ensMapping := make(map[string]string) for i, proposal := range proposals { @@ -387,6 +401,9 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das } if proposal.ClReward.Valid { reward.Cl = proposal.ClReward.Decimal.Mul(decimal.NewFromInt(1e18)) + if rpValidator, ok := rpValidators[proposal.Proposer]; ok && protocolModes.RocketPool { + reward.Cl = reward.Cl.Mul(d.getRocketPoolOperatorFactor(rpValidator)) + } } proposals[i].Reward = proposal.ElReward.Decimal.Add(proposal.ClReward.Decimal) data[i].Reward = &reward diff --git a/backend/pkg/api/data_access/vdb_helpers.go b/backend/pkg/api/data_access/vdb_helpers.go index 969f181c6..f4b47ff13 100644 --- a/backend/pkg/api/data_access/vdb_helpers.go +++ b/backend/pkg/api/data_access/vdb_helpers.go @@ -155,7 +155,7 @@ func (d *DataAccessService) getTimeToNextWithdrawal(distance uint64) time.Time { return timeToWithdrawal } -func (d *DataAccessService) getRocketPoolOperators(ctx context.Context, validatorIndices []t.VDBValidator) (map[t.VDBValidator]t.RpOperatorInfo, error) { +func (d *DataAccessService) getRocketPoolMinipoolInfos(ctx context.Context, validatorIndices []t.VDBValidator) (map[t.VDBValidator]t.RpMinipoolInfo, error) { validatorMapping, err := d.services.GetCurrentValidatorMapping() if err != nil { return nil, err @@ -174,18 +174,15 @@ func (d *DataAccessService) getRocketPoolOperators(ctx context.Context, validato NodeFee float64 `db:"node_fee"` NodeDepositBalance decimal.Decimal `db:"node_deposit_balance"` UserDepositBalance decimal.Decimal `db:"user_deposit_balance"` - SmoothingPoolOptIn bool `db:"smoothing_pool_opted_in"` }{} query := ` SELECT - rplm.pubkey, - rplm.node_fee, - rplm.node_deposit_balance, - rplm.user_deposit_balance, - COALESCE(rpln.smoothing_pool_opted_in, false) AS smoothing_pool_opted_in - FROM rocketpool_minipools rplm - LEFT JOIN rocketpool_nodes rpln ON rplm.node_address = rpln.address + pubkey, + node_fee, + node_deposit_balance, + user_deposit_balance + FROM rocketpool_minipools WHERE pubkey = ANY($1) AND node_deposit_balance IS NOT NULL AND user_deposit_balance IS NOT NULL` err = d.alloyReader.SelectContext(ctx, &queryResult, query, pubKeyList) @@ -193,30 +190,26 @@ func (d *DataAccessService) getRocketPoolOperators(ctx context.Context, validato return nil, fmt.Errorf("error retrieving rocketpool validators data: %w", err) } - rpValidators := make(map[t.VDBValidator]t.RpOperatorInfo) + rpValidators := make(map[t.VDBValidator]t.RpMinipoolInfo) for _, res := range queryResult { publicKey := hexutil.Encode(res.Pubkey) - rpValidators[pubKeyToIndex[publicKey]] = t.RpOperatorInfo{ + rpValidators[pubKeyToIndex[publicKey]] = t.RpMinipoolInfo{ NodeFee: res.NodeFee, NodeDepositBalance: res.NodeDepositBalance, UserDepositBalance: res.UserDepositBalance, - SmoothingPoolOptIn: res.SmoothingPoolOptIn, } } return rpValidators, nil } -func (d *DataAccessService) getRocketPoolOperatorReward(operator t.RpOperatorInfo, reward, effBalance decimal.Decimal) decimal.Decimal { - if reward.GreaterThan(decimal.Zero) && effBalance.GreaterThanOrEqual(decimal.New(32, 18)) { - fullDeposit := operator.UserDepositBalance.Add(operator.NodeDepositBalance) - operatorShare := operator.NodeDepositBalance.Div(fullDeposit) - invOperatorShare := decimal.NewFromInt(1).Sub(operatorShare) +func (d *DataAccessService) getRocketPoolOperatorFactor(minipool t.RpMinipoolInfo) decimal.Decimal { + fullDeposit := minipool.UserDepositBalance.Add(minipool.NodeDepositBalance) + operatorShare := minipool.NodeDepositBalance.Div(fullDeposit) + invOperatorShare := decimal.NewFromInt(1).Sub(operatorShare) - commissionReward := reward.Mul(invOperatorShare).Mul(decimal.NewFromFloat(operator.NodeFee)) - rpReward := reward.Mul(operatorShare).Add(commissionReward) + commissionReward := invOperatorShare.Mul(decimal.NewFromFloat(minipool.NodeFee)) + operatorFactor := operatorShare.Add(commissionReward) - return rpReward - } - return reward + return operatorFactor } diff --git a/backend/pkg/api/data_access/vdb_management.go b/backend/pkg/api/data_access/vdb_management.go index b2ed347d7..1413c61e8 100644 --- a/backend/pkg/api/data_access/vdb_management.go +++ b/backend/pkg/api/data_access/vdb_management.go @@ -329,7 +329,7 @@ func (d *DataAccessService) GetValidatorDashboardOverview(ctx context.Context, d data.Groups = append(data.Groups, t.VDBOverviewGroup{Id: t.DefaultGroupId, Name: t.DefaultGroupName, Count: uint64(len(validators))}) } - rpValidators, err := d.getRocketPoolOperators(ctx, validators) + rpValidators, err := d.getRocketPoolMinipoolInfos(ctx, validators) if err != nil { return nil, fmt.Errorf("error retrieving rocketpool validators: %w", err) } diff --git a/backend/pkg/api/data_access/vdb_rewards.go b/backend/pkg/api/data_access/vdb_rewards.go index 2212bee81..dda852ea3 100644 --- a/backend/pkg/api/data_access/vdb_rewards.go +++ b/backend/pkg/api/data_access/vdb_rewards.go @@ -53,11 +53,11 @@ func (d *DataAccessService) GetValidatorDashboardRewards(ctx context.Context, da if strings.HasPrefix(search, "0x") && utils.IsHash(search) { search = strings.ToLower(search) - // Get the current validator state to convert pubkey to index validatorMapping, err := d.services.GetCurrentValidatorMapping() if err != nil { return nil, nil, err } + if index, ok := validatorMapping.ValidatorIndices[search]; ok { indexSearch = int64(index) } else { @@ -80,21 +80,22 @@ func (d *DataAccessService) GetValidatorDashboardRewards(ctx context.Context, da groupIdSearchMap := make(map[uint64]bool, 0) // ------------------------------------------------------------------------------------------------------------------ - // Build the query that serves as base for both the main and EL rewards queries + // Build the main and EL rewards queries rewardsDs := goqu.Dialect("postgres"). From(goqu.L("validator_dashboard_data_epoch e")). With("validators", goqu.L("(SELECT validator_index as validator_index, group_id FROM users_val_dashboards_validators WHERE dashboard_id = ?)", dashboardId.Id)). Select( goqu.L("e.epoch"), - goqu.L(`SUM(COALESCE(e.attestations_reward, 0) + COALESCE(e.blocks_cl_reward, 0) + COALESCE(e.sync_rewards, 0)) AS cl_rewards`), - goqu.L("SUM(COALESCE(e.attestations_scheduled, 0)) AS attestations_scheduled"), - goqu.L("SUM(COALESCE(e.attestations_executed, 0)) AS attestations_executed"), - goqu.L("SUM(COALESCE(e.blocks_scheduled, 0)) AS blocks_scheduled"), - goqu.L("SUM(COALESCE(e.blocks_proposed, 0)) AS blocks_proposed"), - goqu.L("SUM(COALESCE(e.sync_scheduled, 0)) AS sync_scheduled"), - goqu.L("SUM(COALESCE(e.sync_executed, 0)) AS sync_executed"), - goqu.L("SUM(CASE WHEN e.slashed THEN 1 ELSE 0 END) AS slashed_in_epoch"), - goqu.L("SUM(COALESCE(e.blocks_slashing_count, 0)) AS slashed_amount")). + goqu.L("e.validator_index"), + goqu.L(`(e.attestations_reward + e.blocks_cl_reward + e.sync_rewards) AS cl_rewards`), + goqu.L("e.attestations_scheduled"), + goqu.L("e.attestations_executed"), + goqu.L("e.blocks_scheduled"), + goqu.L("e.blocks_proposed"), + goqu.L("e.sync_scheduled"), + goqu.L("e.sync_executed"), + goqu.L("e.slashed"), + goqu.L("e.blocks_slashing_count")). Where(goqu.L("e.epoch_timestamp >= fromUnixTimestamp(?)", utils.EpochToTime(startEpoch).Unix())) elDs := goqu.Dialect("postgres"). @@ -220,8 +221,7 @@ func (d *DataAccessService) GetValidatorDashboardRewards(ctx context.Context, da if dashboardId.AggregateGroups { rewardsDs = rewardsDs. - SelectAppend(goqu.L("?::smallint AS result_group_id", t.DefaultGroupId)). - GroupBy(goqu.L("e.epoch")) + SelectAppend(goqu.L("?::smallint AS result_group_id", t.DefaultGroupId)) elDs = elDs. SelectAppend(goqu.L("?::smallint AS result_group_id", t.DefaultGroupId)). @@ -236,8 +236,7 @@ func (d *DataAccessService) GetValidatorDashboardRewards(ctx context.Context, da } } else { rewardsDs = rewardsDs. - SelectAppend(goqu.L("v.group_id AS result_group_id")). - GroupBy(goqu.L("e.epoch"), goqu.L("result_group_id")) + SelectAppend(goqu.L("v.group_id AS result_group_id")) elDs = elDs. SelectAppend(goqu.L("v.group_id AS result_group_id")). GroupBy(goqu.L("b.epoch"), goqu.L("result_group_id")) @@ -254,8 +253,7 @@ func (d *DataAccessService) GetValidatorDashboardRewards(ctx context.Context, da // In case a list of validators is provided set the group to the default id rewardsDs = rewardsDs. SelectAppend(goqu.L("?::smallint AS result_group_id", t.DefaultGroupId)). - Where(goqu.L("e.validator_index IN ?", dashboardId.Validators)). - GroupBy(goqu.L("e.epoch")) + Where(goqu.L("e.validator_index IN ?", dashboardId.Validators)) elDs = elDs. SelectAppend(goqu.L("?::smallint AS result_group_id", t.DefaultGroupId)). Where(goqu.L("b.proposer = ANY(?)", pq.Array(dashboardId.Validators))). @@ -294,21 +292,39 @@ func (d *DataAccessService) GetValidatorDashboardRewards(ctx context.Context, da // ------------------------------------------------------------------------------------------------------------------ // Build the main query and get the data - queryResult := []struct { - Epoch uint64 `db:"epoch"` - GroupId int64 `db:"result_group_id"` - ClRewards int64 `db:"cl_rewards"` - AttestationsScheduled uint64 `db:"attestations_scheduled"` - AttestationsExecuted uint64 `db:"attestations_executed"` - BlocksScheduled uint64 `db:"blocks_scheduled"` - BlocksProposed uint64 `db:"blocks_proposed"` - SyncScheduled uint64 `db:"sync_scheduled"` - SyncExecuted uint64 `db:"sync_executed"` - SlashedInEpoch uint64 `db:"slashed_in_epoch"` - SlashedAmount uint64 `db:"slashed_amount"` - }{} + + type QueryResultSum struct { + Epoch uint64 + GroupId int64 + ClRewards decimal.Decimal + AttestationsScheduled uint64 + AttestationsExecuted uint64 + BlocksScheduled uint64 + BlocksProposed uint64 + SyncScheduled uint64 + SyncExecuted uint64 + Slashed uint64 + BlocksSlashingCount uint64 + } + var queryResultSum []QueryResultSum wg.Go(func() error { + type QueryResult struct { + Epoch uint64 `db:"epoch"` + GroupId int64 `db:"result_group_id"` + ValidatorIndex uint64 `db:"validator_index"` + ClRewards int64 `db:"cl_rewards"` + AttestationsScheduled uint64 `db:"attestations_scheduled"` + AttestationsExecuted uint64 `db:"attestations_executed"` + BlocksScheduled uint64 `db:"blocks_scheduled"` + BlocksProposed uint64 `db:"blocks_proposed"` + SyncScheduled uint64 `db:"sync_scheduled"` + SyncExecuted uint64 `db:"sync_executed"` + Slashed bool `db:"slashed"` + BlocksSlashingCount uint64 `db:"blocks_slashing_count"` + } + var queryResult []QueryResult + query, args, err := rewardsDs.Prepared(true).ToSQL() if err != nil { return fmt.Errorf("error preparing query: %v", err) @@ -318,6 +334,50 @@ func (d *DataAccessService) GetValidatorDashboardRewards(ctx context.Context, da if err != nil { return fmt.Errorf("error retrieving rewards data: %v", err) } + + rpValidators := make(map[uint64]t.RpMinipoolInfo) + if protocolModes.RocketPool { + validators := make([]uint64, 0, len(queryResult)) + for _, row := range queryResult { + validators = append(validators, row.ValidatorIndex) + } + + rpValidators, err = d.getRocketPoolMinipoolInfos(ctx, validators) + if err != nil { + return err + } + } + + for _, row := range queryResult { + if len(queryResultSum) == 0 || + queryResultSum[len(queryResultSum)-1].Epoch != row.Epoch || + queryResultSum[len(queryResultSum)-1].GroupId != row.GroupId { + queryResultSum = append(queryResultSum, QueryResultSum{ + Epoch: row.Epoch, + GroupId: row.GroupId, + }) + } + + current := &queryResultSum[len(queryResultSum)-1] + + current.AttestationsScheduled += row.AttestationsScheduled + current.AttestationsExecuted += row.AttestationsExecuted + current.BlocksScheduled += row.BlocksScheduled + current.BlocksProposed += row.BlocksProposed + current.SyncScheduled += row.SyncScheduled + current.SyncExecuted += row.SyncExecuted + if row.Slashed { + current.Slashed++ + } + current.BlocksSlashingCount += row.BlocksSlashingCount + + reward := utils.GWeiToWei(big.NewInt(row.ClRewards)) + if rpValidator, ok := rpValidators[row.ValidatorIndex]; ok && protocolModes.RocketPool { + reward = reward.Mul(d.getRocketPoolOperatorFactor(rpValidator)) + } + current.ClRewards = current.ClRewards.Add(reward) + } + return nil }) @@ -361,7 +421,7 @@ func (d *DataAccessService) GetValidatorDashboardRewards(ctx context.Context, da type TotalEpochInfo struct { Groups []uint64 - ClRewards int64 + ClRewards decimal.Decimal ElRewards decimal.Decimal AttestationsScheduled uint64 AttestationsExecuted uint64 @@ -373,7 +433,7 @@ func (d *DataAccessService) GetValidatorDashboardRewards(ctx context.Context, da } totalEpochInfo := make(map[uint64]*TotalEpochInfo, 0) - for _, res := range queryResult { + for _, res := range queryResultSum { duty := t.VDBRewardsTableDuty{} if res.AttestationsScheduled > 0 { attestationPercentage := (float64(res.AttestationsExecuted) / float64(res.AttestationsScheduled)) * 100.0 @@ -388,7 +448,7 @@ func (d *DataAccessService) GetValidatorDashboardRewards(ctx context.Context, da duty.Sync = &SyncPercentage } - slashings := res.SlashedInEpoch + res.SlashedAmount + slashings := res.Slashed + res.BlocksSlashingCount if slashings > 0 { duty.Slashing = &slashings } @@ -401,7 +461,7 @@ func (d *DataAccessService) GetValidatorDashboardRewards(ctx context.Context, da GroupId: res.GroupId, Reward: t.ClElValue[decimal.Decimal]{ El: elRewards[res.Epoch][res.GroupId], - Cl: utils.GWeiToWei(big.NewInt(res.ClRewards)), + Cl: res.ClRewards, }, }) @@ -410,7 +470,7 @@ func (d *DataAccessService) GetValidatorDashboardRewards(ctx context.Context, da totalEpochInfo[res.Epoch] = &TotalEpochInfo{} } totalEpochInfo[res.Epoch].Groups = append(totalEpochInfo[res.Epoch].Groups, uint64(res.GroupId)) - totalEpochInfo[res.Epoch].ClRewards += res.ClRewards + totalEpochInfo[res.Epoch].ClRewards = totalEpochInfo[res.Epoch].ClRewards.Add(res.ClRewards) totalEpochInfo[res.Epoch].ElRewards = totalEpochInfo[res.Epoch].ElRewards.Add(elRewards[res.Epoch][res.GroupId]) totalEpochInfo[res.Epoch].AttestationsScheduled += res.AttestationsScheduled totalEpochInfo[res.Epoch].AttestationsExecuted += res.AttestationsExecuted @@ -457,7 +517,7 @@ func (d *DataAccessService) GetValidatorDashboardRewards(ctx context.Context, da GroupId: t.AllGroups, Reward: t.ClElValue[decimal.Decimal]{ El: totalInfo.ElRewards, - Cl: utils.GWeiToWei(big.NewInt(totalInfo.ClRewards)), + Cl: totalInfo.ClRewards, }, } } @@ -527,31 +587,32 @@ func (d *DataAccessService) GetValidatorDashboardGroupRewards(ctx context.Contex } // ------------------------------------------------------------------------------------------------------------------ - // Build the query that serves as base for both the main and EL rewards queries + // Build the main and EL rewards queries rewardsDs := goqu.Dialect("postgres"). From(goqu.L("validator_dashboard_data_epoch e")). With("validators", goqu.L("(SELECT validator_index as validator_index, group_id FROM users_val_dashboards_validators WHERE dashboard_id = ?)", dashboardId.Id)). Select( - goqu.L("COALESCE(e.attestations_source_reward, 0) AS attestations_source_reward"), - goqu.L("COALESCE(e.attestations_target_reward, 0) AS attestations_target_reward"), - goqu.L("COALESCE(e.attestations_head_reward, 0) AS attestations_head_reward"), - goqu.L("COALESCE(e.attestations_inactivity_reward, 0) AS attestations_inactivity_reward"), - goqu.L("COALESCE(e.attestations_inclusion_reward, 0) AS attestations_inclusion_reward"), - goqu.L("COALESCE(e.attestations_scheduled, 0) AS attestations_scheduled"), - goqu.L("COALESCE(e.attestation_head_executed, 0) AS attestation_head_executed"), - goqu.L("COALESCE(e.attestation_source_executed, 0) AS attestation_source_executed"), - goqu.L("COALESCE(e.attestation_target_executed, 0) AS attestation_target_executed"), - goqu.L("COALESCE(e.blocks_scheduled, 0) AS blocks_scheduled"), - goqu.L("COALESCE(e.blocks_proposed, 0) AS blocks_proposed"), - goqu.L("COALESCE(e.blocks_cl_reward, 0) AS blocks_cl_reward"), - goqu.L("COALESCE(e.sync_scheduled, 0) AS sync_scheduled"), - goqu.L("COALESCE(e.sync_executed, 0) AS sync_executed"), - goqu.L("COALESCE(e.sync_rewards, 0) AS sync_rewards"), - goqu.L("(CASE WHEN e.slashed THEN 1 ELSE 0 END) AS slashed_in_epoch"), - goqu.L("COALESCE(e.blocks_slashing_count, 0) AS slashed_amount"), - goqu.L("COALESCE(e.blocks_cl_slasher_reward, 0) AS slasher_reward"), - goqu.L("COALESCE(e.blocks_cl_attestations_reward, 0) AS blocks_cl_attestations_reward"), - goqu.L("COALESCE(e.blocks_cl_sync_aggregate_reward, 0) AS blocks_cl_sync_aggregate_reward")). + goqu.L("e.validator_index"), + goqu.L("e.attestations_source_reward"), + goqu.L("e.attestations_target_reward"), + goqu.L("e.attestations_head_reward"), + goqu.L("e.attestations_inactivity_reward"), + goqu.L("e.attestations_inclusion_reward"), + goqu.L("e.attestations_scheduled"), + goqu.L("e.attestation_head_executed"), + goqu.L("e.attestation_source_executed"), + goqu.L("e.attestation_target_executed"), + goqu.L("e.blocks_scheduled"), + goqu.L("e.blocks_proposed"), + goqu.L("e.blocks_cl_reward"), + goqu.L("e.sync_scheduled"), + goqu.L("e.sync_executed"), + goqu.L("e.sync_rewards"), + goqu.L("e.slashed"), + goqu.L("e.blocks_slashing_count"), + goqu.L("e.blocks_cl_slasher_reward"), + goqu.L("e.blocks_cl_attestations_reward"), + goqu.L("e.blocks_cl_sync_aggregate_reward")). Where(goqu.L("e.epoch_timestamp = fromUnixTimestamp(?)", utils.EpochToTime(epoch).Unix())) elDs := goqu.Dialect("postgres"). @@ -593,6 +654,8 @@ func (d *DataAccessService) GetValidatorDashboardGroupRewards(ctx context.Contex // ------------------------------------------------------------------------------------------------------------------ // Build the main query and get the data queryResult := []struct { + ValidatorIndex uint64 `db:"validator_index"` + AttestationSourceReward decimal.Decimal `db:"attestations_source_reward"` AttestationTargetReward decimal.Decimal `db:"attestations_target_reward"` AttestationHeadReward decimal.Decimal `db:"attestations_head_reward"` @@ -612,10 +675,10 @@ func (d *DataAccessService) GetValidatorDashboardGroupRewards(ctx context.Contex SyncExecuted uint32 `db:"sync_executed"` SyncRewards decimal.Decimal `db:"sync_rewards"` - SlashedInEpoch bool `db:"slashed_in_epoch"` - SlashedAmount uint32 `db:"slashed_amount"` - SlasherRewards decimal.Decimal `db:"slasher_reward"` + Slashed bool `db:"slashed"` + BlocksSlashingCount uint32 `db:"blocks_slashing_count"` + BlocksClSlasherReward decimal.Decimal `db:"blocks_cl_slasher_reward"` BlocksClAttestationsReward decimal.Decimal `db:"blocks_cl_attestations_reward"` BlockClSyncAggregateReward decimal.Decimal `db:"blocks_cl_sync_aggregate_reward"` }{} @@ -656,45 +719,62 @@ func (d *DataAccessService) GetValidatorDashboardGroupRewards(ctx context.Contex // ------------------------------------------------------------------------------------------------------------------ // Create the result + rpValidators := make(map[uint64]t.RpMinipoolInfo) + if protocolModes.RocketPool { + validators := make([]uint64, 0, len(queryResult)) + for _, row := range queryResult { + validators = append(validators, row.ValidatorIndex) + } + + rpValidators, err = d.getRocketPoolMinipoolInfos(ctx, validators) + if err != nil { + return nil, err + } + } + gWei := decimal.NewFromInt(1e9) for _, entry := range queryResult { - ret.AttestationsHead.Income = ret.AttestationsHead.Income.Add(entry.AttestationHeadReward.Mul(gWei)) + rpFactor := decimal.NewFromInt(1) + if rpValidator, ok := rpValidators[entry.ValidatorIndex]; ok && protocolModes.RocketPool { + rpFactor = d.getRocketPoolOperatorFactor(rpValidator) + } + + ret.AttestationsHead.Income = ret.AttestationsHead.Income.Add(entry.AttestationHeadReward.Mul(gWei).Mul(rpFactor)) ret.AttestationsHead.StatusCount.Success += uint64(entry.AttestationHeadExecuted) ret.AttestationsHead.StatusCount.Failed += uint64(entry.AttestationsScheduled) - uint64(entry.AttestationHeadExecuted) - ret.AttestationsSource.Income = ret.AttestationsSource.Income.Add(entry.AttestationSourceReward.Mul(gWei)) + ret.AttestationsSource.Income = ret.AttestationsSource.Income.Add(entry.AttestationSourceReward.Mul(gWei).Mul(rpFactor)) ret.AttestationsSource.StatusCount.Success += uint64(entry.AttestationSourceExecuted) ret.AttestationsSource.StatusCount.Failed += uint64(entry.AttestationsScheduled) - uint64(entry.AttestationSourceExecuted) - ret.AttestationsTarget.Income = ret.AttestationsTarget.Income.Add(entry.AttestationTargetReward.Mul(gWei)) + ret.AttestationsTarget.Income = ret.AttestationsTarget.Income.Add(entry.AttestationTargetReward.Mul(gWei).Mul(rpFactor)) ret.AttestationsTarget.StatusCount.Success += uint64(entry.AttestationTargetExecuted) ret.AttestationsTarget.StatusCount.Failed += uint64(entry.AttestationsScheduled) - uint64(entry.AttestationTargetExecuted) - ret.Inactivity.Income = ret.Inactivity.Income.Add(entry.AttestationInactivitytReward.Mul(gWei)) + ret.Inactivity.Income = ret.Inactivity.Income.Add(entry.AttestationInactivitytReward.Mul(gWei).Mul(rpFactor)) if entry.AttestationInactivitytReward.LessThan(decimal.Zero) { ret.Inactivity.StatusCount.Failed++ } else { ret.Inactivity.StatusCount.Success++ } - ret.Proposal.Income = ret.Proposal.Income.Add(entry.BlocksClReward.Mul(gWei)) + ret.Proposal.Income = ret.Proposal.Income.Add(entry.BlocksClReward.Mul(gWei).Mul(rpFactor)) ret.Proposal.StatusCount.Success += uint64(entry.BlocksProposed) ret.Proposal.StatusCount.Failed += uint64(entry.BlocksScheduled) - uint64(entry.BlocksProposed) - ret.Sync.Income = ret.Sync.Income.Add(entry.SyncRewards.Mul(gWei)) + ret.Sync.Income = ret.Sync.Income.Add(entry.SyncRewards.Mul(gWei).Mul(rpFactor)) ret.Sync.StatusCount.Success += uint64(entry.SyncExecuted) ret.Sync.StatusCount.Failed += uint64(entry.SyncScheduled) - uint64(entry.SyncExecuted) - ret.Slashing.Income = ret.Slashing.Income.Add(entry.SlasherRewards.Mul(gWei)) - ret.Slashing.StatusCount.Success += uint64(entry.SlashedAmount) - if entry.SlashedInEpoch { + ret.Slashing.StatusCount.Success += uint64(entry.BlocksSlashingCount) + if entry.Slashed { ret.Slashing.StatusCount.Failed++ } - ret.ProposalClAttIncReward = ret.ProposalClAttIncReward.Add(entry.BlocksClAttestationsReward.Mul(gWei)) - ret.ProposalClSyncIncReward = ret.ProposalClSyncIncReward.Add(entry.BlockClSyncAggregateReward.Mul(gWei)) - ret.ProposalClSlashingIncReward = ret.ProposalClSlashingIncReward.Add(entry.SlasherRewards.Mul(gWei)) + ret.ProposalClAttIncReward = ret.ProposalClAttIncReward.Add(entry.BlocksClAttestationsReward.Mul(gWei).Mul(rpFactor)) + ret.ProposalClSyncIncReward = ret.ProposalClSyncIncReward.Add(entry.BlockClSyncAggregateReward.Mul(gWei).Mul(rpFactor)) + ret.ProposalClSlashingIncReward = ret.ProposalClSlashingIncReward.Add(entry.BlocksClSlasherReward.Mul(gWei).Mul(rpFactor)) } ret.Proposal.Income = ret.Proposal.Income.Add(elRewards) @@ -719,9 +799,10 @@ func (d *DataAccessService) GetValidatorDashboardRewardsChart(ctx context.Contex } // ------------------------------------------------------------------------------------------------------------------ - // Build the query that serves as base for both the main and EL rewards queries + // Build the main and EL rewards queries rewardsDs := goqu.Dialect("postgres"). Select( + goqu.L("e.validator_index"), goqu.L("e.epoch"), goqu.L(`SUM(COALESCE(e.attestations_reward, 0) + COALESCE(e.blocks_cl_reward, 0) + COALESCE(e.sync_rewards, 0)) AS cl_rewards`)). From(goqu.L("validator_dashboard_data_epoch e")). @@ -756,7 +837,6 @@ func (d *DataAccessService) GetValidatorDashboardRewardsChart(ctx context.Contex if dashboardId.AggregateGroups { rewardsDs = rewardsDs. SelectAppend(goqu.L("?::smallint AS result_group_id", t.DefaultGroupId)). - GroupBy(goqu.L("e.epoch")). Order(goqu.L("e.epoch").Asc()) elDs = elDs. SelectAppend(goqu.L("?::smallint AS result_group_id", t.DefaultGroupId)). @@ -765,7 +845,6 @@ func (d *DataAccessService) GetValidatorDashboardRewardsChart(ctx context.Contex } else { rewardsDs = rewardsDs. SelectAppend(goqu.L("v.group_id AS result_group_id")). - GroupBy(goqu.L("e.epoch"), goqu.L("result_group_id")). Order(goqu.L("e.epoch").Asc(), goqu.L("result_group_id").Asc()) elDs = elDs. SelectAppend(goqu.L("v.group_id AS result_group_id")). @@ -777,7 +856,6 @@ func (d *DataAccessService) GetValidatorDashboardRewardsChart(ctx context.Contex rewardsDs = rewardsDs. SelectAppend(goqu.L("?::smallint AS result_group_id", t.DefaultGroupId)). Where(goqu.L("e.validator_index IN ?", dashboardId.Validators)). - GroupBy(goqu.L("e.epoch")). Order(goqu.L("e.epoch").Asc()) elDs = elDs. SelectAppend(goqu.L("?::smallint AS result_group_id", t.DefaultGroupId)). @@ -788,13 +866,21 @@ func (d *DataAccessService) GetValidatorDashboardRewardsChart(ctx context.Contex // ------------------------------------------------------------------------------------------------------------------ // Build the main query and get the data - queryResult := []struct { - Epoch uint64 `db:"epoch"` - GroupId uint64 `db:"result_group_id"` - ClRewards int64 `db:"cl_rewards"` - }{} + type QueryResultSum struct { + Epoch uint64 + GroupId uint64 + ClRewards decimal.Decimal + } + var queryResultSum []QueryResultSum wg.Go(func() error { + queryResult := []struct { + ValidatorIndex uint64 `db:"validator_index"` + Epoch uint64 `db:"epoch"` + GroupId uint64 `db:"result_group_id"` + ClRewards int64 `db:"cl_rewards"` + }{} + query, args, err := rewardsDs.Prepared(true).ToSQL() if err != nil { return fmt.Errorf("error preparing query: %v", err) @@ -804,6 +890,38 @@ func (d *DataAccessService) GetValidatorDashboardRewardsChart(ctx context.Contex if err != nil { return fmt.Errorf("error retrieving rewards chart data: %v", err) } + + rpValidators := make(map[uint64]t.RpMinipoolInfo) + if protocolModes.RocketPool { + validators := make([]uint64, 0, len(queryResult)) + for _, row := range queryResult { + validators = append(validators, row.ValidatorIndex) + } + + rpValidators, err = d.getRocketPoolMinipoolInfos(ctx, validators) + if err != nil { + return err + } + } + + for _, entry := range queryResult { + if len(queryResultSum) == 0 || + queryResultSum[len(queryResultSum)-1].Epoch != entry.Epoch || + queryResultSum[len(queryResultSum)-1].GroupId != entry.GroupId { + queryResultSum = append(queryResultSum, QueryResultSum{ + Epoch: entry.Epoch, + GroupId: entry.GroupId, + }) + } + + current := &queryResultSum[len(queryResultSum)-1] + reward := utils.GWeiToWei(big.NewInt(entry.ClRewards)) + if rpValidator, ok := rpValidators[entry.ValidatorIndex]; ok && protocolModes.RocketPool { + reward = reward.Mul(d.getRocketPoolOperatorFactor(rpValidator)) + } + current.ClRewards = current.ClRewards.Add(reward) + } + return nil }) @@ -846,7 +964,7 @@ func (d *DataAccessService) GetValidatorDashboardRewardsChart(ctx context.Contex epochData := make(map[uint64]map[uint64]t.ClElValue[decimal.Decimal]) epochList := make([]uint64, 0) - for _, res := range queryResult { + for _, res := range queryResultSum { if _, ok := epochData[res.Epoch]; !ok { epochData[res.Epoch] = make(map[uint64]t.ClElValue[decimal.Decimal]) epochList = append(epochList, res.Epoch) @@ -854,7 +972,7 @@ func (d *DataAccessService) GetValidatorDashboardRewardsChart(ctx context.Contex epochData[res.Epoch][res.GroupId] = t.ClElValue[decimal.Decimal]{ El: elRewards[res.Epoch][res.GroupId], - Cl: utils.GWeiToWei(big.NewInt(res.ClRewards)), + Cl: res.ClRewards, } } @@ -954,31 +1072,31 @@ func (d *DataAccessService) GetValidatorDashboardDuties(ctx context.Context, das rewardsDs := goqu.Dialect("postgres"). Select( goqu.L("e.validator_index"), - goqu.L("COALESCE(e.attestations_scheduled, 0) AS attestations_scheduled"), - goqu.L("COALESCE(e.attestation_source_executed, 0) AS attestation_source_executed"), - goqu.L("COALESCE(e.attestations_source_reward, 0) AS attestations_source_reward"), - goqu.L("COALESCE(e.attestation_target_executed, 0) AS attestation_target_executed"), - goqu.L("COALESCE(e.attestations_target_reward, 0) AS attestations_target_reward"), - goqu.L("COALESCE(e.attestation_head_executed, 0) AS attestation_head_executed"), - goqu.L("COALESCE(e.attestations_head_reward, 0) AS attestations_head_reward"), - goqu.L("COALESCE(e.sync_scheduled, 0) AS sync_scheduled"), - goqu.L("COALESCE(e.sync_executed, 0) AS sync_executed"), - goqu.L("COALESCE(e.sync_rewards, 0) AS sync_rewards"), - goqu.L("e.slashed AS slashed_in_epoch"), - goqu.L("COALESCE(e.blocks_slashing_count, 0) AS slashed_amount"), - goqu.L("COALESCE(e.blocks_cl_slasher_reward, 0) AS slasher_reward"), - goqu.L("COALESCE(e.blocks_scheduled, 0) AS blocks_scheduled"), - goqu.L("COALESCE(e.blocks_proposed, 0) AS blocks_proposed"), - goqu.L("COALESCE(e.blocks_cl_attestations_reward, 0) AS blocks_cl_attestations_reward"), - goqu.L("COALESCE(e.blocks_cl_sync_aggregate_reward, 0) AS blocks_cl_sync_aggregate_reward")). + goqu.L("e.attestations_scheduled"), + goqu.L("e.attestation_source_executed"), + goqu.L("e.attestations_source_reward"), + goqu.L("e.attestation_target_executed"), + goqu.L("e.attestations_target_reward"), + goqu.L("e.attestation_head_executed"), + goqu.L("e.attestations_head_reward"), + goqu.L("e.sync_scheduled"), + goqu.L("e.sync_executed"), + goqu.L("e.sync_rewards"), + goqu.L("e.slashed"), + goqu.L("e.blocks_slashing_count"), + goqu.L("e.blocks_cl_slasher_reward"), + goqu.L("e.blocks_scheduled"), + goqu.L("e.blocks_proposed"), + goqu.L("e.blocks_cl_attestations_reward"), + goqu.L("e.blocks_cl_sync_aggregate_reward")). From(goqu.L("validator_dashboard_data_epoch e")). Where(goqu.L("e.epoch_timestamp = fromUnixTimestamp(?)", utils.EpochToTime(epoch).Unix())). Where(goqu.L(` - (COALESCE(e.attestations_scheduled, 0) + - COALESCE(e.sync_scheduled,0) + - COALESCE(e.blocks_scheduled,0) + + (e.attestations_scheduled + + e.sync_scheduled + + e.blocks_scheduled + CASE WHEN e.slashed THEN 1 ELSE 0 END + - COALESCE(e.blocks_slashing_count, 0)) > 0`)) + e.blocks_slashing_count) > 0`)) elDs := goqu.Dialect("postgres"). Select( @@ -1037,28 +1155,46 @@ func (d *DataAccessService) GetValidatorDashboardDuties(ctx context.Context, das // ------------------------------------------------------------------------------------------------------------------ // Get the main data - queryResult := []struct { - ValidatorIndex uint64 `db:"validator_index"` - AttestationsScheduled uint64 `db:"attestations_scheduled"` - AttestationsSourceExecuted uint64 `db:"attestation_source_executed"` - AttestationsSourceReward int64 `db:"attestations_source_reward"` - AttestationsTargetExecuted uint64 `db:"attestation_target_executed"` - AttestationsTargetReward int64 `db:"attestations_target_reward"` - AttestationsHeadExecuted uint64 `db:"attestation_head_executed"` - AttestationsHeadReward int64 `db:"attestations_head_reward"` - SyncScheduled uint64 `db:"sync_scheduled"` - SyncExecuted uint64 `db:"sync_executed"` - SyncRewards int64 `db:"sync_rewards"` - SlashedInEpoch bool `db:"slashed_in_epoch"` - SlashedAmount uint64 `db:"slashed_amount"` - SlasherReward int64 `db:"slasher_reward"` - BlocksScheduled uint64 `db:"blocks_scheduled"` - BlocksProposed uint64 `db:"blocks_proposed"` - BlocksClAttestationsReward int64 `db:"blocks_cl_attestations_reward"` - BlocksClSyncAggregateReward int64 `db:"blocks_cl_sync_aggregate_reward"` - }{} + type QueryResultBase struct { + ValidatorIndex uint64 `db:"validator_index"` + AttestationsScheduled uint64 `db:"attestations_scheduled"` + AttestationsSourceExecuted uint64 `db:"attestation_source_executed"` + AttestationsTargetExecuted uint64 `db:"attestation_target_executed"` + AttestationsHeadExecuted uint64 `db:"attestation_head_executed"` + SyncScheduled uint64 `db:"sync_scheduled"` + SyncExecuted uint64 `db:"sync_executed"` + Slashed bool `db:"slashed"` + BlocksSlashingCount uint64 `db:"blocks_slashing_count"` + BlocksScheduled uint64 `db:"blocks_scheduled"` + BlocksProposed uint64 `db:"blocks_proposed"` + } + + type QueryResult struct { + QueryResultBase + AttestationsSourceReward int64 `db:"attestations_source_reward"` + AttestationsTargetReward int64 `db:"attestations_target_reward"` + AttestationsHeadReward int64 `db:"attestations_head_reward"` + SyncRewards int64 `db:"sync_rewards"` + BlocksClSlasherReward int64 `db:"blocks_cl_slasher_reward"` + BlocksClAttestationsReward int64 `db:"blocks_cl_attestations_reward"` + BlocksClSyncAggregateReward int64 `db:"blocks_cl_sync_aggregate_reward"` + } + type QueryResultAdjusted struct { + QueryResultBase + AttestationsSourceReward decimal.Decimal + AttestationsTargetReward decimal.Decimal + AttestationsHeadReward decimal.Decimal + SyncRewards decimal.Decimal + BlocksClSlasherReward decimal.Decimal + BlocksClAttestationsReward decimal.Decimal + BlocksClSyncAggregateReward decimal.Decimal + } + + var queryResultAdjusted []QueryResultAdjusted wg.Go(func() error { + var queryResult []QueryResult + query, args, err := rewardsDs.Prepared(true).ToSQL() if err != nil { return fmt.Errorf("error preparing query: %v", err) @@ -1068,6 +1204,53 @@ func (d *DataAccessService) GetValidatorDashboardDuties(ctx context.Context, das if err != nil { return fmt.Errorf("error retrieving validator rewards data: %v", err) } + + rpValidators := make(map[uint64]t.RpMinipoolInfo) + if protocolModes.RocketPool { + validators := make([]uint64, 0, len(queryResult)) + for _, row := range queryResult { + validators = append(validators, row.ValidatorIndex) + } + + rpValidators, err = d.getRocketPoolMinipoolInfos(ctx, validators) + if err != nil { + return err + } + } + + for _, entry := range queryResult { + queryResultAdjusted = append(queryResultAdjusted, QueryResultAdjusted{ + QueryResultBase: QueryResultBase{ + ValidatorIndex: entry.ValidatorIndex, + AttestationsScheduled: entry.AttestationsScheduled, + AttestationsSourceExecuted: entry.AttestationsSourceExecuted, + AttestationsTargetExecuted: entry.AttestationsTargetExecuted, + AttestationsHeadExecuted: entry.AttestationsHeadExecuted, + SyncScheduled: entry.SyncScheduled, + SyncExecuted: entry.SyncExecuted, + Slashed: entry.Slashed, + BlocksSlashingCount: entry.BlocksSlashingCount, + BlocksScheduled: entry.BlocksScheduled, + BlocksProposed: entry.BlocksProposed, + }, + }) + + current := &queryResultAdjusted[len(queryResultAdjusted)-1] + + rpFactor := decimal.NewFromInt(1) + if rpValidator, ok := rpValidators[entry.ValidatorIndex]; ok && protocolModes.RocketPool { + rpFactor = d.getRocketPoolOperatorFactor(rpValidator) + } + + current.AttestationsSourceReward = utils.GWeiToWei(big.NewInt(entry.AttestationsSourceReward)).Mul(rpFactor) + current.AttestationsTargetReward = utils.GWeiToWei(big.NewInt(entry.AttestationsTargetReward)).Mul(rpFactor) + current.AttestationsHeadReward = utils.GWeiToWei(big.NewInt(entry.AttestationsHeadReward)).Mul(rpFactor) + current.SyncRewards = utils.GWeiToWei(big.NewInt(entry.SyncRewards)).Mul(rpFactor) + current.BlocksClSlasherReward = utils.GWeiToWei(big.NewInt(entry.BlocksClSlasherReward)).Mul(rpFactor) + current.BlocksClAttestationsReward = utils.GWeiToWei(big.NewInt(entry.BlocksClAttestationsReward)).Mul(rpFactor) + current.BlocksClSyncAggregateReward = utils.GWeiToWei(big.NewInt(entry.BlocksClSyncAggregateReward)).Mul(rpFactor) + } + return nil }) @@ -1104,11 +1287,10 @@ func (d *DataAccessService) GetValidatorDashboardDuties(ctx context.Context, das // ------------------------------------------------------------------------------------------------------------------ // Create the result cursorData := make([]t.ValidatorDutiesCursor, 0) - for _, res := range queryResult { - clReward := utils.GWeiToWei(big.NewInt( - res.AttestationsHeadReward + res.AttestationsSourceReward + res.AttestationsTargetReward + - res.SyncRewards + - res.BlocksClAttestationsReward + res.BlocksClSyncAggregateReward + res.SlasherReward)) + for _, res := range queryResultAdjusted { + clReward := res.AttestationsHeadReward.Add(res.AttestationsSourceReward).Add(res.AttestationsTargetReward). + Add(res.SyncRewards). + Add(res.BlocksClAttestationsReward).Add(res.BlocksClSyncAggregateReward).Add(res.BlocksClSlasherReward) totalReward := clReward.Add(elRewards[res.ValidatorIndex]) row := t.VDBEpochDutiesTableRow{ @@ -1125,16 +1307,16 @@ func (d *DataAccessService) GetValidatorDashboardDuties(ctx context.Context, das row.Duties.SyncCount = res.SyncExecuted // Get slashing data - if res.SlashedInEpoch || res.SlashedAmount > 0 { + if res.Slashed || res.BlocksSlashingCount > 0 { slashedEvent := t.ValidatorHistoryEvent{ - Income: utils.GWeiToWei(big.NewInt(res.SlasherReward)), + Income: res.BlocksClSlasherReward, } - if res.SlashedInEpoch { - if res.SlashedAmount > 0 { + if res.Slashed { + if res.BlocksSlashingCount > 0 { slashedEvent.Status = "partial" } slashedEvent.Status = "failed" - } else if res.SlashedAmount > 0 { + } else if res.BlocksSlashingCount > 0 { slashedEvent.Status = "success" } row.Duties.Slashing = &slashedEvent @@ -1144,9 +1326,9 @@ func (d *DataAccessService) GetValidatorDashboardDuties(ctx context.Context, das if res.BlocksScheduled > 0 { proposalEvent := t.ValidatorHistoryProposal{ ElIncome: elRewards[res.ValidatorIndex], - ClAttestationInclusionIncome: utils.GWeiToWei(big.NewInt(res.BlocksClAttestationsReward)), - ClSyncInclusionIncome: utils.GWeiToWei(big.NewInt(res.BlocksClSyncAggregateReward)), - ClSlashingInclusionIncome: utils.GWeiToWei(big.NewInt(res.SlasherReward)), + ClAttestationInclusionIncome: res.BlocksClAttestationsReward, + ClSyncInclusionIncome: res.BlocksClSyncAggregateReward, + ClSlashingInclusionIncome: res.BlocksClSlasherReward, } if res.BlocksProposed == 0 { @@ -1274,10 +1456,10 @@ func (d *DataAccessService) GetValidatorDashboardDuties(ctx context.Context, das return result, p, nil } -func (d *DataAccessService) getValidatorHistoryEvent(income int64, scheduledEvents, executedEvents uint64) *t.ValidatorHistoryEvent { +func (d *DataAccessService) getValidatorHistoryEvent(income decimal.Decimal, scheduledEvents, executedEvents uint64) *t.ValidatorHistoryEvent { if scheduledEvents > 0 { validatorHistoryEvent := t.ValidatorHistoryEvent{ - Income: utils.GWeiToWei(big.NewInt(income)), + Income: income, } if executedEvents == 0 { validatorHistoryEvent.Status = "failed" diff --git a/backend/pkg/api/data_access/vdb_summary.go b/backend/pkg/api/data_access/vdb_summary.go index 99c9d51f2..f742c79ac 100644 --- a/backend/pkg/api/data_access/vdb_summary.go +++ b/backend/pkg/api/data_access/vdb_summary.go @@ -42,11 +42,6 @@ func (d *DataAccessService) GetValidatorDashboardSummary(ctx context.Context, da return nil, nil, err } - validatorMapping, err := d.services.GetCurrentValidatorMapping() - if err != nil { - return nil, nil, err - } - // Searching for a group name is not supported when aggregating groups or for guest dashboards groupNameSearchEnabled := !dashboardId.AggregateGroups && dashboardId.Validators == nil @@ -176,14 +171,14 @@ func (d *DataAccessService) GetValidatorDashboardSummary(ctx context.Context, da return result, &paging, nil } - rpValidators := make(map[uint64]t.RpOperatorInfo) + rpValidators := make(map[uint64]t.RpMinipoolInfo) if protocolModes.RocketPool { validators := make([]uint64, 0, len(queryResult)) for _, row := range queryResult { validators = append(validators, row.ValidatorIndex) } - rpValidators, err = d.getRocketPoolOperators(ctx, validators) + rpValidators, err = d.getRocketPoolMinipoolInfos(ctx, validators) if err != nil { return nil, nil, err } @@ -237,12 +232,9 @@ func (d *DataAccessService) GetValidatorDashboardSummary(ctx context.Context, da clRewardWei := utils.GWeiToWei(big.NewInt(row.ClRewards)) if rpValidator, ok := rpValidators[row.ValidatorIndex]; ok && protocolModes.RocketPool { - metadata := validatorMapping.ValidatorMetadata[row.ValidatorIndex] - effectiveBalance := utils.GWeiToWei(big.NewInt(int64(metadata.EffectiveBalance))) - groupSum.ClRewards = groupSum.ClRewards.Add(d.getRocketPoolOperatorReward(rpValidator, clRewardWei, effectiveBalance)) - } else { - groupSum.ClRewards = groupSum.ClRewards.Add(clRewardWei) + clRewardWei = clRewardWei.Mul(d.getRocketPoolOperatorFactor(rpValidator)) } + groupSum.ClRewards = groupSum.ClRewards.Add(clRewardWei) } queryResultSum := slices.Collect(maps.Values(queryResultSumMap)) @@ -740,7 +732,7 @@ func (d *DataAccessService) GetValidatorDashboardGroupSummary(ctx context.Contex validatorArr = validators } - rpValidators, err := d.getRocketPoolOperators(ctx, validatorArr) + rpValidators, err := d.getRocketPoolMinipoolInfos(ctx, validatorArr) if err != nil { return nil, fmt.Errorf("error retrieving rocketpool validators: %w", err) } @@ -800,7 +792,7 @@ func (d *DataAccessService) GetValidatorDashboardGroupSummary(ctx context.Contex return ret, nil } -func (d *DataAccessService) internal_getElClAPR(ctx context.Context, dashboardId t.VDBId, protocolModes t.VDBProtocolModes, groupId int64, rpValidators map[t.VDBValidator]t.RpOperatorInfo, hours int) (elIncome decimal.Decimal, elAPR float64, clIncome decimal.Decimal, clAPR float64, err error) { +func (d *DataAccessService) internal_getElClAPR(ctx context.Context, dashboardId t.VDBId, protocolModes t.VDBProtocolModes, groupId int64, rpValidators map[t.VDBValidator]t.RpMinipoolInfo, hours int) (elIncome decimal.Decimal, elAPR float64, clIncome decimal.Decimal, clAPR float64, err error) { table := "" switch hours { @@ -818,11 +810,6 @@ func (d *DataAccessService) internal_getElClAPR(ctx context.Context, dashboardId return decimal.Zero, 0, decimal.Zero, 0, fmt.Errorf("invalid hours value: %v", hours) } - validatorMapping, err := d.services.GetCurrentValidatorMapping() - if err != nil { - return decimal.Zero, 0, decimal.Zero, 0, err - } - type ClRewardsResult struct { ValidatorIndex uint64 `db:"validator_index"` EpochStart uint64 `db:"epoch_start"` @@ -835,9 +822,7 @@ func (d *DataAccessService) internal_getElClAPR(ctx context.Context, dashboardId Reward decimal.Decimal `db:"el_reward"` } - var rewardsResultTable []ClRewardsResult - var rewardsResultTotal []ClRewardsResult - + var clRewardsResult []ClRewardsResult var elRewardsResult []ElRewardsResult rewardsDs := goqu.Dialect("postgres"). @@ -868,8 +853,8 @@ func (d *DataAccessService) internal_getElClAPR(ctx context.Context, dashboardId return decimal.Zero, 0, decimal.Zero, 0, fmt.Errorf("error preparing query: %v", err) } - err = d.clickhouseReader.SelectContext(ctx, &rewardsResultTable, query, args...) - if err != nil || len(rewardsResultTable) == 0 { + err = d.clickhouseReader.SelectContext(ctx, &clRewardsResult, query, args...) + if err != nil || len(clRewardsResult) == 0 { return decimal.Zero, 0, decimal.Zero, 0, err } @@ -882,7 +867,7 @@ func (d *DataAccessService) internal_getElClAPR(ctx context.Context, dashboardId rewards := decimal.Zero deposits := decimal.Zero - for _, row := range rewardsResultTable { + for _, row := range clRewardsResult { if row.EpochStart < epochStart { epochStart = row.EpochStart } @@ -890,12 +875,9 @@ func (d *DataAccessService) internal_getElClAPR(ctx context.Context, dashboardId epochEnd = row.EpochEnd } - metadata := validatorMapping.ValidatorMetadata[row.ValidatorIndex] reward := utils.GWeiToWei(big.NewInt(row.Reward)) - if rpValidator, ok := rpValidators[row.ValidatorIndex]; ok { - effectiveBalance := utils.GWeiToWei(big.NewInt(int64(metadata.EffectiveBalance))) - rpReward := d.getRocketPoolOperatorReward(rpValidator, reward, effectiveBalance) + rpReward := reward.Mul(d.getRocketPoolOperatorFactor(rpValidator)) aprRewards = aprRewards.Add(rpReward) if protocolModes.RocketPool { rewards = rewards.Add(rpReward) @@ -927,13 +909,13 @@ func (d *DataAccessService) internal_getElClAPR(ctx context.Context, dashboardId return decimal.Zero, 0, decimal.Zero, 0, fmt.Errorf("error preparing query: %v", err) } - err = d.clickhouseReader.SelectContext(ctx, &rewardsResultTotal, query, args...) - if err != nil || len(rewardsResultTotal) == 0 { + err = d.clickhouseReader.SelectContext(ctx, &clRewardsResult, query, args...) + if err != nil || len(clRewardsResult) == 0 { return decimal.Zero, 0, decimal.Zero, 0, err } rewards = decimal.Zero - for _, row := range rewardsResultTotal { + for _, row := range clRewardsResult { if row.EpochStart < epochStartTotal { epochStartTotal = row.EpochStart } @@ -941,12 +923,10 @@ func (d *DataAccessService) internal_getElClAPR(ctx context.Context, dashboardId epochEndTotal = row.EpochEnd } - metadata := validatorMapping.ValidatorMetadata[row.ValidatorIndex] reward := utils.GWeiToWei(big.NewInt(row.Reward)) - if rpValidator, ok := rpValidators[row.ValidatorIndex]; ok && protocolModes.RocketPool { - effectiveBalance := utils.GWeiToWei(big.NewInt(int64(metadata.EffectiveBalance))) - rewards = rewards.Add(d.getRocketPoolOperatorReward(rpValidator, reward, effectiveBalance)) + rpReward := reward.Mul(d.getRocketPoolOperatorFactor(rpValidator)) + rewards = rewards.Add(rpReward) } else { rewards = rewards.Add(reward) } @@ -999,22 +979,26 @@ func (d *DataAccessService) internal_getElClAPR(ctx context.Context, dashboardId return decimal.Zero, 0, decimal.Zero, 0, err } - elRewards := decimal.Zero + aprRewards = decimal.Zero + rewards = decimal.Zero for _, row := range elRewardsResult { reward := row.Reward if rpValidator, ok := rpValidators[row.ValidatorIndex]; ok { - // TODO: This is not quite correct - for epochPeriodEnd, smoothingPoolReward := range rpValidator.SmoothingPoolReward { - if epochPeriodEnd >= epochStart && epochPeriodEnd <= epochEnd { - reward = reward.Add(smoothingPoolReward) - } + rpReward := reward.Mul(d.getRocketPoolOperatorFactor(rpValidator)) + aprRewards = aprRewards.Add(rpReward) + if protocolModes.RocketPool { + rewards = rewards.Add(rpReward) + } else { + rewards = rewards.Add(reward) } + } else { + aprRewards = aprRewards.Add(reward) + rewards = rewards.Add(reward) } - elRewards = elIncome.Add(reward) } if !deposits.IsZero() { - elAPR = elRewards.Div(decimal.NewFromInt(int64(aprDivisor))).Div(deposits).Mul(decimal.NewFromInt(24 * 365 * 100)).InexactFloat64() + elAPR = aprRewards.Div(decimal.NewFromInt(int64(aprDivisor))).Div(deposits).Mul(decimal.NewFromInt(24 * 365 * 100)).InexactFloat64() } if hours == -1 { @@ -1031,21 +1015,18 @@ func (d *DataAccessService) internal_getElClAPR(ctx context.Context, dashboardId return decimal.Zero, 0, decimal.Zero, 0, err } - elRewards = decimal.Zero + rewards = decimal.Zero for _, row := range elRewardsResult { reward := row.Reward - if rpValidator, ok := rpValidators[row.ValidatorIndex]; ok { - // TODO: This is not quite correct - for epochPeriodEnd, smoothingPoolReward := range rpValidator.SmoothingPoolReward { - if epochPeriodEnd >= epochStartTotal && epochPeriodEnd <= epochEndTotal { - reward = reward.Add(smoothingPoolReward) - } - } + if rpValidator, ok := rpValidators[row.ValidatorIndex]; ok && protocolModes.RocketPool { + rpReward := reward.Mul(d.getRocketPoolOperatorFactor(rpValidator)) + rewards = rewards.Add(rpReward) + } else { + rewards = rewards.Add(reward) } - elRewards = elIncome.Add(reward) } } - elIncome = elRewards + elIncome = rewards return elIncome, elAPR, clIncome, clAPR, nil } diff --git a/backend/pkg/api/types/archiver.go b/backend/pkg/api/types/archiver.go index b6c2267d4..b3930771f 100644 --- a/backend/pkg/api/types/archiver.go +++ b/backend/pkg/api/types/archiver.go @@ -18,10 +18,9 @@ type ArchiverDashboardArchiveReason struct { } // TODO: Find a good place for this -type RpOperatorInfo struct { +type RpMinipoolInfo struct { NodeFee float64 NodeDepositBalance decimal.Decimal UserDepositBalance decimal.Decimal - SmoothingPoolOptIn bool SmoothingPoolReward map[uint64]decimal.Decimal } From 764dd892419ebdc8e28a6ca9236a2bd2d6086695 Mon Sep 17 00:00:00 2001 From: Stefan Pletka <124689083+Eisei24@users.noreply.github.com> Date: Mon, 16 Sep 2024 14:55:48 +0200 Subject: [PATCH 03/19] Implemented el rewards --- backend/pkg/api/data_access/vdb_blocks.go | 3 + backend/pkg/api/data_access/vdb_rewards.go | 158 ++++++++++++--------- backend/pkg/api/data_access/vdb_summary.go | 10 +- 3 files changed, 104 insertions(+), 67 deletions(-) diff --git a/backend/pkg/api/data_access/vdb_blocks.go b/backend/pkg/api/data_access/vdb_blocks.go index e8c2aaaab..9f699109a 100644 --- a/backend/pkg/api/data_access/vdb_blocks.go +++ b/backend/pkg/api/data_access/vdb_blocks.go @@ -398,6 +398,9 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das data[i].RewardRecipient = &rewardRecp ensMapping[hexutil.Encode(proposal.FeeRecipient)] = "" reward.El = proposal.ElReward.Decimal.Mul(decimal.NewFromInt(1e18)) + if rpValidator, ok := rpValidators[proposal.Proposer]; ok && protocolModes.RocketPool { + reward.El = reward.El.Mul(d.getRocketPoolOperatorFactor(rpValidator)) + } } if proposal.ClReward.Valid { reward.Cl = proposal.ClReward.Decimal.Mul(decimal.NewFromInt(1e18)) diff --git a/backend/pkg/api/data_access/vdb_rewards.go b/backend/pkg/api/data_access/vdb_rewards.go index dda852ea3..5ed3f3c25 100644 --- a/backend/pkg/api/data_access/vdb_rewards.go +++ b/backend/pkg/api/data_access/vdb_rewards.go @@ -2,7 +2,6 @@ package dataaccess import ( "context" - "database/sql" "fmt" "math/big" "slices" @@ -17,7 +16,6 @@ import ( "github.com/gobitfly/beaconchain/pkg/commons/cache" "github.com/gobitfly/beaconchain/pkg/commons/utils" "github.com/lib/pq" - "github.com/pkg/errors" "github.com/shopspring/decimal" "golang.org/x/sync/errgroup" ) @@ -101,6 +99,7 @@ func (d *DataAccessService) GetValidatorDashboardRewards(ctx context.Context, da elDs := goqu.Dialect("postgres"). Select( goqu.L("b.epoch"), + goqu.L("b.proposer"), goqu.L("SUM(COALESCE(rb.value, ep.fee_recipient_reward * 1e18, 0)) AS el_rewards")). From(goqu.L("users_val_dashboards_validators v")). Where(goqu.L("b.epoch >= ?", startEpoch)). @@ -114,7 +113,8 @@ func (d *DataAccessService) GetValidatorDashboardRewards(ctx context.Context, da goqu.MAX("value").As("value")). GroupBy("exec_block_hash").As("rb"), goqu.On(goqu.L("rb.exec_block_hash = b.exec_block_hash")), - ) + ). + GroupBy(goqu.L("b.proposer")) if dashboardId.Validators == nil { rewardsDs = rewardsDs. @@ -224,8 +224,7 @@ func (d *DataAccessService) GetValidatorDashboardRewards(ctx context.Context, da SelectAppend(goqu.L("?::smallint AS result_group_id", t.DefaultGroupId)) elDs = elDs. - SelectAppend(goqu.L("?::smallint AS result_group_id", t.DefaultGroupId)). - GroupBy(goqu.L("b.epoch")) + SelectAppend(goqu.L("?::smallint AS result_group_id", t.DefaultGroupId)) if isReverseDirection { rewardsDs = rewardsDs.Order(goqu.L("e.epoch").Desc()) @@ -238,8 +237,7 @@ func (d *DataAccessService) GetValidatorDashboardRewards(ctx context.Context, da rewardsDs = rewardsDs. SelectAppend(goqu.L("v.group_id AS result_group_id")) elDs = elDs. - SelectAppend(goqu.L("v.group_id AS result_group_id")). - GroupBy(goqu.L("b.epoch"), goqu.L("result_group_id")) + SelectAppend(goqu.L("v.group_id AS result_group_id")) if isReverseDirection { rewardsDs = rewardsDs.Order(goqu.L("e.epoch").Desc(), goqu.L("result_group_id").Desc()) @@ -256,8 +254,7 @@ func (d *DataAccessService) GetValidatorDashboardRewards(ctx context.Context, da Where(goqu.L("e.validator_index IN ?", dashboardId.Validators)) elDs = elDs. SelectAppend(goqu.L("?::smallint AS result_group_id", t.DefaultGroupId)). - Where(goqu.L("b.proposer = ANY(?)", pq.Array(dashboardId.Validators))). - GroupBy(goqu.L("b.epoch")) + Where(goqu.L("b.proposer = ANY(?)", pq.Array(dashboardId.Validators))) if currentCursor.IsValid() { rewardsDs = rewardsDs.Where(goqu.L(fmt.Sprintf("e.epoch_timestamp %s fromUnixTimestamp(?)", sortSearchDirection), utils.EpochToTime(currentCursor.Epoch).Unix())) @@ -290,6 +287,21 @@ func (d *DataAccessService) GetValidatorDashboardRewards(ctx context.Context, da } } + // ------------------------------------------------------------------------------------------------------------------ + // Get rocketpool minipool infos if needed + rpValidators := make(map[uint64]t.RpMinipoolInfo) + if protocolModes.RocketPool { + validators, err := d.getDashboardValidators(ctx, dashboardId, nil) + if err != nil { + return nil, nil, fmt.Errorf("error retrieving validators from dashboard id: %v", err) + } + + rpValidators, err = d.getRocketPoolMinipoolInfos(ctx, validators) + if err != nil { + return nil, nil, err + } + } + // ------------------------------------------------------------------------------------------------------------------ // Build the main query and get the data @@ -335,19 +347,6 @@ func (d *DataAccessService) GetValidatorDashboardRewards(ctx context.Context, da return fmt.Errorf("error retrieving rewards data: %v", err) } - rpValidators := make(map[uint64]t.RpMinipoolInfo) - if protocolModes.RocketPool { - validators := make([]uint64, 0, len(queryResult)) - for _, row := range queryResult { - validators = append(validators, row.ValidatorIndex) - } - - rpValidators, err = d.getRocketPoolMinipoolInfos(ctx, validators) - if err != nil { - return err - } - } - for _, row := range queryResult { if len(queryResultSum) == 0 || queryResultSum[len(queryResultSum)-1].Epoch != row.Epoch || @@ -386,6 +385,7 @@ func (d *DataAccessService) GetValidatorDashboardRewards(ctx context.Context, da elRewards := make(map[uint64]map[int64]decimal.Decimal) wg.Go(func() error { elQueryResult := []struct { + Proposer uint64 `db:"proposer"` Epoch uint64 `db:"epoch"` GroupId int64 `db:"result_group_id"` ElRewards decimal.Decimal `db:"el_rewards"` @@ -405,7 +405,12 @@ func (d *DataAccessService) GetValidatorDashboardRewards(ctx context.Context, da if _, ok := elRewards[entry.Epoch]; !ok { elRewards[entry.Epoch] = make(map[int64]decimal.Decimal) } - elRewards[entry.Epoch][entry.GroupId] = entry.ElRewards + + reward := entry.ElRewards + if rpValidator, ok := rpValidators[entry.Proposer]; ok && protocolModes.RocketPool { + reward = reward.Mul(d.getRocketPoolOperatorFactor(rpValidator)) + } + elRewards[entry.Epoch][entry.GroupId] = elRewards[entry.Epoch][entry.GroupId].Add(reward) } return nil }) @@ -617,6 +622,7 @@ func (d *DataAccessService) GetValidatorDashboardGroupRewards(ctx context.Contex elDs := goqu.Dialect("postgres"). Select( + goqu.L("b.proposer"), goqu.L("COALESCE(SUM(COALESCE(rb.value, ep.fee_recipient_reward * 1e18, 0)), 0) AS blocks_el_reward")). From(goqu.L("users_val_dashboards_validators v")). LeftJoin(goqu.L("blocks b"), goqu.On(goqu.L("v.validator_index = b.proposer AND b.status = '1'"))). @@ -630,7 +636,8 @@ func (d *DataAccessService) GetValidatorDashboardGroupRewards(ctx context.Contex GroupBy("exec_block_hash").As("rb"), goqu.On(goqu.L("rb.exec_block_hash = b.exec_block_hash")), ). - Where(goqu.L("b.epoch = ?", epoch)) + Where(goqu.L("b.epoch = ?", epoch)). + GroupBy(goqu.L("b.proposer")) // handle the case when we have a list of validators @@ -698,17 +705,26 @@ func (d *DataAccessService) GetValidatorDashboardGroupRewards(ctx context.Contex // ------------------------------------------------------------------------------------------------------------------ // Get the EL rewards - var elRewards decimal.Decimal + elRewards := make(map[uint64]decimal.Decimal) wg.Go(func() error { + elQueryResult := []struct { + Proposer uint64 `db:"proposer"` + ElRewards decimal.Decimal `db:"blocks_el_reward"` + }{} + query, args, err := elDs.Prepared(true).ToSQL() if err != nil { return fmt.Errorf("error preparing query: %v", err) } - err = d.readerDb.GetContext(ctx, &elRewards, query, args...) - if err != nil && !errors.Is(err, sql.ErrNoRows) { + err = d.readerDb.SelectContext(ctx, &elQueryResult, query, args...) + if err != nil { return fmt.Errorf("error retrieving el rewards data for group rewards: %v", err) } + + for _, entry := range elQueryResult { + elRewards[entry.Proposer] = entry.ElRewards + } return nil }) @@ -759,7 +775,8 @@ func (d *DataAccessService) GetValidatorDashboardGroupRewards(ctx context.Contex ret.Inactivity.StatusCount.Success++ } - ret.Proposal.Income = ret.Proposal.Income.Add(entry.BlocksClReward.Mul(gWei).Mul(rpFactor)) + ret.Proposal.Income = ret.Proposal.Income.Add(entry.BlocksClReward.Mul(gWei).Add(elRewards[entry.ValidatorIndex]).Mul(rpFactor)) + ret.ProposalElReward = ret.ProposalElReward.Add(elRewards[entry.ValidatorIndex]) ret.Proposal.StatusCount.Success += uint64(entry.BlocksProposed) ret.Proposal.StatusCount.Failed += uint64(entry.BlocksScheduled) - uint64(entry.BlocksProposed) @@ -777,9 +794,6 @@ func (d *DataAccessService) GetValidatorDashboardGroupRewards(ctx context.Contex ret.ProposalClSlashingIncReward = ret.ProposalClSlashingIncReward.Add(entry.BlocksClSlasherReward.Mul(gWei).Mul(rpFactor)) } - ret.Proposal.Income = ret.Proposal.Income.Add(elRewards) - ret.ProposalElReward = elRewards - return ret, nil } @@ -811,6 +825,7 @@ func (d *DataAccessService) GetValidatorDashboardRewardsChart(ctx context.Contex elDs := goqu.Dialect("postgres"). Select( + goqu.L("b.proposer"), goqu.L("b.epoch"), goqu.L("SUM(COALESCE(rb.value, ep.fee_recipient_reward * 1e18, 0)) AS el_rewards")). From(goqu.L("users_val_dashboards_validators v")). @@ -825,7 +840,8 @@ func (d *DataAccessService) GetValidatorDashboardRewardsChart(ctx context.Contex GroupBy("exec_block_hash").As("rb"), goqu.On(goqu.L("rb.exec_block_hash = b.exec_block_hash")), ). - Where(goqu.L("b.epoch >= ?", startEpoch)) + Where(goqu.L("b.epoch >= ?", startEpoch)). + GroupBy(goqu.L("b.proposer")) if dashboardId.Validators == nil { rewardsDs = rewardsDs. @@ -840,7 +856,6 @@ func (d *DataAccessService) GetValidatorDashboardRewardsChart(ctx context.Contex Order(goqu.L("e.epoch").Asc()) elDs = elDs. SelectAppend(goqu.L("?::smallint AS result_group_id", t.DefaultGroupId)). - GroupBy(goqu.L("b.epoch")). Order(goqu.L("b.epoch").Asc()) } else { rewardsDs = rewardsDs. @@ -848,7 +863,6 @@ func (d *DataAccessService) GetValidatorDashboardRewardsChart(ctx context.Contex Order(goqu.L("e.epoch").Asc(), goqu.L("result_group_id").Asc()) elDs = elDs. SelectAppend(goqu.L("v.group_id AS result_group_id")). - GroupBy(goqu.L("b.epoch"), goqu.L("result_group_id")). Order(goqu.L("b.epoch").Asc(), goqu.L("result_group_id").Asc()) } } else { @@ -860,10 +874,24 @@ func (d *DataAccessService) GetValidatorDashboardRewardsChart(ctx context.Contex elDs = elDs. SelectAppend(goqu.L("?::smallint AS result_group_id", t.DefaultGroupId)). Where(goqu.L("b.proposer = ANY(?)", pq.Array(dashboardId.Validators))). - GroupBy(goqu.L("b.epoch")). Order(goqu.L("b.epoch").Asc()) } + // ------------------------------------------------------------------------------------------------------------------ + // Get rocketpool minipool infos if needed + rpValidators := make(map[uint64]t.RpMinipoolInfo) + if protocolModes.RocketPool { + validators, err := d.getDashboardValidators(ctx, dashboardId, nil) + if err != nil { + return nil, fmt.Errorf("error retrieving validators from dashboard id: %v", err) + } + + rpValidators, err = d.getRocketPoolMinipoolInfos(ctx, validators) + if err != nil { + return nil, err + } + } + // ------------------------------------------------------------------------------------------------------------------ // Build the main query and get the data type QueryResultSum struct { @@ -891,19 +919,6 @@ func (d *DataAccessService) GetValidatorDashboardRewardsChart(ctx context.Contex return fmt.Errorf("error retrieving rewards chart data: %v", err) } - rpValidators := make(map[uint64]t.RpMinipoolInfo) - if protocolModes.RocketPool { - validators := make([]uint64, 0, len(queryResult)) - for _, row := range queryResult { - validators = append(validators, row.ValidatorIndex) - } - - rpValidators, err = d.getRocketPoolMinipoolInfos(ctx, validators) - if err != nil { - return err - } - } - for _, entry := range queryResult { if len(queryResultSum) == 0 || queryResultSum[len(queryResultSum)-1].Epoch != entry.Epoch || @@ -930,6 +945,7 @@ func (d *DataAccessService) GetValidatorDashboardRewardsChart(ctx context.Contex elRewards := make(map[uint64]map[uint64]decimal.Decimal) wg.Go(func() error { elQueryResult := []struct { + Proposer uint64 `db:"proposer"` Epoch uint64 `db:"epoch"` GroupId uint64 `db:"result_group_id"` ElRewards decimal.Decimal `db:"el_rewards"` @@ -949,8 +965,14 @@ func (d *DataAccessService) GetValidatorDashboardRewardsChart(ctx context.Contex if _, ok := elRewards[entry.Epoch]; !ok { elRewards[entry.Epoch] = make(map[uint64]decimal.Decimal) } - elRewards[entry.Epoch][entry.GroupId] = entry.ElRewards + + reward := entry.ElRewards + if rpValidator, ok := rpValidators[entry.Proposer]; ok && protocolModes.RocketPool { + reward = reward.Mul(d.getRocketPoolOperatorFactor(rpValidator)) + } + elRewards[entry.Epoch][entry.GroupId] = elRewards[entry.Epoch][entry.GroupId].Add(reward) } + return nil }) @@ -1153,6 +1175,21 @@ func (d *DataAccessService) GetValidatorDashboardDuties(ctx context.Context, das elDs = elDs.Where(goqu.L("b.proposer = ANY(?)", pq.Array(validators))) } + // ------------------------------------------------------------------------------------------------------------------ + // Get rocketpool minipool infos if needed + rpValidators := make(map[uint64]t.RpMinipoolInfo) + if protocolModes.RocketPool { + validators, err := d.getDashboardValidators(ctx, dashboardId, nil) + if err != nil { + return nil, nil, fmt.Errorf("error retrieving validators from dashboard id: %v", err) + } + + rpValidators, err = d.getRocketPoolMinipoolInfos(ctx, validators) + if err != nil { + return nil, nil, err + } + } + // ------------------------------------------------------------------------------------------------------------------ // Get the main data type QueryResultBase struct { @@ -1205,19 +1242,6 @@ func (d *DataAccessService) GetValidatorDashboardDuties(ctx context.Context, das return fmt.Errorf("error retrieving validator rewards data: %v", err) } - rpValidators := make(map[uint64]t.RpMinipoolInfo) - if protocolModes.RocketPool { - validators := make([]uint64, 0, len(queryResult)) - for _, row := range queryResult { - validators = append(validators, row.ValidatorIndex) - } - - rpValidators, err = d.getRocketPoolMinipoolInfos(ctx, validators) - if err != nil { - return err - } - } - for _, entry := range queryResult { queryResultAdjusted = append(queryResultAdjusted, QueryResultAdjusted{ QueryResultBase: QueryResultBase{ @@ -1259,8 +1283,8 @@ func (d *DataAccessService) GetValidatorDashboardDuties(ctx context.Context, das elRewards := make(map[uint64]decimal.Decimal) wg.Go(func() error { elQueryResult := []struct { - ValidatorIndex uint64 `db:"proposer"` - ElRewards decimal.Decimal `db:"el_rewards"` + Proposer uint64 `db:"proposer"` + ElRewards decimal.Decimal `db:"el_rewards"` }{} query, args, err := elDs.Prepared(true).ToSQL() @@ -1274,7 +1298,11 @@ func (d *DataAccessService) GetValidatorDashboardDuties(ctx context.Context, das } for _, entry := range elQueryResult { - elRewards[entry.ValidatorIndex] = entry.ElRewards + reward := entry.ElRewards + if rpValidator, ok := rpValidators[entry.Proposer]; ok && protocolModes.RocketPool { + reward = reward.Mul(d.getRocketPoolOperatorFactor(rpValidator)) + } + elRewards[entry.Proposer] = reward } return nil }) diff --git a/backend/pkg/api/data_access/vdb_summary.go b/backend/pkg/api/data_access/vdb_summary.go index f742c79ac..6c357d3b6 100644 --- a/backend/pkg/api/data_access/vdb_summary.go +++ b/backend/pkg/api/data_access/vdb_summary.go @@ -243,6 +243,7 @@ func (d *DataAccessService) GetValidatorDashboardSummary(ctx context.Context, da elRewards := make(map[int64]decimal.Decimal) ds = goqu.Dialect("postgres"). Select( + goqu.L("b.proposer"), goqu.L("SUM(COALESCE(rb.value, ep.fee_recipient_reward * 1e18, 0)) AS el_rewards")). From(goqu.L("blocks b")). LeftJoin(goqu.L("execution_payloads ep"), goqu.On(goqu.L("ep.block_hash = b.exec_block_hash"))). @@ -256,7 +257,7 @@ func (d *DataAccessService) GetValidatorDashboardSummary(ctx context.Context, da goqu.On(goqu.L("rb.exec_block_hash = b.exec_block_hash")), ). Where(goqu.L("b.epoch >= ? AND b.epoch <= ? AND b.status = '1'", epochStart, epochEnd)). - GroupBy(goqu.L("result_group_id")) + GroupBy(goqu.L("b.proposer")) if len(validators) > 0 { ds = ds. @@ -277,6 +278,7 @@ func (d *DataAccessService) GetValidatorDashboardSummary(ctx context.Context, da } var elRewardsQueryResult []struct { + Proposer uint64 `db:"proposer"` GroupId int64 `db:"result_group_id"` ElRewards decimal.Decimal `db:"el_rewards"` } @@ -292,7 +294,11 @@ func (d *DataAccessService) GetValidatorDashboardSummary(ctx context.Context, da } for _, entry := range elRewardsQueryResult { - elRewards[entry.GroupId] = entry.ElRewards + elReward := entry.ElRewards + if rpValidator, ok := rpValidators[entry.Proposer]; ok && protocolModes.RocketPool { + elReward = elReward.Mul(d.getRocketPoolOperatorFactor(rpValidator)) + } + elRewards[entry.GroupId] = elRewards[entry.GroupId].Add(elReward) } // ------------------------------------------------------------------------------------------------------------------ From a4d6f8904f392163be7657a84b2669a7ae57ba07 Mon Sep 17 00:00:00 2001 From: Stefan Pletka <124689083+Eisei24@users.noreply.github.com> Date: Mon, 28 Oct 2024 16:29:30 +0100 Subject: [PATCH 04/19] Changed getRocketPoolMinipoolInfos handling and RPL APR calculation --- backend/pkg/api/data_access/mobile.go | 10 +++- backend/pkg/api/data_access/vdb_blocks.go | 7 +-- backend/pkg/api/data_access/vdb_helpers.go | 60 ++++++++++--------- backend/pkg/api/data_access/vdb_management.go | 20 +++---- backend/pkg/api/data_access/vdb_rewards.go | 31 ++-------- backend/pkg/api/data_access/vdb_summary.go | 43 ++++--------- 6 files changed, 68 insertions(+), 103 deletions(-) diff --git a/backend/pkg/api/data_access/mobile.go b/backend/pkg/api/data_access/mobile.go index 373688439..9e8380502 100644 --- a/backend/pkg/api/data_access/mobile.go +++ b/backend/pkg/api/data_access/mobile.go @@ -174,6 +174,12 @@ func (d *DataAccessService) GetValidatorDashboardMobileWidget(ctx context.Contex data.NetworkEfficiency = utils.CalculateTotalEfficiency( efficiency.AttestationEfficiency[enums.AllTime], efficiency.ProposalEfficiency[enums.AllTime], efficiency.SyncEfficiency[enums.AllTime]) + protocolModes := t.VDBProtocolModes{RocketPool: true} + rpValidators, err := d.getRocketPoolMinipoolInfos(ctx, wrappedDashboardId, t.AllGroups) + if err != nil { + return nil, fmt.Errorf("error retrieving rocketpool validators: %w", err) + } + // Validator status eg.Go(func() error { validatorMapping, err := d.services.GetCurrentValidatorMapping() @@ -259,7 +265,7 @@ func (d *DataAccessService) GetValidatorDashboardMobileWidget(ctx context.Contex retrieveApr := func(hours int, apr *float64) { eg.Go(func() error { - _, elApr, _, clApr, err := d.internal_getElClAPR(ctx, wrappedDashboardId, -1, hours) + _, elApr, _, clApr, err := d.internal_getElClAPR(ctx, wrappedDashboardId, protocolModes, -1, rpValidators, hours) if err != nil { return err } @@ -270,7 +276,7 @@ func (d *DataAccessService) GetValidatorDashboardMobileWidget(ctx context.Contex retrieveRewards := func(hours int, rewards *decimal.Decimal) { eg.Go(func() error { - clRewards, _, elRewards, _, err := d.internal_getElClAPR(ctx, wrappedDashboardId, -1, hours) + clRewards, _, elRewards, _, err := d.internal_getElClAPR(ctx, wrappedDashboardId, protocolModes, -1, rpValidators, hours) if err != nil { return err } diff --git a/backend/pkg/api/data_access/vdb_blocks.go b/backend/pkg/api/data_access/vdb_blocks.go index e9df9496d..dadacebba 100644 --- a/backend/pkg/api/data_access/vdb_blocks.go +++ b/backend/pkg/api/data_access/vdb_blocks.go @@ -336,12 +336,7 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das // Get the rocketpool minipool infos rpValidators := make(map[uint64]t.RpMinipoolInfo) if protocolModes.RocketPool { - var proposers []uint64 - for _, row := range proposals { - proposers = append(proposers, row.Proposer) - } - - rpValidators, err = d.getRocketPoolMinipoolInfos(ctx, proposers) + rpValidators, err = d.getRocketPoolMinipoolInfos(ctx, dashboardId, t.AllGroups) if err != nil { return nil, nil, err } diff --git a/backend/pkg/api/data_access/vdb_helpers.go b/backend/pkg/api/data_access/vdb_helpers.go index 41e698139..648ea4580 100644 --- a/backend/pkg/api/data_access/vdb_helpers.go +++ b/backend/pkg/api/data_access/vdb_helpers.go @@ -7,7 +7,6 @@ import ( "time" "github.com/doug-martin/goqu/v9" - "github.com/ethereum/go-ethereum/common/hexutil" "github.com/gobitfly/beaconchain/pkg/api/enums" t "github.com/gobitfly/beaconchain/pkg/api/types" "github.com/gobitfly/beaconchain/pkg/commons/cache" @@ -133,45 +132,52 @@ func (d *DataAccessService) getTimeToNextWithdrawal(distance uint64) time.Time { return timeToWithdrawal } -func (d *DataAccessService) getRocketPoolMinipoolInfos(ctx context.Context, validatorIndices []t.VDBValidator) (map[t.VDBValidator]t.RpMinipoolInfo, error) { - validatorMapping, err := d.services.GetCurrentValidatorMapping() - if err != nil { - return nil, err - } - - pubKeyList := make([][]byte, 0, len(validatorIndices)) - pubKeyToIndex := make(map[string]t.VDBValidator, len(validatorIndices)) - for _, validator := range validatorIndices { - publicKey := validatorMapping.ValidatorMetadata[validator].PublicKey - pubKeyList = append(pubKeyList, publicKey) - pubKeyToIndex[hexutil.Encode(publicKey)] = validator - } - +func (d *DataAccessService) getRocketPoolMinipoolInfos(ctx context.Context, dashboardId t.VDBId, groupId int64) (map[t.VDBValidator]t.RpMinipoolInfo, error) { queryResult := []struct { - Pubkey []byte `db:"pubkey"` + ValidatorIndex uint64 `db:"validatorindex"` NodeFee float64 `db:"node_fee"` NodeDepositBalance decimal.Decimal `db:"node_deposit_balance"` UserDepositBalance decimal.Decimal `db:"user_deposit_balance"` }{} - query := ` - SELECT - pubkey, - node_fee, - node_deposit_balance, - user_deposit_balance - FROM rocketpool_minipools - WHERE pubkey = ANY($1) AND node_deposit_balance IS NOT NULL AND user_deposit_balance IS NOT NULL` + ds := goqu.Dialect("postgres"). + Select( + goqu.L("v.validatorindex"), + goqu.L("rplm.node_fee"), + goqu.L("rplm.node_deposit_balance"), + goqu.L("rplm.user_deposit_balance")). + From(goqu.L("rocketpool_minipools AS rplm")). + LeftJoin(goqu.L("validators AS v"), goqu.On(goqu.L("rplm.pubkey = v.pubkey"))). + Where(goqu.L("node_deposit_balance IS NOT NULL")). + Where(goqu.L("user_deposit_balance IS NOT NULL")) + + if len(dashboardId.Validators) == 0 { + ds = ds. + LeftJoin(goqu.L("users_val_dashboards_validators uvdv"), goqu.On(goqu.L("uvdv.validator_index = v.validatorindex"))). + Where(goqu.L("uvdv.dashboard_id = ?", dashboardId.Id)) + + if groupId != t.AllGroups { + ds = ds. + Where(goqu.L("uvdv.group_id = ?", groupId)) + } + } else { + ds = ds. + Where(goqu.L("v.validatorindex = ANY(?)", pq.Array(dashboardId.Validators))) + } + + query, args, err := ds.Prepared(true).ToSQL() + if err != nil { + return nil, fmt.Errorf("error preparing query: %w", err) + } - err = d.alloyReader.SelectContext(ctx, &queryResult, query, pubKeyList) + err = d.alloyReader.SelectContext(ctx, &queryResult, query, args...) if err != nil { return nil, fmt.Errorf("error retrieving rocketpool validators data: %w", err) } rpValidators := make(map[t.VDBValidator]t.RpMinipoolInfo) for _, res := range queryResult { - publicKey := hexutil.Encode(res.Pubkey) - rpValidators[pubKeyToIndex[publicKey]] = t.RpMinipoolInfo{ + rpValidators[res.ValidatorIndex] = t.RpMinipoolInfo{ NodeFee: res.NodeFee, NodeDepositBalance: res.NodeDepositBalance, UserDepositBalance: res.UserDepositBalance, diff --git a/backend/pkg/api/data_access/vdb_management.go b/backend/pkg/api/data_access/vdb_management.go index fff753ad2..bd4cb2525 100644 --- a/backend/pkg/api/data_access/vdb_management.go +++ b/backend/pkg/api/data_access/vdb_management.go @@ -331,16 +331,7 @@ func (d *DataAccessService) GetValidatorDashboardOverview(ctx context.Context, d }) } - validators, err := d.getDashboardValidators(ctx, dashboardId, nil) - if err != nil { - return nil, fmt.Errorf("error retrieving validators from dashboard id: %w", err) - } - - if dashboardId.Validators != nil || dashboardId.AggregateGroups { - data.Groups = append(data.Groups, t.VDBOverviewGroup{Id: t.DefaultGroupId, Name: t.DefaultGroupName, Count: uint64(len(validators))}) - } - - rpValidators, err := d.getRocketPoolMinipoolInfos(ctx, validators) + rpValidators, err := d.getRocketPoolMinipoolInfos(ctx, dashboardId, t.AllGroups) if err != nil { return nil, fmt.Errorf("error retrieving rocketpool validators: %w", err) } @@ -352,6 +343,15 @@ func (d *DataAccessService) GetValidatorDashboardOverview(ctx context.Context, d return err } + validators, err := d.getDashboardValidators(ctx, dashboardId, nil) + if err != nil { + return fmt.Errorf("error retrieving validators from dashboard id: %w", err) + } + + if dashboardId.Validators != nil || dashboardId.AggregateGroups { + data.Groups = append(data.Groups, t.VDBOverviewGroup{Id: t.DefaultGroupId, Name: t.DefaultGroupName, Count: uint64(len(validators))}) + } + // Create a new sub-dashboard to get the total cl deposits for non-rocketpool validators var nonRpDashboardId t.VDBId diff --git a/backend/pkg/api/data_access/vdb_rewards.go b/backend/pkg/api/data_access/vdb_rewards.go index e040d84f7..3c4580fe3 100644 --- a/backend/pkg/api/data_access/vdb_rewards.go +++ b/backend/pkg/api/data_access/vdb_rewards.go @@ -292,12 +292,7 @@ func (d *DataAccessService) GetValidatorDashboardRewards(ctx context.Context, da // Get rocketpool minipool infos if needed rpValidators := make(map[uint64]t.RpMinipoolInfo) if protocolModes.RocketPool { - validators, err := d.getDashboardValidators(ctx, dashboardId, nil) - if err != nil { - return nil, nil, fmt.Errorf("error retrieving validators from dashboard id: %v", err) - } - - rpValidators, err = d.getRocketPoolMinipoolInfos(ctx, validators) + rpValidators, err = d.getRocketPoolMinipoolInfos(ctx, dashboardId, t.AllGroups) if err != nil { return nil, nil, err } @@ -739,12 +734,7 @@ func (d *DataAccessService) GetValidatorDashboardGroupRewards(ctx context.Contex // Create the result rpValidators := make(map[uint64]t.RpMinipoolInfo) if protocolModes.RocketPool { - validators := make([]uint64, 0, len(queryResult)) - for _, row := range queryResult { - validators = append(validators, row.ValidatorIndex) - } - - rpValidators, err = d.getRocketPoolMinipoolInfos(ctx, validators) + rpValidators, err = d.getRocketPoolMinipoolInfos(ctx, dashboardId, groupId) if err != nil { return nil, err } @@ -806,6 +796,7 @@ func (d *DataAccessService) GetValidatorDashboardRewardsChart(ctx context.Contex // series id is group id, series property is 'cl' or 'el' wg := errgroup.Group{} + var err error latestFinalizedEpoch := cache.LatestFinalizedEpoch.Get() const epochLookBack = 224 @@ -884,12 +875,7 @@ func (d *DataAccessService) GetValidatorDashboardRewardsChart(ctx context.Contex // Get rocketpool minipool infos if needed rpValidators := make(map[uint64]t.RpMinipoolInfo) if protocolModes.RocketPool { - validators, err := d.getDashboardValidators(ctx, dashboardId, nil) - if err != nil { - return nil, fmt.Errorf("error retrieving validators from dashboard id: %v", err) - } - - rpValidators, err = d.getRocketPoolMinipoolInfos(ctx, validators) + rpValidators, err = d.getRocketPoolMinipoolInfos(ctx, dashboardId, t.AllGroups) if err != nil { return nil, err } @@ -979,7 +965,7 @@ func (d *DataAccessService) GetValidatorDashboardRewardsChart(ctx context.Contex return nil }) - err := wg.Wait() + err = wg.Wait() if err != nil { return nil, fmt.Errorf("error retrieving validator dashboard rewards chart data: %v", err) } @@ -1183,12 +1169,7 @@ func (d *DataAccessService) GetValidatorDashboardDuties(ctx context.Context, das // Get rocketpool minipool infos if needed rpValidators := make(map[uint64]t.RpMinipoolInfo) if protocolModes.RocketPool { - validators, err := d.getDashboardValidators(ctx, dashboardId, nil) - if err != nil { - return nil, nil, fmt.Errorf("error retrieving validators from dashboard id: %v", err) - } - - rpValidators, err = d.getRocketPoolMinipoolInfos(ctx, validators) + rpValidators, err = d.getRocketPoolMinipoolInfos(ctx, dashboardId, groupId) if err != nil { return nil, nil, err } diff --git a/backend/pkg/api/data_access/vdb_summary.go b/backend/pkg/api/data_access/vdb_summary.go index beebfbaea..92741b0f1 100644 --- a/backend/pkg/api/data_access/vdb_summary.go +++ b/backend/pkg/api/data_access/vdb_summary.go @@ -173,12 +173,7 @@ func (d *DataAccessService) GetValidatorDashboardSummary(ctx context.Context, da rpValidators := make(map[uint64]t.RpMinipoolInfo) if protocolModes.RocketPool { - validators := make([]uint64, 0, len(queryResult)) - for _, row := range queryResult { - validators = append(validators, row.ValidatorIndex) - } - - rpValidators, err = d.getRocketPoolMinipoolInfos(ctx, validators) + rpValidators, err = d.getRocketPoolMinipoolInfos(ctx, dashboardId, t.AllGroups) if err != nil { return nil, nil, err } @@ -739,7 +734,7 @@ func (d *DataAccessService) GetValidatorDashboardGroupSummary(ctx context.Contex validatorArr = validators } - rpValidators, err := d.getRocketPoolMinipoolInfos(ctx, validatorArr) + rpValidators, err := d.getRocketPoolMinipoolInfos(ctx, dashboardId, groupId) if err != nil { return nil, fmt.Errorf("error retrieving rocketpool validators: %w", err) } @@ -870,7 +865,6 @@ func (d *DataAccessService) internal_getElClAPR(ctx context.Context, dashboardId epochStartTotal := uint64(math.MaxInt32) epochEndTotal := uint64(0) - aprRewards := decimal.Zero rewards := decimal.Zero deposits := decimal.Zero @@ -883,17 +877,10 @@ func (d *DataAccessService) internal_getElClAPR(ctx context.Context, dashboardId } reward := utils.GWeiToWei(big.NewInt(row.Reward)) - if rpValidator, ok := rpValidators[row.ValidatorIndex]; ok { - rpReward := reward.Mul(d.getRocketPoolOperatorFactor(rpValidator)) - aprRewards = aprRewards.Add(rpReward) - if protocolModes.RocketPool { - rewards = rewards.Add(rpReward) - } else { - rewards = rewards.Add(reward) - } + if rpValidator, ok := rpValidators[row.ValidatorIndex]; ok && protocolModes.RocketPool { + rewards = rewards.Add(reward.Mul(d.getRocketPoolOperatorFactor(rpValidator))) deposits = deposits.Add(rpValidator.NodeDepositBalance) } else { - aprRewards = aprRewards.Add(reward) rewards = rewards.Add(reward) deposits = deposits.Add(decimal.New(32, 18)) } @@ -904,7 +891,7 @@ func (d *DataAccessService) internal_getElClAPR(ctx context.Context, dashboardId aprDivisor = 90 * 24 } if !deposits.IsZero() { - clAPR = aprRewards.Div(decimal.NewFromInt(int64(aprDivisor))).Div(deposits).Mul(decimal.NewFromInt(24 * 365 * 100)).InexactFloat64() + clAPR = rewards.Div(decimal.NewFromInt(int64(aprDivisor))).Div(deposits).Mul(decimal.NewFromInt(24 * 365 * 100)).InexactFloat64() } if hours == -1 { @@ -932,8 +919,7 @@ func (d *DataAccessService) internal_getElClAPR(ctx context.Context, dashboardId reward := utils.GWeiToWei(big.NewInt(row.Reward)) if rpValidator, ok := rpValidators[row.ValidatorIndex]; ok && protocolModes.RocketPool { - rpReward := reward.Mul(d.getRocketPoolOperatorFactor(rpValidator)) - rewards = rewards.Add(rpReward) + rewards = rewards.Add(reward.Mul(d.getRocketPoolOperatorFactor(rpValidator))) } else { rewards = rewards.Add(reward) } @@ -987,26 +973,18 @@ func (d *DataAccessService) internal_getElClAPR(ctx context.Context, dashboardId return decimal.Zero, 0, decimal.Zero, 0, err } - aprRewards = decimal.Zero rewards = decimal.Zero for _, row := range elRewardsResult { reward := row.Reward - if rpValidator, ok := rpValidators[row.ValidatorIndex]; ok { - rpReward := reward.Mul(d.getRocketPoolOperatorFactor(rpValidator)) - aprRewards = aprRewards.Add(rpReward) - if protocolModes.RocketPool { - rewards = rewards.Add(rpReward) - } else { - rewards = rewards.Add(reward) - } + if rpValidator, ok := rpValidators[row.ValidatorIndex]; ok && protocolModes.RocketPool { + rewards = rewards.Add(reward.Mul(d.getRocketPoolOperatorFactor(rpValidator))) } else { - aprRewards = aprRewards.Add(reward) rewards = rewards.Add(reward) } } if !deposits.IsZero() { - elAPR = aprRewards.Div(decimal.NewFromInt(int64(aprDivisor))).Div(deposits).Mul(decimal.NewFromInt(24 * 365 * 100)).InexactFloat64() + elAPR = rewards.Div(decimal.NewFromInt(int64(aprDivisor))).Div(deposits).Mul(decimal.NewFromInt(24 * 365 * 100)).InexactFloat64() } if hours == -1 { @@ -1027,8 +1005,7 @@ func (d *DataAccessService) internal_getElClAPR(ctx context.Context, dashboardId for _, row := range elRewardsResult { reward := row.Reward if rpValidator, ok := rpValidators[row.ValidatorIndex]; ok && protocolModes.RocketPool { - rpReward := reward.Mul(d.getRocketPoolOperatorFactor(rpValidator)) - rewards = rewards.Add(rpReward) + rewards = rewards.Add(reward.Mul(d.getRocketPoolOperatorFactor(rpValidator))) } else { rewards = rewards.Add(reward) } From 30f9374269897c5f516470be6cc46febd5ce6904 Mon Sep 17 00:00:00 2001 From: Stefan Pletka <124689083+Eisei24@users.noreply.github.com> Date: Mon, 28 Oct 2024 16:57:39 +0100 Subject: [PATCH 05/19] Add a parameter condition for a query --- backend/pkg/api/data_access/vdb_summary.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/backend/pkg/api/data_access/vdb_summary.go b/backend/pkg/api/data_access/vdb_summary.go index 92741b0f1..9d21d5b50 100644 --- a/backend/pkg/api/data_access/vdb_summary.go +++ b/backend/pkg/api/data_access/vdb_summary.go @@ -734,9 +734,12 @@ func (d *DataAccessService) GetValidatorDashboardGroupSummary(ctx context.Contex validatorArr = validators } - rpValidators, err := d.getRocketPoolMinipoolInfos(ctx, dashboardId, groupId) - if err != nil { - return nil, fmt.Errorf("error retrieving rocketpool validators: %w", err) + rpValidators := make(map[uint64]t.RpMinipoolInfo) + if protocolModes.RocketPool { + rpValidators, err = d.getRocketPoolMinipoolInfos(ctx, dashboardId, t.AllGroups) + if err != nil { + return nil, fmt.Errorf("error retrieving rocketpool validators: %w", err) + } } _, ret.Apr.El, _, ret.Apr.Cl, err = d.internal_getElClAPR(ctx, dashboardId, protocolModes, groupId, rpValidators, hours) From a1a04753300875fdc02a2f2c98c34f41b500c12b Mon Sep 17 00:00:00 2001 From: Stefan Pletka <124689083+Eisei24@users.noreply.github.com> Date: Tue, 29 Oct 2024 16:39:58 +0100 Subject: [PATCH 06/19] Changed total withdrawals query to goqu --- .../pkg/api/data_access/vdb_withdrawals.go | 52 ++++++++----------- 1 file changed, 23 insertions(+), 29 deletions(-) diff --git a/backend/pkg/api/data_access/vdb_withdrawals.go b/backend/pkg/api/data_access/vdb_withdrawals.go index 93124cd73..ca29d46ed 100644 --- a/backend/pkg/api/data_access/vdb_withdrawals.go +++ b/backend/pkg/api/data_access/vdb_withdrawals.go @@ -11,6 +11,7 @@ import ( "strconv" "strings" + "github.com/doug-martin/goqu/v9" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/gobitfly/beaconchain/pkg/api/enums" @@ -458,43 +459,36 @@ func (d *DataAccessService) GetValidatorDashboardTotalWithdrawals(ctx context.Co Amount int64 `db:"acc_withdrawals_amount"` }{} - withdrawalsQuery := ` - WITH validators AS ( - SELECT validator_index FROM users_val_dashboards_validators WHERE (dashboard_id = $1) - ) - SELECT - validator_index, - SUM(withdrawals_amount) AS acc_withdrawals_amount, - MAX(epoch_end) AS epoch_end - FROM validator_dashboard_data_rolling_total FINAL - INNER JOIN validators v ON validator_dashboard_data_rolling_total.validator_index = v.validator_index - WHERE validator_index IN (select validator_index FROM validators) - GROUP BY validator_index - ` - - if dashboardId.Validators != nil { - withdrawalsQuery = ` - SELECT - validator_index, - SUM(withdrawals_amount) AS acc_withdrawals_amount, - MAX(epoch_end) AS epoch_end - from validator_dashboard_data_rolling_total FINAL - where validator_index IN ($1) - group by validator_index - ` - } - dashboardValidators := make([]t.VDBValidator, 0) if dashboardId.Validators != nil { dashboardValidators = dashboardId.Validators } - if len(dashboardValidators) > 0 { - err = d.clickhouseReader.SelectContext(ctx, &queryResult, withdrawalsQuery, dashboardValidators) + ds := goqu.Dialect("postgres"). + Select( + goqu.L("validator_index"), + goqu.L("SUM(withdrawals_amount) AS acc_withdrawals_amount"), + goqu.L("MAX(epoch_end) AS epoch_end"), + ). + From(goqu.L("validator_dashboard_data_rolling_total AS t FINAL")). + GroupBy("validator_index") + + if dashboardId.Validators == nil { + ds = ds. + With("validators", goqu.L("(SELECT validator_index FROM users_val_dashboards_validators WHERE (dashboard_id = ?))", dashboardId.Id)). + InnerJoin(goqu.L("validators v"), goqu.On(goqu.L("t.validator_index = v.validator_index"))). + Where(goqu.L("validator_index IN (SELECT validator_index FROM validators)")) } else { - err = d.clickhouseReader.SelectContext(ctx, &queryResult, withdrawalsQuery, dashboardId.Id) + ds = ds. + Where(goqu.L("validator_index IN ?", dashboardValidators)) + } + + query, args, err := ds.Prepared(true).ToSQL() + if err != nil { + return nil, fmt.Errorf("error preparing total withdrawals query: %w", err) } + err = d.clickhouseReader.SelectContext(ctx, &queryResult, query, args...) if err != nil { return nil, fmt.Errorf("error getting total withdrawals for validators: %+v: %w", dashboardId, err) } From 7d39468a8bb3a287567d4c394b8e137638dfde84 Mon Sep 17 00:00:00 2001 From: Stefan Pletka <124689083+Eisei24@users.noreply.github.com> Date: Wed, 30 Oct 2024 16:31:04 +0100 Subject: [PATCH 07/19] Changed withdrawals query to goqu --- .../pkg/api/data_access/vdb_withdrawals.go | 74 ++++++++++--------- 1 file changed, 38 insertions(+), 36 deletions(-) diff --git a/backend/pkg/api/data_access/vdb_withdrawals.go b/backend/pkg/api/data_access/vdb_withdrawals.go index ca29d46ed..d0c08c1c4 100644 --- a/backend/pkg/api/data_access/vdb_withdrawals.go +++ b/backend/pkg/api/data_access/vdb_withdrawals.go @@ -12,6 +12,7 @@ import ( "strings" "github.com/doug-martin/goqu/v9" + "github.com/doug-martin/goqu/v9/exp" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/gobitfly/beaconchain/pkg/api/enums" @@ -40,11 +41,17 @@ func (d *DataAccessService) GetValidatorDashboardWithdrawals(ctx context.Context } // Prepare the sorting + isReverseDirection := (colSort.Desc && !currentCursor.IsReverse()) || (!colSort.Desc && currentCursor.IsReverse()) sortSearchDirection := ">" - sortSearchOrder := " ASC" - if (colSort.Desc && !currentCursor.IsReverse()) || (!colSort.Desc && currentCursor.IsReverse()) { + if isReverseDirection { sortSearchDirection = "<" - sortSearchOrder = " DESC" + } + + orderFunc := func(col string) exp.OrderedExpression { + if isReverseDirection { + return goqu.I(col).Desc() + } + return goqu.I(col).Asc() } // Analyze the search term @@ -120,28 +127,20 @@ func (d *DataAccessService) GetValidatorDashboardWithdrawals(ctx context.Context Amount uint64 `db:"amount"` }{} - queryParams := []interface{}{} - withdrawalsQuery := ` - SELECT - w.block_slot, - b.exec_block_number, - w.withdrawalindex, - w.validatorindex, - w.address, - w.amount - FROM - blocks_withdrawals w - INNER JOIN blocks b ON w.block_slot = b.slot AND w.block_root = b.blockroot AND b.status = '1' - ` - - // Limit the query to relevant validators - queryParams = append(queryParams, pq.Array(validators)) - whereQuery := fmt.Sprintf(` - WHERE - validatorindex = ANY ($%d)`, len(queryParams)) + ds := goqu.Dialect("postgres"). + Select( + goqu.L("w.block_slot"), + goqu.L("b.exec_block_number"), + goqu.L("w.withdrawalindex"), + goqu.L("w.validatorindex"), + goqu.L("w.address"), + goqu.L("w.amount"), + ). + From(goqu.L("blocks_withdrawals AS w")). + InnerJoin(goqu.L("blocks AS b"), goqu.On(goqu.L("w.block_slot = b.slot AND w.block_root = b.blockroot AND b.status = '1'"))). + Where(goqu.L("validatorindex = ANY(?)", validators)) // Limit the query using sorting and the cursor - orderQuery := "" sortColName := "" sortColCursor := interface{}(nil) switch colSort.Column { @@ -161,35 +160,38 @@ func (d *DataAccessService) GetValidatorDashboardWithdrawals(ctx context.Context colSort.Column == enums.VDBWithdrawalsColumns.Slot { if currentCursor.IsValid() { // If we have a valid cursor only check the results before/after it - queryParams = append(queryParams, currentCursor.Slot, currentCursor.WithdrawalIndex) - whereQuery += fmt.Sprintf(" AND (w.block_slot%[1]s$%[2]d OR (w.block_slot=$%[2]d AND w.withdrawalindex%[1]s$%[3]d))", - sortSearchDirection, len(queryParams)-1, len(queryParams)) + ds = ds. + Where(goqu.L(fmt.Sprintf("(w.block_slot%[1]s? OR (w.block_slot=? AND w.withdrawalindex%[1]s?))", + sortSearchDirection), currentCursor.Slot, currentCursor.Slot, currentCursor.WithdrawalIndex)) } - orderQuery = fmt.Sprintf(" ORDER BY w.block_slot %[1]s, w.withdrawalindex %[1]s", sortSearchOrder) + ds = ds. + Order(orderFunc("w.block_slot"), orderFunc("w.withdrawalindex")) } else { if currentCursor.IsValid() { // If we have a valid cursor only check the results before/after it - queryParams = append(queryParams, sortColCursor, currentCursor.Slot, currentCursor.WithdrawalIndex) // The additional WHERE requirement is // WHERE sortColName>cursor OR (sortColName=cursor AND (block_slot>cursor OR (block_slot=cursor AND withdrawalindex>cursor))) // with the > flipped if the sort is descending - whereQuery += fmt.Sprintf(" AND (%[1]s%[2]s$%[3]d OR (%[1]s=$%[3]d AND (w.block_slot%[2]s$%[4]d OR (w.block_slot=$%[4]d AND w.withdrawalindex%[2]s$%[5]d))))", - sortColName, sortSearchDirection, len(queryParams)-2, len(queryParams)-1, len(queryParams)) + ds = ds. + Where(goqu.L(fmt.Sprintf("(%[1]s%[2]s? OR (%[1]s=? AND (w.block_slot%[2]s? OR (w.block_slot=? AND w.withdrawalindex%[2]s?))))", sortColName, sortSearchDirection), sortColCursor, sortColCursor, currentCursor.Slot, currentCursor.Slot, currentCursor.WithdrawalIndex)) } // The ordering is // ORDER BY sortColName ASC, block_slot ASC, withdrawalindex ASC // with the ASC flipped if the sort is descending - orderQuery = fmt.Sprintf(" ORDER BY %[1]s %[2]s, w.block_slot %[2]s, w.withdrawalindex %[2]s", - sortColName, sortSearchOrder) + ds = ds. + Order(orderFunc(sortColName), orderFunc("w.block_slot"), orderFunc("w.withdrawalindex")) } - queryParams = append(queryParams, limit+1) - limitQuery := fmt.Sprintf(" LIMIT $%d", len(queryParams)) + ds = ds. + Limit(uint(limit) + 1) - withdrawalsQuery += whereQuery + orderQuery + limitQuery + query, args, err := ds.Prepared(true).ToSQL() + if err != nil { + return nil, nil, fmt.Errorf("error preparing withdrawals query: %w", err) + } - err = d.readerDb.SelectContext(ctx, &queryResult, withdrawalsQuery, queryParams...) + err = d.alloyReader.SelectContext(ctx, &queryResult, query, args...) if err != nil { return nil, nil, fmt.Errorf("error getting withdrawals for dashboardId: %d (%d validators): %w", dashboardId.Id, len(validators), err) } From 82816e526deb9d30d48938dd21d4c02bc0d30d3d Mon Sep 17 00:00:00 2001 From: Stefan Pletka <124689083+Eisei24@users.noreply.github.com> Date: Mon, 4 Nov 2024 16:06:12 +0100 Subject: [PATCH 08/19] Adjusted RPL withdrawals --- .../pkg/api/data_access/vdb_withdrawals.go | 50 +++++++++++++++---- 1 file changed, 41 insertions(+), 9 deletions(-) diff --git a/backend/pkg/api/data_access/vdb_withdrawals.go b/backend/pkg/api/data_access/vdb_withdrawals.go index d0c08c1c4..795c614ae 100644 --- a/backend/pkg/api/data_access/vdb_withdrawals.go +++ b/backend/pkg/api/data_access/vdb_withdrawals.go @@ -226,17 +226,29 @@ func (d *DataAccessService) GetValidatorDashboardWithdrawals(ctx context.Context return nil, nil, err } + rpValidators := make(map[uint64]t.RpMinipoolInfo) + if protocolModes.RocketPool { + rpValidators, err = d.getRocketPoolMinipoolInfos(ctx, dashboardId, t.AllGroups) + if err != nil { + return nil, nil, err + } + } + // Create the result cursorData := make([]t.WithdrawalsCursor, 0) for i, withdrawal := range queryResult { address := hexutil.Encode(withdrawal.Address) + amount := utils.GWeiToWei(big.NewInt(int64(withdrawal.Amount))) + if rpValidator, ok := rpValidators[withdrawal.ValidatorIndex]; ok && protocolModes.RocketPool { + amount = amount.Mul(d.getRocketPoolOperatorFactor(rpValidator)) + } result = append(result, t.VDBWithdrawalsTableRow{ Epoch: withdrawal.BlockSlot / utils.Config.Chain.ClConfig.SlotsPerEpoch, Slot: withdrawal.BlockSlot, Index: withdrawal.ValidatorIndex, Recipient: *addressMapping[address], GroupId: validatorGroupMap[withdrawal.ValidatorIndex], - Amount: utils.GWeiToWei(big.NewInt(int64(withdrawal.Amount))), + Amount: amount, }) result[i].Recipient.IsContract = contractStatuses[i] == types.CONTRACT_CREATION || contractStatuses[i] == types.CONTRACT_PRESENT cursorData = append(cursorData, t.WithdrawalsCursor{ @@ -275,6 +287,10 @@ func (d *DataAccessService) GetValidatorDashboardWithdrawals(ctx context.Context nextData.GroupId = validatorGroupMap[nextData.Index] // TODO integrate label/ens data for "next" row // nextData.Recipient.Ens = addressEns[string(nextData.Recipient.Hash)] + + if rpValidator, ok := rpValidators[nextData.Index]; ok && protocolModes.RocketPool { + nextData.Amount = nextData.Amount.Mul(d.getRocketPoolOperatorFactor(rpValidator)) + } } else { // If there is no next data, add a missing estimate row nextData = &t.VDBWithdrawalsTableRow{ @@ -500,34 +516,50 @@ func (d *DataAccessService) GetValidatorDashboardTotalWithdrawals(ctx context.Co return result, nil } - var totalAmount int64 + rpValidators := make(map[uint64]t.RpMinipoolInfo) + if protocolModes.RocketPool { + rpValidators, err = d.getRocketPoolMinipoolInfos(ctx, dashboardId, t.AllGroups) + if err != nil { + return nil, err + } + } + var validators []t.VDBValidator lastEpoch := queryResult[0].Epoch lastSlot := (lastEpoch+1)*utils.Config.Chain.ClConfig.SlotsPerEpoch - 1 for _, res := range queryResult { - // Calculate the total amount of withdrawals - totalAmount += res.Amount + amount := utils.GWeiToWei(big.NewInt(res.Amount)) + if rpValidator, ok := rpValidators[res.ValidatorIndex]; ok && protocolModes.RocketPool { + amount = amount.Mul(d.getRocketPoolOperatorFactor(rpValidator)) + } + result.TotalAmount = result.TotalAmount.Add(amount) // Calculate the current validators validators = append(validators, res.ValidatorIndex) } - var latestWithdrawalsAmount int64 - err = d.readerDb.GetContext(ctx, &latestWithdrawalsAmount, ` + err = d.readerDb.SelectContext(ctx, &queryResult, ` SELECT - COALESCE(SUM(w.amount), 0) + w.validatorindex AS validator_index, + SUM(w.amount) AS acc_withdrawals_amount FROM blocks_withdrawals w INNER JOIN blocks b ON w.block_slot = b.slot AND w.block_root = b.blockroot AND b.status = '1' WHERE w.block_slot > $1 AND w.validatorindex = ANY ($2) + GROUP BY w.validatorindex `, lastSlot, validators) if err != nil && !errors.Is(err, sql.ErrNoRows) { return nil, fmt.Errorf("error getting latest withdrawals for validators: %+v: %w", dashboardId, err) } - totalAmount += latestWithdrawalsAmount - result.TotalAmount = utils.GWeiToWei(big.NewInt(totalAmount)) + for _, res := range queryResult { + amount := utils.GWeiToWei(big.NewInt(res.Amount)) + if rpValidator, ok := rpValidators[res.ValidatorIndex]; ok && protocolModes.RocketPool { + amount = amount.Mul(d.getRocketPoolOperatorFactor(rpValidator)) + } + result.TotalAmount = result.TotalAmount.Add(amount) + } return result, nil } From ad862c1b9da5fd8b249c766914339f53c3f862a1 Mon Sep 17 00:00:00 2001 From: Stefan Pletka <124689083+Eisei24@users.noreply.github.com> Date: Thu, 7 Nov 2024 16:57:19 +0100 Subject: [PATCH 09/19] Rewrote RPInfo structure --- backend/pkg/api/data_access/mobile.go | 8 +-- backend/pkg/api/data_access/vdb_blocks.go | 16 +++-- backend/pkg/api/data_access/vdb_helpers.go | 54 +++++++++++---- backend/pkg/api/data_access/vdb_management.go | 6 +- backend/pkg/api/data_access/vdb_rewards.go | 58 ++++++++++------ backend/pkg/api/data_access/vdb_summary.go | 68 +++++++++++-------- .../pkg/api/data_access/vdb_withdrawals.go | 32 +++++---- backend/pkg/api/types/archiver.go | 9 --- backend/pkg/api/types/rocketpool.go | 13 ++++ 9 files changed, 165 insertions(+), 99 deletions(-) diff --git a/backend/pkg/api/data_access/mobile.go b/backend/pkg/api/data_access/mobile.go index 9e8380502..bf6996144 100644 --- a/backend/pkg/api/data_access/mobile.go +++ b/backend/pkg/api/data_access/mobile.go @@ -175,9 +175,9 @@ func (d *DataAccessService) GetValidatorDashboardMobileWidget(ctx context.Contex efficiency.AttestationEfficiency[enums.AllTime], efficiency.ProposalEfficiency[enums.AllTime], efficiency.SyncEfficiency[enums.AllTime]) protocolModes := t.VDBProtocolModes{RocketPool: true} - rpValidators, err := d.getRocketPoolMinipoolInfos(ctx, wrappedDashboardId, t.AllGroups) + rpInfos, err := d.getRocketPoolInfos(ctx, wrappedDashboardId, t.AllGroups) if err != nil { - return nil, fmt.Errorf("error retrieving rocketpool validators: %w", err) + return nil, fmt.Errorf("error retrieving rocketpool infos: %w", err) } // Validator status @@ -265,7 +265,7 @@ func (d *DataAccessService) GetValidatorDashboardMobileWidget(ctx context.Contex retrieveApr := func(hours int, apr *float64) { eg.Go(func() error { - _, elApr, _, clApr, err := d.internal_getElClAPR(ctx, wrappedDashboardId, protocolModes, -1, rpValidators, hours) + _, elApr, _, clApr, err := d.internal_getElClAPR(ctx, wrappedDashboardId, t.AllGroups, protocolModes, rpInfos, hours) if err != nil { return err } @@ -276,7 +276,7 @@ func (d *DataAccessService) GetValidatorDashboardMobileWidget(ctx context.Contex retrieveRewards := func(hours int, rewards *decimal.Decimal) { eg.Go(func() error { - clRewards, _, elRewards, _, err := d.internal_getElClAPR(ctx, wrappedDashboardId, protocolModes, -1, rpValidators, hours) + clRewards, _, elRewards, _, err := d.internal_getElClAPR(ctx, wrappedDashboardId, t.AllGroups, protocolModes, rpInfos, hours) if err != nil { return err } diff --git a/backend/pkg/api/data_access/vdb_blocks.go b/backend/pkg/api/data_access/vdb_blocks.go index dadacebba..1df0697d7 100644 --- a/backend/pkg/api/data_access/vdb_blocks.go +++ b/backend/pkg/api/data_access/vdb_blocks.go @@ -334,9 +334,9 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das } // Get the rocketpool minipool infos - rpValidators := make(map[uint64]t.RpMinipoolInfo) + var rpInfos *t.RPInfo if protocolModes.RocketPool { - rpValidators, err = d.getRocketPoolMinipoolInfos(ctx, dashboardId, t.AllGroups) + rpInfos, err = d.getRocketPoolInfos(ctx, dashboardId, t.AllGroups) if err != nil { return nil, nil, err } @@ -393,14 +393,18 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das TraceIdx: -1, }) reward.El = proposal.ElReward.Decimal.Mul(decimal.NewFromInt(1e18)) - if rpValidator, ok := rpValidators[proposal.Proposer]; ok && protocolModes.RocketPool { - reward.El = reward.El.Mul(d.getRocketPoolOperatorFactor(rpValidator)) + if rpInfos != nil { + if rpValidator, ok := rpInfos.Minipool[proposal.Proposer]; ok && protocolModes.RocketPool { + reward.El = reward.El.Mul(d.getRocketPoolOperatorFactor(rpValidator)) + } } } if proposal.ClReward.Valid { reward.Cl = proposal.ClReward.Decimal.Mul(decimal.NewFromInt(1e18)) - if rpValidator, ok := rpValidators[proposal.Proposer]; ok && protocolModes.RocketPool { - reward.Cl = reward.Cl.Mul(d.getRocketPoolOperatorFactor(rpValidator)) + if rpInfos != nil { + if rpValidator, ok := rpInfos.Minipool[proposal.Proposer]; ok && protocolModes.RocketPool { + reward.Cl = reward.Cl.Mul(d.getRocketPoolOperatorFactor(rpValidator)) + } } } proposals[i].Reward = proposal.ElReward.Decimal.Add(proposal.ClReward.Decimal) diff --git a/backend/pkg/api/data_access/vdb_helpers.go b/backend/pkg/api/data_access/vdb_helpers.go index 648ea4580..274daa2e6 100644 --- a/backend/pkg/api/data_access/vdb_helpers.go +++ b/backend/pkg/api/data_access/vdb_helpers.go @@ -7,6 +7,7 @@ import ( "time" "github.com/doug-martin/goqu/v9" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/gobitfly/beaconchain/pkg/api/enums" t "github.com/gobitfly/beaconchain/pkg/api/types" "github.com/gobitfly/beaconchain/pkg/commons/cache" @@ -132,24 +133,32 @@ func (d *DataAccessService) getTimeToNextWithdrawal(distance uint64) time.Time { return timeToWithdrawal } -func (d *DataAccessService) getRocketPoolMinipoolInfos(ctx context.Context, dashboardId t.VDBId, groupId int64) (map[t.VDBValidator]t.RpMinipoolInfo, error) { +func (d *DataAccessService) getRocketPoolInfos(ctx context.Context, dashboardId t.VDBId, groupId int64) (*t.RPInfo, error) { queryResult := []struct { - ValidatorIndex uint64 `db:"validatorindex"` - NodeFee float64 `db:"node_fee"` - NodeDepositBalance decimal.Decimal `db:"node_deposit_balance"` - UserDepositBalance decimal.Decimal `db:"user_deposit_balance"` + ValidatorIndex uint64 `db:"validatorindex"` + NodeAddress []byte `db:"node_address"` + NodeFee float64 `db:"node_fee"` + NodeDepositBalance decimal.Decimal `db:"node_deposit_balance"` + UserDepositBalance decimal.Decimal `db:"user_deposit_balance"` + EndTime sql.NullTime `db:"end_time"` + SmoothingPoolEth *decimal.Decimal `db:"smoothing_pool_eth"` }{} ds := goqu.Dialect("postgres"). Select( goqu.L("v.validatorindex"), + goqu.L("rplm.node_address"), goqu.L("rplm.node_fee"), goqu.L("rplm.node_deposit_balance"), - goqu.L("rplm.user_deposit_balance")). + goqu.L("rplm.user_deposit_balance"), + goqu.L("rplrs.end_time"), + goqu.L("rplrs.smoothing_pool_eth"), + ). From(goqu.L("rocketpool_minipools AS rplm")). LeftJoin(goqu.L("validators AS v"), goqu.On(goqu.L("rplm.pubkey = v.pubkey"))). - Where(goqu.L("node_deposit_balance IS NOT NULL")). - Where(goqu.L("user_deposit_balance IS NOT NULL")) + LeftJoin(goqu.L("rocketpool_rewards_summary AS rplrs"), goqu.On(goqu.L("rplm.node_address = rplrs.node_address"))). + Where(goqu.L("rplm.node_deposit_balance IS NOT NULL")). + Where(goqu.L("rplm.user_deposit_balance IS NOT NULL")) if len(dashboardId.Validators) == 0 { ds = ds. @@ -175,19 +184,34 @@ func (d *DataAccessService) getRocketPoolMinipoolInfos(ctx context.Context, dash return nil, fmt.Errorf("error retrieving rocketpool validators data: %w", err) } - rpValidators := make(map[t.VDBValidator]t.RpMinipoolInfo) + rpInfo := t.RPInfo{ + Node: make(map[string]t.RPNodeInfo), + Minipool: make(map[uint64]t.RPMinipoolInfo), + } for _, res := range queryResult { - rpValidators[res.ValidatorIndex] = t.RpMinipoolInfo{ - NodeFee: res.NodeFee, - NodeDepositBalance: res.NodeDepositBalance, - UserDepositBalance: res.UserDepositBalance, + if _, ok := rpInfo.Minipool[res.ValidatorIndex]; !ok { + rpInfo.Minipool[res.ValidatorIndex] = t.RPMinipoolInfo{ + NodeFee: res.NodeFee, + NodeDepositBalance: res.NodeDepositBalance, + UserDepositBalance: res.UserDepositBalance, + } + } + + node := hexutil.Encode(res.NodeAddress) + if _, ok := rpInfo.Node[node]; !ok && res.EndTime.Valid && res.SmoothingPoolEth != nil { + epoch := utils.TimeToEpoch(res.EndTime.Time) + + rpInfo.Node[node] = t.RPNodeInfo{ + SmoothingPoolReward: make(map[uint64]decimal.Decimal), + } + rpInfo.Node[node].SmoothingPoolReward[uint64(epoch)] = *res.SmoothingPoolEth } } - return rpValidators, nil + return &rpInfo, nil } -func (d *DataAccessService) getRocketPoolOperatorFactor(minipool t.RpMinipoolInfo) decimal.Decimal { +func (d *DataAccessService) getRocketPoolOperatorFactor(minipool t.RPMinipoolInfo) decimal.Decimal { fullDeposit := minipool.UserDepositBalance.Add(minipool.NodeDepositBalance) operatorShare := minipool.NodeDepositBalance.Div(fullDeposit) invOperatorShare := decimal.NewFromInt(1).Sub(operatorShare) diff --git a/backend/pkg/api/data_access/vdb_management.go b/backend/pkg/api/data_access/vdb_management.go index bd4cb2525..03454f751 100644 --- a/backend/pkg/api/data_access/vdb_management.go +++ b/backend/pkg/api/data_access/vdb_management.go @@ -331,7 +331,7 @@ func (d *DataAccessService) GetValidatorDashboardOverview(ctx context.Context, d }) } - rpValidators, err := d.getRocketPoolMinipoolInfos(ctx, dashboardId, t.AllGroups) + rpInfos, err := d.getRocketPoolInfos(ctx, dashboardId, t.AllGroups) if err != nil { return nil, fmt.Errorf("error retrieving rocketpool validators: %w", err) } @@ -376,7 +376,7 @@ func (d *DataAccessService) GetValidatorDashboardOverview(ctx context.Context, d validatorBalance := utils.GWeiToWei(big.NewInt(int64(metadata.Balance))) effectiveBalance := utils.GWeiToWei(big.NewInt(int64(metadata.EffectiveBalance))) - if rpValidator, ok := rpValidators[validator]; ok { + if rpValidator, ok := rpInfos.Minipool[validator]; ok { if protocolModes.RocketPool { // Calculate the balance of the operator fullDeposit := rpValidator.UserDepositBalance.Add(rpValidator.NodeDepositBalance) @@ -417,7 +417,7 @@ func (d *DataAccessService) GetValidatorDashboardOverview(ctx context.Context, d retrieveRewardsAndEfficiency := func(table string, hours int, rewards *t.ClElValue[decimal.Decimal], apr *t.ClElValue[float64], efficiency *float64) { // Rewards + APR eg.Go(func() error { - (*rewards).El, (*apr).El, (*rewards).Cl, (*apr).Cl, err = d.internal_getElClAPR(ctx, dashboardId, protocolModes, -1, rpValidators, hours) + (*rewards).El, (*apr).El, (*rewards).Cl, (*apr).Cl, err = d.internal_getElClAPR(ctx, dashboardId, t.AllGroups, protocolModes, rpInfos, hours) if err != nil { return err } diff --git a/backend/pkg/api/data_access/vdb_rewards.go b/backend/pkg/api/data_access/vdb_rewards.go index 3c4580fe3..e5549dc03 100644 --- a/backend/pkg/api/data_access/vdb_rewards.go +++ b/backend/pkg/api/data_access/vdb_rewards.go @@ -290,9 +290,9 @@ func (d *DataAccessService) GetValidatorDashboardRewards(ctx context.Context, da // ------------------------------------------------------------------------------------------------------------------ // Get rocketpool minipool infos if needed - rpValidators := make(map[uint64]t.RpMinipoolInfo) + var rpInfos *t.RPInfo if protocolModes.RocketPool { - rpValidators, err = d.getRocketPoolMinipoolInfos(ctx, dashboardId, t.AllGroups) + rpInfos, err = d.getRocketPoolInfos(ctx, dashboardId, t.AllGroups) if err != nil { return nil, nil, err } @@ -367,8 +367,10 @@ func (d *DataAccessService) GetValidatorDashboardRewards(ctx context.Context, da current.BlocksSlashingCount += row.BlocksSlashingCount reward := utils.GWeiToWei(big.NewInt(row.ClRewards)) - if rpValidator, ok := rpValidators[row.ValidatorIndex]; ok && protocolModes.RocketPool { - reward = reward.Mul(d.getRocketPoolOperatorFactor(rpValidator)) + if rpInfos != nil { + if rpValidator, ok := rpInfos.Minipool[row.ValidatorIndex]; ok && protocolModes.RocketPool { + reward = reward.Mul(d.getRocketPoolOperatorFactor(rpValidator)) + } } current.ClRewards = current.ClRewards.Add(reward) } @@ -403,8 +405,10 @@ func (d *DataAccessService) GetValidatorDashboardRewards(ctx context.Context, da } reward := entry.ElRewards - if rpValidator, ok := rpValidators[entry.Proposer]; ok && protocolModes.RocketPool { - reward = reward.Mul(d.getRocketPoolOperatorFactor(rpValidator)) + if rpInfos != nil { + if rpValidator, ok := rpInfos.Minipool[entry.Proposer]; ok && protocolModes.RocketPool { + reward = reward.Mul(d.getRocketPoolOperatorFactor(rpValidator)) + } } elRewards[entry.Epoch][entry.GroupId] = elRewards[entry.Epoch][entry.GroupId].Add(reward) } @@ -732,9 +736,9 @@ func (d *DataAccessService) GetValidatorDashboardGroupRewards(ctx context.Contex // ------------------------------------------------------------------------------------------------------------------ // Create the result - rpValidators := make(map[uint64]t.RpMinipoolInfo) + var rpInfos *t.RPInfo if protocolModes.RocketPool { - rpValidators, err = d.getRocketPoolMinipoolInfos(ctx, dashboardId, groupId) + rpInfos, err = d.getRocketPoolInfos(ctx, dashboardId, groupId) if err != nil { return nil, err } @@ -744,8 +748,10 @@ func (d *DataAccessService) GetValidatorDashboardGroupRewards(ctx context.Contex for _, entry := range queryResult { rpFactor := decimal.NewFromInt(1) - if rpValidator, ok := rpValidators[entry.ValidatorIndex]; ok && protocolModes.RocketPool { - rpFactor = d.getRocketPoolOperatorFactor(rpValidator) + if rpInfos != nil { + if rpValidator, ok := rpInfos.Minipool[entry.ValidatorIndex]; ok && protocolModes.RocketPool { + rpFactor = d.getRocketPoolOperatorFactor(rpValidator) + } } ret.AttestationsHead.Income = ret.AttestationsHead.Income.Add(entry.AttestationHeadReward.Mul(gWei).Mul(rpFactor)) @@ -873,9 +879,9 @@ func (d *DataAccessService) GetValidatorDashboardRewardsChart(ctx context.Contex // ------------------------------------------------------------------------------------------------------------------ // Get rocketpool minipool infos if needed - rpValidators := make(map[uint64]t.RpMinipoolInfo) + var rpInfos *t.RPInfo if protocolModes.RocketPool { - rpValidators, err = d.getRocketPoolMinipoolInfos(ctx, dashboardId, t.AllGroups) + rpInfos, err = d.getRocketPoolInfos(ctx, dashboardId, t.AllGroups) if err != nil { return nil, err } @@ -920,8 +926,10 @@ func (d *DataAccessService) GetValidatorDashboardRewardsChart(ctx context.Contex current := &queryResultSum[len(queryResultSum)-1] reward := utils.GWeiToWei(big.NewInt(entry.ClRewards)) - if rpValidator, ok := rpValidators[entry.ValidatorIndex]; ok && protocolModes.RocketPool { - reward = reward.Mul(d.getRocketPoolOperatorFactor(rpValidator)) + if rpInfos != nil { + if rpValidator, ok := rpInfos.Minipool[entry.ValidatorIndex]; ok && protocolModes.RocketPool { + reward = reward.Mul(d.getRocketPoolOperatorFactor(rpValidator)) + } } current.ClRewards = current.ClRewards.Add(reward) } @@ -956,8 +964,10 @@ func (d *DataAccessService) GetValidatorDashboardRewardsChart(ctx context.Contex } reward := entry.ElRewards - if rpValidator, ok := rpValidators[entry.Proposer]; ok && protocolModes.RocketPool { - reward = reward.Mul(d.getRocketPoolOperatorFactor(rpValidator)) + if rpInfos != nil { + if rpValidator, ok := rpInfos.Minipool[entry.Proposer]; ok && protocolModes.RocketPool { + reward = reward.Mul(d.getRocketPoolOperatorFactor(rpValidator)) + } } elRewards[entry.Epoch][entry.GroupId] = elRewards[entry.Epoch][entry.GroupId].Add(reward) } @@ -1167,9 +1177,9 @@ func (d *DataAccessService) GetValidatorDashboardDuties(ctx context.Context, das // ------------------------------------------------------------------------------------------------------------------ // Get rocketpool minipool infos if needed - rpValidators := make(map[uint64]t.RpMinipoolInfo) + var rpInfos *t.RPInfo if protocolModes.RocketPool { - rpValidators, err = d.getRocketPoolMinipoolInfos(ctx, dashboardId, groupId) + rpInfos, err = d.getRocketPoolInfos(ctx, dashboardId, groupId) if err != nil { return nil, nil, err } @@ -1247,8 +1257,10 @@ func (d *DataAccessService) GetValidatorDashboardDuties(ctx context.Context, das current := &queryResultAdjusted[len(queryResultAdjusted)-1] rpFactor := decimal.NewFromInt(1) - if rpValidator, ok := rpValidators[entry.ValidatorIndex]; ok && protocolModes.RocketPool { - rpFactor = d.getRocketPoolOperatorFactor(rpValidator) + if rpInfos != nil { + if rpValidator, ok := rpInfos.Minipool[entry.ValidatorIndex]; ok && protocolModes.RocketPool { + rpFactor = d.getRocketPoolOperatorFactor(rpValidator) + } } current.AttestationsSourceReward = utils.GWeiToWei(big.NewInt(entry.AttestationsSourceReward)).Mul(rpFactor) @@ -1284,8 +1296,10 @@ func (d *DataAccessService) GetValidatorDashboardDuties(ctx context.Context, das for _, entry := range elQueryResult { reward := entry.ElRewards - if rpValidator, ok := rpValidators[entry.Proposer]; ok && protocolModes.RocketPool { - reward = reward.Mul(d.getRocketPoolOperatorFactor(rpValidator)) + if rpInfos != nil { + if rpValidator, ok := rpInfos.Minipool[entry.Proposer]; ok && protocolModes.RocketPool { + reward = reward.Mul(d.getRocketPoolOperatorFactor(rpValidator)) + } } elRewards[entry.Proposer] = reward } diff --git a/backend/pkg/api/data_access/vdb_summary.go b/backend/pkg/api/data_access/vdb_summary.go index 9d21d5b50..a9ef0346b 100644 --- a/backend/pkg/api/data_access/vdb_summary.go +++ b/backend/pkg/api/data_access/vdb_summary.go @@ -171,9 +171,9 @@ func (d *DataAccessService) GetValidatorDashboardSummary(ctx context.Context, da return result, &paging, nil } - rpValidators := make(map[uint64]t.RpMinipoolInfo) + var rpInfos *t.RPInfo if protocolModes.RocketPool { - rpValidators, err = d.getRocketPoolMinipoolInfos(ctx, dashboardId, t.AllGroups) + rpInfos, err = d.getRocketPoolInfos(ctx, dashboardId, t.AllGroups) if err != nil { return nil, nil, err } @@ -226,8 +226,10 @@ func (d *DataAccessService) GetValidatorDashboardSummary(ctx context.Context, da groupSum.SyncScheduled += row.SyncScheduled clRewardWei := utils.GWeiToWei(big.NewInt(row.ClRewards)) - if rpValidator, ok := rpValidators[row.ValidatorIndex]; ok && protocolModes.RocketPool { - clRewardWei = clRewardWei.Mul(d.getRocketPoolOperatorFactor(rpValidator)) + if rpInfos != nil { + if rpValidator, ok := rpInfos.Minipool[row.ValidatorIndex]; ok && protocolModes.RocketPool { + clRewardWei = clRewardWei.Mul(d.getRocketPoolOperatorFactor(rpValidator)) + } } groupSum.ClRewards = groupSum.ClRewards.Add(clRewardWei) } @@ -291,8 +293,10 @@ func (d *DataAccessService) GetValidatorDashboardSummary(ctx context.Context, da for _, entry := range elRewardsQueryResult { elReward := entry.ElRewards - if rpValidator, ok := rpValidators[entry.Proposer]; ok && protocolModes.RocketPool { - elReward = elReward.Mul(d.getRocketPoolOperatorFactor(rpValidator)) + if rpInfos != nil { + if rpValidator, ok := rpInfos.Minipool[entry.Proposer]; ok && protocolModes.RocketPool { + elReward = elReward.Mul(d.getRocketPoolOperatorFactor(rpValidator)) + } } elRewards[entry.GroupId] = elRewards[entry.GroupId].Add(elReward) } @@ -734,15 +738,15 @@ func (d *DataAccessService) GetValidatorDashboardGroupSummary(ctx context.Contex validatorArr = validators } - rpValidators := make(map[uint64]t.RpMinipoolInfo) + var rpInfos *t.RPInfo if protocolModes.RocketPool { - rpValidators, err = d.getRocketPoolMinipoolInfos(ctx, dashboardId, t.AllGroups) + rpInfos, err = d.getRocketPoolInfos(ctx, dashboardId, groupId) if err != nil { return nil, fmt.Errorf("error retrieving rocketpool validators: %w", err) } } - _, ret.Apr.El, _, ret.Apr.Cl, err = d.internal_getElClAPR(ctx, dashboardId, protocolModes, groupId, rpValidators, hours) + _, ret.Apr.El, _, ret.Apr.Cl, err = d.internal_getElClAPR(ctx, dashboardId, groupId, protocolModes, rpInfos, hours) if err != nil { return nil, err } @@ -797,7 +801,7 @@ func (d *DataAccessService) GetValidatorDashboardGroupSummary(ctx context.Contex return ret, nil } -func (d *DataAccessService) internal_getElClAPR(ctx context.Context, dashboardId t.VDBId, protocolModes t.VDBProtocolModes, groupId int64, rpValidators map[t.VDBValidator]t.RpMinipoolInfo, hours int) (elIncome decimal.Decimal, elAPR float64, clIncome decimal.Decimal, clAPR float64, err error) { +func (d *DataAccessService) internal_getElClAPR(ctx context.Context, dashboardId t.VDBId, groupId int64, protocolModes t.VDBProtocolModes, rpInfos *t.RPInfo, hours int) (elIncome decimal.Decimal, elAPR float64, clIncome decimal.Decimal, clAPR float64, err error) { table := "" switch hours { @@ -880,13 +884,15 @@ func (d *DataAccessService) internal_getElClAPR(ctx context.Context, dashboardId } reward := utils.GWeiToWei(big.NewInt(row.Reward)) - if rpValidator, ok := rpValidators[row.ValidatorIndex]; ok && protocolModes.RocketPool { - rewards = rewards.Add(reward.Mul(d.getRocketPoolOperatorFactor(rpValidator))) - deposits = deposits.Add(rpValidator.NodeDepositBalance) - } else { - rewards = rewards.Add(reward) - deposits = deposits.Add(decimal.New(32, 18)) + if rpInfos != nil { + if rpValidator, ok := rpInfos.Minipool[row.ValidatorIndex]; ok && protocolModes.RocketPool { + rewards = rewards.Add(reward.Mul(d.getRocketPoolOperatorFactor(rpValidator))) + deposits = deposits.Add(rpValidator.NodeDepositBalance) + continue + } } + rewards = rewards.Add(reward) + deposits = deposits.Add(decimal.New(32, 18)) } aprDivisor := hours @@ -921,11 +927,13 @@ func (d *DataAccessService) internal_getElClAPR(ctx context.Context, dashboardId } reward := utils.GWeiToWei(big.NewInt(row.Reward)) - if rpValidator, ok := rpValidators[row.ValidatorIndex]; ok && protocolModes.RocketPool { - rewards = rewards.Add(reward.Mul(d.getRocketPoolOperatorFactor(rpValidator))) - } else { - rewards = rewards.Add(reward) + if rpInfos != nil { + if rpValidator, ok := rpInfos.Minipool[row.ValidatorIndex]; ok && protocolModes.RocketPool { + rewards = rewards.Add(reward.Mul(d.getRocketPoolOperatorFactor(rpValidator))) + continue + } } + rewards = rewards.Add(reward) } } clIncome = rewards @@ -979,11 +987,13 @@ func (d *DataAccessService) internal_getElClAPR(ctx context.Context, dashboardId rewards = decimal.Zero for _, row := range elRewardsResult { reward := row.Reward - if rpValidator, ok := rpValidators[row.ValidatorIndex]; ok && protocolModes.RocketPool { - rewards = rewards.Add(reward.Mul(d.getRocketPoolOperatorFactor(rpValidator))) - } else { - rewards = rewards.Add(reward) + if rpInfos != nil { + if rpValidator, ok := rpInfos.Minipool[row.ValidatorIndex]; ok && protocolModes.RocketPool { + rewards = rewards.Add(reward.Mul(d.getRocketPoolOperatorFactor(rpValidator))) + continue + } } + rewards = rewards.Add(reward) } if !deposits.IsZero() { @@ -1007,11 +1017,13 @@ func (d *DataAccessService) internal_getElClAPR(ctx context.Context, dashboardId rewards = decimal.Zero for _, row := range elRewardsResult { reward := row.Reward - if rpValidator, ok := rpValidators[row.ValidatorIndex]; ok && protocolModes.RocketPool { - rewards = rewards.Add(reward.Mul(d.getRocketPoolOperatorFactor(rpValidator))) - } else { - rewards = rewards.Add(reward) + if rpInfos != nil { + if rpValidator, ok := rpInfos.Minipool[row.ValidatorIndex]; ok && protocolModes.RocketPool { + rewards = rewards.Add(reward.Mul(d.getRocketPoolOperatorFactor(rpValidator))) + continue + } } + rewards = rewards.Add(reward) } } elIncome = rewards diff --git a/backend/pkg/api/data_access/vdb_withdrawals.go b/backend/pkg/api/data_access/vdb_withdrawals.go index 795c614ae..a6314cadf 100644 --- a/backend/pkg/api/data_access/vdb_withdrawals.go +++ b/backend/pkg/api/data_access/vdb_withdrawals.go @@ -226,9 +226,9 @@ func (d *DataAccessService) GetValidatorDashboardWithdrawals(ctx context.Context return nil, nil, err } - rpValidators := make(map[uint64]t.RpMinipoolInfo) + var rpInfos *t.RPInfo if protocolModes.RocketPool { - rpValidators, err = d.getRocketPoolMinipoolInfos(ctx, dashboardId, t.AllGroups) + rpInfos, err = d.getRocketPoolInfos(ctx, dashboardId, t.AllGroups) if err != nil { return nil, nil, err } @@ -239,8 +239,10 @@ func (d *DataAccessService) GetValidatorDashboardWithdrawals(ctx context.Context for i, withdrawal := range queryResult { address := hexutil.Encode(withdrawal.Address) amount := utils.GWeiToWei(big.NewInt(int64(withdrawal.Amount))) - if rpValidator, ok := rpValidators[withdrawal.ValidatorIndex]; ok && protocolModes.RocketPool { - amount = amount.Mul(d.getRocketPoolOperatorFactor(rpValidator)) + if rpInfos != nil { + if rpValidator, ok := rpInfos.Minipool[withdrawal.ValidatorIndex]; ok && protocolModes.RocketPool { + amount = amount.Mul(d.getRocketPoolOperatorFactor(rpValidator)) + } } result = append(result, t.VDBWithdrawalsTableRow{ Epoch: withdrawal.BlockSlot / utils.Config.Chain.ClConfig.SlotsPerEpoch, @@ -288,8 +290,10 @@ func (d *DataAccessService) GetValidatorDashboardWithdrawals(ctx context.Context // TODO integrate label/ens data for "next" row // nextData.Recipient.Ens = addressEns[string(nextData.Recipient.Hash)] - if rpValidator, ok := rpValidators[nextData.Index]; ok && protocolModes.RocketPool { - nextData.Amount = nextData.Amount.Mul(d.getRocketPoolOperatorFactor(rpValidator)) + if rpInfos != nil { + if rpValidator, ok := rpInfos.Minipool[nextData.Index]; ok && protocolModes.RocketPool { + nextData.Amount = nextData.Amount.Mul(d.getRocketPoolOperatorFactor(rpValidator)) + } } } else { // If there is no next data, add a missing estimate row @@ -516,9 +520,9 @@ func (d *DataAccessService) GetValidatorDashboardTotalWithdrawals(ctx context.Co return result, nil } - rpValidators := make(map[uint64]t.RpMinipoolInfo) + var rpInfos *t.RPInfo if protocolModes.RocketPool { - rpValidators, err = d.getRocketPoolMinipoolInfos(ctx, dashboardId, t.AllGroups) + rpInfos, err = d.getRocketPoolInfos(ctx, dashboardId, t.AllGroups) if err != nil { return nil, err } @@ -530,8 +534,10 @@ func (d *DataAccessService) GetValidatorDashboardTotalWithdrawals(ctx context.Co for _, res := range queryResult { amount := utils.GWeiToWei(big.NewInt(res.Amount)) - if rpValidator, ok := rpValidators[res.ValidatorIndex]; ok && protocolModes.RocketPool { - amount = amount.Mul(d.getRocketPoolOperatorFactor(rpValidator)) + if rpInfos != nil { + if rpValidator, ok := rpInfos.Minipool[res.ValidatorIndex]; ok && protocolModes.RocketPool { + amount = amount.Mul(d.getRocketPoolOperatorFactor(rpValidator)) + } } result.TotalAmount = result.TotalAmount.Add(amount) @@ -555,8 +561,10 @@ func (d *DataAccessService) GetValidatorDashboardTotalWithdrawals(ctx context.Co for _, res := range queryResult { amount := utils.GWeiToWei(big.NewInt(res.Amount)) - if rpValidator, ok := rpValidators[res.ValidatorIndex]; ok && protocolModes.RocketPool { - amount = amount.Mul(d.getRocketPoolOperatorFactor(rpValidator)) + if rpInfos != nil { + if rpValidator, ok := rpInfos.Minipool[res.ValidatorIndex]; ok && protocolModes.RocketPool { + amount = amount.Mul(d.getRocketPoolOperatorFactor(rpValidator)) + } } result.TotalAmount = result.TotalAmount.Add(amount) } diff --git a/backend/pkg/api/types/archiver.go b/backend/pkg/api/types/archiver.go index b3930771f..a2ead994e 100644 --- a/backend/pkg/api/types/archiver.go +++ b/backend/pkg/api/types/archiver.go @@ -2,7 +2,6 @@ package types import ( "github.com/gobitfly/beaconchain/pkg/api/enums" - "github.com/shopspring/decimal" ) type ArchiverDashboard struct { @@ -16,11 +15,3 @@ type ArchiverDashboardArchiveReason struct { DashboardId uint64 ArchivedReason enums.VDBArchivedReason } - -// TODO: Find a good place for this -type RpMinipoolInfo struct { - NodeFee float64 - NodeDepositBalance decimal.Decimal - UserDepositBalance decimal.Decimal - SmoothingPoolReward map[uint64]decimal.Decimal -} diff --git a/backend/pkg/api/types/rocketpool.go b/backend/pkg/api/types/rocketpool.go index 1d173bce8..9e4f006eb 100644 --- a/backend/pkg/api/types/rocketpool.go +++ b/backend/pkg/api/types/rocketpool.go @@ -8,3 +8,16 @@ type RPNetworkStats struct { EffectiveRPLStaked decimal.Decimal `db:"effective_rpl_staked"` RPLPrice decimal.Decimal `db:"rpl_price"` } + +type RPInfo struct { + Node map[string]RPNodeInfo + Minipool map[uint64]RPMinipoolInfo +} +type RPNodeInfo struct { + SmoothingPoolReward map[uint64]decimal.Decimal +} +type RPMinipoolInfo struct { + NodeFee float64 + NodeDepositBalance decimal.Decimal + UserDepositBalance decimal.Decimal +} From 1a7e70047a8b4d722f5798dc05eb73a636dfb454 Mon Sep 17 00:00:00 2001 From: Stefan Pletka <124689083+Eisei24@users.noreply.github.com> Date: Tue, 12 Nov 2024 16:06:22 +0100 Subject: [PATCH 10/19] Removed obsolete variable --- backend/pkg/api/data_access/vdb_withdrawals.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/backend/pkg/api/data_access/vdb_withdrawals.go b/backend/pkg/api/data_access/vdb_withdrawals.go index a6314cadf..60bce0394 100644 --- a/backend/pkg/api/data_access/vdb_withdrawals.go +++ b/backend/pkg/api/data_access/vdb_withdrawals.go @@ -481,11 +481,6 @@ func (d *DataAccessService) GetValidatorDashboardTotalWithdrawals(ctx context.Co Amount int64 `db:"acc_withdrawals_amount"` }{} - dashboardValidators := make([]t.VDBValidator, 0) - if dashboardId.Validators != nil { - dashboardValidators = dashboardId.Validators - } - ds := goqu.Dialect("postgres"). Select( goqu.L("validator_index"), @@ -502,7 +497,7 @@ func (d *DataAccessService) GetValidatorDashboardTotalWithdrawals(ctx context.Co Where(goqu.L("validator_index IN (SELECT validator_index FROM validators)")) } else { ds = ds. - Where(goqu.L("validator_index IN ?", dashboardValidators)) + Where(goqu.L("validator_index IN ?", dashboardId.Validators)) } query, args, err := ds.Prepared(true).ToSQL() From 61da10103863317d969a19fd56245154f927aa04 Mon Sep 17 00:00:00 2001 From: Stefan Pletka <124689083+Eisei24@users.noreply.github.com> Date: Thu, 14 Nov 2024 09:13:21 +0100 Subject: [PATCH 11/19] Adjusted APR calculation to smoothing pool --- backend/pkg/api/data_access/vdb_helpers.go | 46 +++++++++++++++++++--- backend/pkg/api/data_access/vdb_summary.go | 43 +++++++++++++++++++- 2 files changed, 83 insertions(+), 6 deletions(-) diff --git a/backend/pkg/api/data_access/vdb_helpers.go b/backend/pkg/api/data_access/vdb_helpers.go index 274daa2e6..e06dd85d1 100644 --- a/backend/pkg/api/data_access/vdb_helpers.go +++ b/backend/pkg/api/data_access/vdb_helpers.go @@ -198,12 +198,14 @@ func (d *DataAccessService) getRocketPoolInfos(ctx context.Context, dashboardId } node := hexutil.Encode(res.NodeAddress) - if _, ok := rpInfo.Node[node]; !ok && res.EndTime.Valid && res.SmoothingPoolEth != nil { - epoch := utils.TimeToEpoch(res.EndTime.Time) - - rpInfo.Node[node] = t.RPNodeInfo{ - SmoothingPoolReward: make(map[uint64]decimal.Decimal), + if res.EndTime.Valid && res.SmoothingPoolEth != nil { + if _, ok := rpInfo.Node[node]; !ok { + rpInfo.Node[node] = t.RPNodeInfo{ + SmoothingPoolReward: make(map[uint64]decimal.Decimal), + } } + + epoch := utils.TimeToEpoch(res.EndTime.Time) rpInfo.Node[node].SmoothingPoolReward[uint64(epoch)] = *res.SmoothingPoolEth } } @@ -211,6 +213,40 @@ func (d *DataAccessService) getRocketPoolInfos(ctx context.Context, dashboardId return &rpInfo, nil } +func (d *DataAccessService) getRocketPoolNodeDeposits(ctx context.Context, nodeAddresses [][]byte) (map[string]decimal.Decimal, error) { + queryResult := []struct { + NodeAddress []byte `db:"node_address"` + NodeDepositBalance decimal.Decimal `db:"acc_node_deposit_balance"` + }{} + + ds := goqu.Dialect("postgres"). + Select( + goqu.L("node_address"), + goqu.L("COALESCE(SUM(node_deposit_balance), 0) AS acc_node_deposit_balance"), + ). + From(goqu.L("rocketpool_minipools")). + Where(goqu.L("node_address = ANY(?)", nodeAddresses)). + GroupBy(goqu.L("node_address")) + + query, args, err := ds.Prepared(true).ToSQL() + if err != nil { + return nil, fmt.Errorf("error preparing query: %w", err) + } + + err = d.alloyReader.SelectContext(ctx, &queryResult, query, args...) + if err != nil { + return nil, fmt.Errorf("error retrieving rocketpool node deposits data: %w", err) + } + + result := make(map[string]decimal.Decimal) + for _, res := range queryResult { + node := hexutil.Encode(res.NodeAddress) + result[node] = res.NodeDepositBalance + } + + return result, nil +} + func (d *DataAccessService) getRocketPoolOperatorFactor(minipool t.RPMinipoolInfo) decimal.Decimal { fullDeposit := minipool.UserDepositBalance.Add(minipool.NodeDepositBalance) operatorShare := minipool.NodeDepositBalance.Div(fullDeposit) diff --git a/backend/pkg/api/data_access/vdb_summary.go b/backend/pkg/api/data_access/vdb_summary.go index a9ef0346b..8a9c3bfdf 100644 --- a/backend/pkg/api/data_access/vdb_summary.go +++ b/backend/pkg/api/data_access/vdb_summary.go @@ -16,6 +16,7 @@ import ( "github.com/doug-martin/goqu/v9" _ "github.com/doug-martin/goqu/v9/dialect/postgres" + "github.com/ethereum/go-ethereum/common/hexutil" "github.com/gobitfly/beaconchain/pkg/api/enums" t "github.com/gobitfly/beaconchain/pkg/api/types" "github.com/gobitfly/beaconchain/pkg/commons/cache" @@ -995,10 +996,41 @@ func (d *DataAccessService) internal_getElClAPR(ctx context.Context, dashboardId } rewards = rewards.Add(reward) } + smoothingPoolRewards := decimal.Zero + smoothingPoolDeposits := decimal.Zero + var smoothingPoolNodes [][]byte + if rpInfos != nil && protocolModes.RocketPool { + for node, nodeInfo := range rpInfos.Node { + for epoch, reward := range nodeInfo.SmoothingPoolReward { + if epoch >= epochStart && epoch <= epochEnd { + smoothingPoolRewards = smoothingPoolRewards.Add(reward) + nodeAddress, err := hexutil.Decode(node) + if err != nil { + return decimal.Zero, 0, decimal.Zero, 0, fmt.Errorf("error decoding node address: %w", err) + } + smoothingPoolNodes = append(smoothingPoolNodes, nodeAddress) + } + } + } + } + nodeDeposits, err := d.getRocketPoolNodeDeposits(ctx, smoothingPoolNodes) + if err != nil { + return decimal.Zero, 0, decimal.Zero, 0, err + } + for _, deposit := range nodeDeposits { + smoothingPoolDeposits = smoothingPoolDeposits.Add(deposit) + } + + rewardsRatio := decimal.Zero if !deposits.IsZero() { - elAPR = rewards.Div(decimal.NewFromInt(int64(aprDivisor))).Div(deposits).Mul(decimal.NewFromInt(24 * 365 * 100)).InexactFloat64() + rewardsRatio = rewards.Div(deposits) } + smoothingPoolRewardsRatio := decimal.Zero + if !smoothingPoolDeposits.IsZero() { + smoothingPoolRewardsRatio = smoothingPoolRewards.Div(smoothingPoolDeposits) + } + elAPR = rewardsRatio.Add(smoothingPoolRewardsRatio).Div(decimal.NewFromInt(int64(aprDivisor))).Mul(decimal.NewFromInt(24 * 365 * 100)).InexactFloat64() if hours == -1 { elTotalDs := elDs. @@ -1025,6 +1057,15 @@ func (d *DataAccessService) internal_getElClAPR(ctx context.Context, dashboardId } rewards = rewards.Add(reward) } + if rpInfos != nil && protocolModes.RocketPool { + for _, nodeInfo := range rpInfos.Node { + for epoch, reward := range nodeInfo.SmoothingPoolReward { + if epoch >= epochStart && epoch <= epochEnd { + rewards = rewards.Add(reward) + } + } + } + } } elIncome = rewards From 7146aa9a957e8833567407b41d5e2378beb5f6e2 Mon Sep 17 00:00:00 2001 From: Stefan Pletka <124689083+Eisei24@users.noreply.github.com> Date: Tue, 19 Nov 2024 12:09:54 +0100 Subject: [PATCH 12/19] Added smoothing pool reward --- backend/pkg/api/data_access/vdb_blocks.go | 8 +- backend/pkg/api/data_access/vdb_helpers.go | 156 +++++++++--------- backend/pkg/api/data_access/vdb_rewards.go | 101 ++++++++++-- backend/pkg/api/data_access/vdb_summary.go | 92 ++++++----- .../pkg/api/data_access/vdb_withdrawals.go | 16 +- backend/pkg/api/types/rocketpool.go | 12 +- 6 files changed, 226 insertions(+), 159 deletions(-) diff --git a/backend/pkg/api/data_access/vdb_blocks.go b/backend/pkg/api/data_access/vdb_blocks.go index 1df0697d7..def4bbd61 100644 --- a/backend/pkg/api/data_access/vdb_blocks.go +++ b/backend/pkg/api/data_access/vdb_blocks.go @@ -393,16 +393,16 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das TraceIdx: -1, }) reward.El = proposal.ElReward.Decimal.Mul(decimal.NewFromInt(1e18)) - if rpInfos != nil { - if rpValidator, ok := rpInfos.Minipool[proposal.Proposer]; ok && protocolModes.RocketPool { + if rpInfos != nil && protocolModes.RocketPool { + if rpValidator, ok := rpInfos.Minipool[proposal.Proposer]; ok /*TODO: && "Not a smoothing pool reward"*/ { reward.El = reward.El.Mul(d.getRocketPoolOperatorFactor(rpValidator)) } } } if proposal.ClReward.Valid { reward.Cl = proposal.ClReward.Decimal.Mul(decimal.NewFromInt(1e18)) - if rpInfos != nil { - if rpValidator, ok := rpInfos.Minipool[proposal.Proposer]; ok && protocolModes.RocketPool { + if rpInfos != nil && protocolModes.RocketPool { + if rpValidator, ok := rpInfos.Minipool[proposal.Proposer]; ok { reward.Cl = reward.Cl.Mul(d.getRocketPoolOperatorFactor(rpValidator)) } } diff --git a/backend/pkg/api/data_access/vdb_helpers.go b/backend/pkg/api/data_access/vdb_helpers.go index e06dd85d1..c942d56c5 100644 --- a/backend/pkg/api/data_access/vdb_helpers.go +++ b/backend/pkg/api/data_access/vdb_helpers.go @@ -15,6 +15,7 @@ import ( "github.com/lib/pq" "github.com/pkg/errors" "github.com/shopspring/decimal" + "golang.org/x/sync/errgroup" ) //////////////////// Helper functions (must be used by more than one VDB endpoint!) @@ -134,6 +135,8 @@ func (d *DataAccessService) getTimeToNextWithdrawal(distance uint64) time.Time { } func (d *DataAccessService) getRocketPoolInfos(ctx context.Context, dashboardId t.VDBId, groupId int64) (*t.RPInfo, error) { + wg := errgroup.Group{} + queryResult := []struct { ValidatorIndex uint64 `db:"validatorindex"` NodeAddress []byte `db:"node_address"` @@ -144,109 +147,106 @@ func (d *DataAccessService) getRocketPoolInfos(ctx context.Context, dashboardId SmoothingPoolEth *decimal.Decimal `db:"smoothing_pool_eth"` }{} - ds := goqu.Dialect("postgres"). - Select( - goqu.L("v.validatorindex"), - goqu.L("rplm.node_address"), - goqu.L("rplm.node_fee"), - goqu.L("rplm.node_deposit_balance"), - goqu.L("rplm.user_deposit_balance"), - goqu.L("rplrs.end_time"), - goqu.L("rplrs.smoothing_pool_eth"), - ). - From(goqu.L("rocketpool_minipools AS rplm")). - LeftJoin(goqu.L("validators AS v"), goqu.On(goqu.L("rplm.pubkey = v.pubkey"))). - LeftJoin(goqu.L("rocketpool_rewards_summary AS rplrs"), goqu.On(goqu.L("rplm.node_address = rplrs.node_address"))). - Where(goqu.L("rplm.node_deposit_balance IS NOT NULL")). - Where(goqu.L("rplm.user_deposit_balance IS NOT NULL")) - - if len(dashboardId.Validators) == 0 { - ds = ds. - LeftJoin(goqu.L("users_val_dashboards_validators uvdv"), goqu.On(goqu.L("uvdv.validator_index = v.validatorindex"))). - Where(goqu.L("uvdv.dashboard_id = ?", dashboardId.Id)) + wg.Go(func() error { + ds := goqu.Dialect("postgres"). + Select( + goqu.L("v.validatorindex"), + goqu.L("rplm.node_address"), + goqu.L("rplm.node_fee"), + goqu.L("rplm.node_deposit_balance"), + goqu.L("rplm.user_deposit_balance"), + goqu.L("rplrs.end_time"), + goqu.L("rplrs.smoothing_pool_eth"), + ). + From(goqu.L("rocketpool_minipools AS rplm")). + LeftJoin(goqu.L("validators AS v"), goqu.On(goqu.L("rplm.pubkey = v.pubkey"))). + LeftJoin(goqu.L("rocketpool_rewards_summary AS rplrs"), goqu.On(goqu.L("rplm.node_address = rplrs.node_address"))). + Where(goqu.L("rplm.node_deposit_balance IS NOT NULL")). + Where(goqu.L("rplm.user_deposit_balance IS NOT NULL")) + + if len(dashboardId.Validators) == 0 { + ds = ds. + LeftJoin(goqu.L("users_val_dashboards_validators uvdv"), goqu.On(goqu.L("uvdv.validator_index = v.validatorindex"))). + Where(goqu.L("uvdv.dashboard_id = ?", dashboardId.Id)) - if groupId != t.AllGroups { + if groupId != t.AllGroups { + ds = ds. + Where(goqu.L("uvdv.group_id = ?", groupId)) + } + } else { ds = ds. - Where(goqu.L("uvdv.group_id = ?", groupId)) + Where(goqu.L("v.validatorindex = ANY(?)", pq.Array(dashboardId.Validators))) } - } else { - ds = ds. - Where(goqu.L("v.validatorindex = ANY(?)", pq.Array(dashboardId.Validators))) - } - query, args, err := ds.Prepared(true).ToSQL() - if err != nil { - return nil, fmt.Errorf("error preparing query: %w", err) - } + query, args, err := ds.Prepared(true).ToSQL() + if err != nil { + return fmt.Errorf("error preparing query: %w", err) + } + + err = d.alloyReader.SelectContext(ctx, &queryResult, query, args...) + if err != nil { + return fmt.Errorf("error retrieving rocketpool validators data: %w", err) + } + + return nil + }) + + nodeMinipoolCount := make(map[string]uint64) + wg.Go(func() error { + queryResult := []struct { + NodeAddress []byte `db:"node_address"` + MinipoolCount uint64 `db:"minipool_count"` + }{} + + err := d.alloyReader.SelectContext(ctx, &queryResult, ` + SELECT + node_address, + COUNT(node_address) AS minipool_count + FROM rocketpool_minipools + GROUP BY node_address`) + if err != nil { + return fmt.Errorf("error retrieving rocketpool node deposits data: %w", err) + } + + for _, res := range queryResult { + node := hexutil.Encode(res.NodeAddress) + nodeMinipoolCount[node] = res.MinipoolCount + } + + return nil + }) - err = d.alloyReader.SelectContext(ctx, &queryResult, query, args...) + err := wg.Wait() if err != nil { - return nil, fmt.Errorf("error retrieving rocketpool validators data: %w", err) + return nil, err } rpInfo := t.RPInfo{ - Node: make(map[string]t.RPNodeInfo), Minipool: make(map[uint64]t.RPMinipoolInfo), } + for _, res := range queryResult { if _, ok := rpInfo.Minipool[res.ValidatorIndex]; !ok { rpInfo.Minipool[res.ValidatorIndex] = t.RPMinipoolInfo{ - NodeFee: res.NodeFee, - NodeDepositBalance: res.NodeDepositBalance, - UserDepositBalance: res.UserDepositBalance, + NodeFee: res.NodeFee, + NodeDepositBalance: res.NodeDepositBalance, + UserDepositBalance: res.UserDepositBalance, + SmoothingPoolRewards: make(map[uint64]decimal.Decimal), } } node := hexutil.Encode(res.NodeAddress) if res.EndTime.Valid && res.SmoothingPoolEth != nil { - if _, ok := rpInfo.Node[node]; !ok { - rpInfo.Node[node] = t.RPNodeInfo{ - SmoothingPoolReward: make(map[uint64]decimal.Decimal), - } - } - - epoch := utils.TimeToEpoch(res.EndTime.Time) - rpInfo.Node[node].SmoothingPoolReward[uint64(epoch)] = *res.SmoothingPoolEth + epoch := uint64(utils.TimeToEpoch(res.EndTime.Time)) + splitReward := res.SmoothingPoolEth.Div(decimal.NewFromUint64(nodeMinipoolCount[node])) + rpInfo.Minipool[res.ValidatorIndex].SmoothingPoolRewards[epoch] = + rpInfo.Minipool[res.ValidatorIndex].SmoothingPoolRewards[epoch].Add(splitReward) } } return &rpInfo, nil } -func (d *DataAccessService) getRocketPoolNodeDeposits(ctx context.Context, nodeAddresses [][]byte) (map[string]decimal.Decimal, error) { - queryResult := []struct { - NodeAddress []byte `db:"node_address"` - NodeDepositBalance decimal.Decimal `db:"acc_node_deposit_balance"` - }{} - - ds := goqu.Dialect("postgres"). - Select( - goqu.L("node_address"), - goqu.L("COALESCE(SUM(node_deposit_balance), 0) AS acc_node_deposit_balance"), - ). - From(goqu.L("rocketpool_minipools")). - Where(goqu.L("node_address = ANY(?)", nodeAddresses)). - GroupBy(goqu.L("node_address")) - - query, args, err := ds.Prepared(true).ToSQL() - if err != nil { - return nil, fmt.Errorf("error preparing query: %w", err) - } - - err = d.alloyReader.SelectContext(ctx, &queryResult, query, args...) - if err != nil { - return nil, fmt.Errorf("error retrieving rocketpool node deposits data: %w", err) - } - - result := make(map[string]decimal.Decimal) - for _, res := range queryResult { - node := hexutil.Encode(res.NodeAddress) - result[node] = res.NodeDepositBalance - } - - return result, nil -} - func (d *DataAccessService) getRocketPoolOperatorFactor(minipool t.RPMinipoolInfo) decimal.Decimal { fullDeposit := minipool.UserDepositBalance.Add(minipool.NodeDepositBalance) operatorShare := minipool.NodeDepositBalance.Div(fullDeposit) diff --git a/backend/pkg/api/data_access/vdb_rewards.go b/backend/pkg/api/data_access/vdb_rewards.go index e5549dc03..fb68e0c8e 100644 --- a/backend/pkg/api/data_access/vdb_rewards.go +++ b/backend/pkg/api/data_access/vdb_rewards.go @@ -315,6 +315,7 @@ func (d *DataAccessService) GetValidatorDashboardRewards(ctx context.Context, da BlocksSlashingCount uint64 } var queryResultSum []QueryResultSum + smoothingPoolRewards := make(map[uint64]map[int64]decimal.Decimal, 0) // epoch -> group -> reward wg.Go(func() error { type QueryResult struct { @@ -343,6 +344,7 @@ func (d *DataAccessService) GetValidatorDashboardRewards(ctx context.Context, da return fmt.Errorf("error retrieving rewards data: %v", err) } + validatorGroupMap := make(map[uint64]int64) for _, row := range queryResult { if len(queryResultSum) == 0 || queryResultSum[len(queryResultSum)-1].Epoch != row.Epoch || @@ -353,6 +355,8 @@ func (d *DataAccessService) GetValidatorDashboardRewards(ctx context.Context, da }) } + validatorGroupMap[row.ValidatorIndex] = row.GroupId + current := &queryResultSum[len(queryResultSum)-1] current.AttestationsScheduled += row.AttestationsScheduled @@ -367,14 +371,27 @@ func (d *DataAccessService) GetValidatorDashboardRewards(ctx context.Context, da current.BlocksSlashingCount += row.BlocksSlashingCount reward := utils.GWeiToWei(big.NewInt(row.ClRewards)) - if rpInfos != nil { - if rpValidator, ok := rpInfos.Minipool[row.ValidatorIndex]; ok && protocolModes.RocketPool { + if rpInfos != nil && protocolModes.RocketPool { + if rpValidator, ok := rpInfos.Minipool[row.ValidatorIndex]; ok { reward = reward.Mul(d.getRocketPoolOperatorFactor(rpValidator)) } } current.ClRewards = current.ClRewards.Add(reward) } + // Calculate smoothing pool rewards + // Has to be done here in the cl and not el part because here we have the list of all relevant validators + if rpInfos != nil && protocolModes.RocketPool { + for validatorIndex, groupId := range validatorGroupMap { + for epoch, reward := range rpInfos.Minipool[validatorIndex].SmoothingPoolRewards { + if _, ok := smoothingPoolRewards[epoch]; !ok { + smoothingPoolRewards[epoch] = make(map[int64]decimal.Decimal) + } + smoothingPoolRewards[epoch][groupId] = smoothingPoolRewards[epoch][groupId].Add(reward) + } + } + } + return nil }) @@ -405,8 +422,8 @@ func (d *DataAccessService) GetValidatorDashboardRewards(ctx context.Context, da } reward := entry.ElRewards - if rpInfos != nil { - if rpValidator, ok := rpInfos.Minipool[entry.Proposer]; ok && protocolModes.RocketPool { + if rpInfos != nil && protocolModes.RocketPool { + if rpValidator, ok := rpInfos.Minipool[entry.Proposer]; ok { reward = reward.Mul(d.getRocketPoolOperatorFactor(rpValidator)) } } @@ -420,6 +437,18 @@ func (d *DataAccessService) GetValidatorDashboardRewards(ctx context.Context, da return nil, nil, fmt.Errorf("error retrieving validator dashboard rewards data: %v", err) } + // Add smoothing pool rewards to el rewards + if rpInfos != nil && protocolModes.RocketPool { + for epoch, groupRewards := range smoothingPoolRewards { + for groupId, reward := range groupRewards { + if _, ok := elRewards[epoch]; !ok { + elRewards[epoch] = make(map[int64]decimal.Decimal) + } + elRewards[epoch][groupId] = elRewards[epoch][groupId].Add(reward) + } + } + } + // ------------------------------------------------------------------------------------------------------------------ // Create the result without the total rewards first resultWoTotal := make([]t.VDBRewardsTableRow, 0) @@ -436,8 +465,8 @@ func (d *DataAccessService) GetValidatorDashboardRewards(ctx context.Context, da SyncExecuted uint64 Slashings uint64 } - totalEpochInfo := make(map[uint64]*TotalEpochInfo, 0) + totalEpochInfo := make(map[uint64]*TotalEpochInfo, 0) for _, res := range queryResultSum { duty := t.VDBRewardsTableDuty{} if res.AttestationsScheduled > 0 { @@ -748,8 +777,8 @@ func (d *DataAccessService) GetValidatorDashboardGroupRewards(ctx context.Contex for _, entry := range queryResult { rpFactor := decimal.NewFromInt(1) - if rpInfos != nil { - if rpValidator, ok := rpInfos.Minipool[entry.ValidatorIndex]; ok && protocolModes.RocketPool { + if rpInfos != nil && protocolModes.RocketPool { + if rpValidator, ok := rpInfos.Minipool[entry.ValidatorIndex]; ok { rpFactor = d.getRocketPoolOperatorFactor(rpValidator) } } @@ -773,8 +802,17 @@ func (d *DataAccessService) GetValidatorDashboardGroupRewards(ctx context.Contex ret.Inactivity.StatusCount.Success++ } - ret.Proposal.Income = ret.Proposal.Income.Add(entry.BlocksClReward.Mul(gWei).Add(elRewards[entry.ValidatorIndex]).Mul(rpFactor)) - ret.ProposalElReward = ret.ProposalElReward.Add(elRewards[entry.ValidatorIndex]) + elReward := elRewards[entry.ValidatorIndex].Mul(rpFactor) + if rpInfos != nil && protocolModes.RocketPool { + if _, ok := rpInfos.Minipool[entry.ValidatorIndex]; ok { + if _, ok := rpInfos.Minipool[entry.ValidatorIndex].SmoothingPoolRewards[epoch]; ok { + elReward = elReward.Add(rpInfos.Minipool[entry.ValidatorIndex].SmoothingPoolRewards[epoch]) + } + } + } + + ret.Proposal.Income = ret.Proposal.Income.Add(entry.BlocksClReward.Mul(gWei).Mul(rpFactor).Add(elReward)) + ret.ProposalElReward = ret.ProposalElReward.Add(elReward) ret.Proposal.StatusCount.Success += uint64(entry.BlocksProposed) ret.Proposal.StatusCount.Failed += uint64(entry.BlocksScheduled) - uint64(entry.BlocksProposed) @@ -895,6 +933,7 @@ func (d *DataAccessService) GetValidatorDashboardRewardsChart(ctx context.Contex ClRewards decimal.Decimal } var queryResultSum []QueryResultSum + smoothingPoolRewards := make(map[uint64]map[uint64]decimal.Decimal, 0) // epoch -> group -> reward wg.Go(func() error { queryResult := []struct { @@ -914,6 +953,7 @@ func (d *DataAccessService) GetValidatorDashboardRewardsChart(ctx context.Contex return fmt.Errorf("error retrieving rewards chart data: %v", err) } + validatorGroupMap := make(map[uint64]uint64) for _, entry := range queryResult { if len(queryResultSum) == 0 || queryResultSum[len(queryResultSum)-1].Epoch != entry.Epoch || @@ -924,16 +964,31 @@ func (d *DataAccessService) GetValidatorDashboardRewardsChart(ctx context.Contex }) } + validatorGroupMap[entry.ValidatorIndex] = entry.GroupId + current := &queryResultSum[len(queryResultSum)-1] reward := utils.GWeiToWei(big.NewInt(entry.ClRewards)) - if rpInfos != nil { - if rpValidator, ok := rpInfos.Minipool[entry.ValidatorIndex]; ok && protocolModes.RocketPool { + if rpInfos != nil && protocolModes.RocketPool { + if rpValidator, ok := rpInfos.Minipool[entry.ValidatorIndex]; ok { reward = reward.Mul(d.getRocketPoolOperatorFactor(rpValidator)) } } current.ClRewards = current.ClRewards.Add(reward) } + // Calculate smoothing pool rewards + // Has to be done here in the cl and not el part because here we have the list of all relevant validators + if rpInfos != nil && protocolModes.RocketPool { + for validatorIndex, groupId := range validatorGroupMap { + for epoch, reward := range rpInfos.Minipool[validatorIndex].SmoothingPoolRewards { + if _, ok := smoothingPoolRewards[epoch]; !ok { + smoothingPoolRewards[epoch] = make(map[uint64]decimal.Decimal) + } + smoothingPoolRewards[epoch][groupId] = smoothingPoolRewards[epoch][groupId].Add(reward) + } + } + } + return nil }) @@ -964,8 +1019,8 @@ func (d *DataAccessService) GetValidatorDashboardRewardsChart(ctx context.Contex } reward := entry.ElRewards - if rpInfos != nil { - if rpValidator, ok := rpInfos.Minipool[entry.Proposer]; ok && protocolModes.RocketPool { + if rpInfos != nil && protocolModes.RocketPool { + if rpValidator, ok := rpInfos.Minipool[entry.Proposer]; ok { reward = reward.Mul(d.getRocketPoolOperatorFactor(rpValidator)) } } @@ -980,6 +1035,18 @@ func (d *DataAccessService) GetValidatorDashboardRewardsChart(ctx context.Contex return nil, fmt.Errorf("error retrieving validator dashboard rewards chart data: %v", err) } + // Add smoothing pool rewards to el rewards + if rpInfos != nil && protocolModes.RocketPool { + for epoch, groupRewards := range smoothingPoolRewards { + for groupId, reward := range groupRewards { + if _, ok := elRewards[epoch]; !ok { + elRewards[epoch] = make(map[uint64]decimal.Decimal) + } + elRewards[epoch][groupId] = elRewards[epoch][groupId].Add(reward) + } + } + } + // ------------------------------------------------------------------------------------------------------------------ // Create a map structure to store the data epochData := make(map[uint64]map[uint64]t.ClElValue[decimal.Decimal]) @@ -1257,8 +1324,8 @@ func (d *DataAccessService) GetValidatorDashboardDuties(ctx context.Context, das current := &queryResultAdjusted[len(queryResultAdjusted)-1] rpFactor := decimal.NewFromInt(1) - if rpInfos != nil { - if rpValidator, ok := rpInfos.Minipool[entry.ValidatorIndex]; ok && protocolModes.RocketPool { + if rpInfos != nil && protocolModes.RocketPool { + if rpValidator, ok := rpInfos.Minipool[entry.ValidatorIndex]; ok { rpFactor = d.getRocketPoolOperatorFactor(rpValidator) } } @@ -1296,8 +1363,8 @@ func (d *DataAccessService) GetValidatorDashboardDuties(ctx context.Context, das for _, entry := range elQueryResult { reward := entry.ElRewards - if rpInfos != nil { - if rpValidator, ok := rpInfos.Minipool[entry.Proposer]; ok && protocolModes.RocketPool { + if rpInfos != nil && protocolModes.RocketPool { + if rpValidator, ok := rpInfos.Minipool[entry.Proposer]; ok /*TODO: && "Not a smoothing pool reward"*/ { reward = reward.Mul(d.getRocketPoolOperatorFactor(rpValidator)) } } diff --git a/backend/pkg/api/data_access/vdb_summary.go b/backend/pkg/api/data_access/vdb_summary.go index 8a9c3bfdf..ea2fb048c 100644 --- a/backend/pkg/api/data_access/vdb_summary.go +++ b/backend/pkg/api/data_access/vdb_summary.go @@ -16,7 +16,6 @@ import ( "github.com/doug-martin/goqu/v9" _ "github.com/doug-martin/goqu/v9/dialect/postgres" - "github.com/ethereum/go-ethereum/common/hexutil" "github.com/gobitfly/beaconchain/pkg/api/enums" t "github.com/gobitfly/beaconchain/pkg/api/types" "github.com/gobitfly/beaconchain/pkg/commons/cache" @@ -195,11 +194,12 @@ func (d *DataAccessService) GetValidatorDashboardSummary(ctx context.Context, da SyncScheduled uint64 } - queryResultSumMap := make(map[int64]QueryResultSum) - epochStart := int64(math.MaxInt32) epochEnd := int64(0) + queryResultSumMap := make(map[int64]QueryResultSum) + validatorGroupMap := make(map[uint64]int64) + for _, row := range queryResult { if row.EpochStart < epochStart { epochStart = row.EpochStart @@ -215,6 +215,8 @@ func (d *DataAccessService) GetValidatorDashboardSummary(ctx context.Context, da } } + validatorGroupMap[row.ValidatorIndex] = row.GroupId + groupSum := queryResultSumMap[row.GroupId] groupSum.ValidatorIndices = append(groupSum.ValidatorIndices, row.ValidatorIndex) groupSum.AttestationReward = groupSum.AttestationReward.Add(row.AttestationReward) @@ -227,8 +229,8 @@ func (d *DataAccessService) GetValidatorDashboardSummary(ctx context.Context, da groupSum.SyncScheduled += row.SyncScheduled clRewardWei := utils.GWeiToWei(big.NewInt(row.ClRewards)) - if rpInfos != nil { - if rpValidator, ok := rpInfos.Minipool[row.ValidatorIndex]; ok && protocolModes.RocketPool { + if rpInfos != nil && protocolModes.RocketPool { + if rpValidator, ok := rpInfos.Minipool[row.ValidatorIndex]; ok { clRewardWei = clRewardWei.Mul(d.getRocketPoolOperatorFactor(rpValidator)) } } @@ -292,16 +294,28 @@ func (d *DataAccessService) GetValidatorDashboardSummary(ctx context.Context, da return nil, nil, fmt.Errorf("error retrieving data from table blocks: %v", err) } + // Add up EL rewards for _, entry := range elRewardsQueryResult { elReward := entry.ElRewards - if rpInfos != nil { - if rpValidator, ok := rpInfos.Minipool[entry.Proposer]; ok && protocolModes.RocketPool { + if rpInfos != nil && protocolModes.RocketPool { + if rpValidator, ok := rpInfos.Minipool[entry.Proposer]; ok { elReward = elReward.Mul(d.getRocketPoolOperatorFactor(rpValidator)) } } elRewards[entry.GroupId] = elRewards[entry.GroupId].Add(elReward) } + // Add up smoothing pool rewards + if rpInfos != nil && protocolModes.RocketPool { + for validatorIndex, groupId := range validatorGroupMap { + for epoch, reward := range rpInfos.Minipool[validatorIndex].SmoothingPoolRewards { + if epoch >= uint64(epochStart) && epoch <= uint64(epochEnd) { + elRewards[groupId] = elRewards[groupId].Add(reward) + } + } + } + } + // ------------------------------------------------------------------------------------------------------------------ // Get the current and next sync committee validators latestEpoch := cache.LatestEpoch.Get() @@ -873,6 +887,8 @@ func (d *DataAccessService) internal_getElClAPR(ctx context.Context, dashboardId epochStartTotal := uint64(math.MaxInt32) epochEndTotal := uint64(0) + validatorGroupMap := make(map[uint64]int64) + rewards := decimal.Zero deposits := decimal.Zero @@ -884,9 +900,11 @@ func (d *DataAccessService) internal_getElClAPR(ctx context.Context, dashboardId epochEnd = row.EpochEnd } + validatorGroupMap[row.ValidatorIndex] = groupId + reward := utils.GWeiToWei(big.NewInt(row.Reward)) - if rpInfos != nil { - if rpValidator, ok := rpInfos.Minipool[row.ValidatorIndex]; ok && protocolModes.RocketPool { + if rpInfos != nil && protocolModes.RocketPool { + if rpValidator, ok := rpInfos.Minipool[row.ValidatorIndex]; ok { rewards = rewards.Add(reward.Mul(d.getRocketPoolOperatorFactor(rpValidator))) deposits = deposits.Add(rpValidator.NodeDepositBalance) continue @@ -928,8 +946,8 @@ func (d *DataAccessService) internal_getElClAPR(ctx context.Context, dashboardId } reward := utils.GWeiToWei(big.NewInt(row.Reward)) - if rpInfos != nil { - if rpValidator, ok := rpInfos.Minipool[row.ValidatorIndex]; ok && protocolModes.RocketPool { + if rpInfos != nil && protocolModes.RocketPool { + if rpValidator, ok := rpInfos.Minipool[row.ValidatorIndex]; ok { rewards = rewards.Add(reward.Mul(d.getRocketPoolOperatorFactor(rpValidator))) continue } @@ -985,52 +1003,33 @@ func (d *DataAccessService) internal_getElClAPR(ctx context.Context, dashboardId return decimal.Zero, 0, decimal.Zero, 0, err } + // Add up EL rewards rewards = decimal.Zero for _, row := range elRewardsResult { reward := row.Reward - if rpInfos != nil { - if rpValidator, ok := rpInfos.Minipool[row.ValidatorIndex]; ok && protocolModes.RocketPool { + if rpInfos != nil && protocolModes.RocketPool { + if rpValidator, ok := rpInfos.Minipool[row.ValidatorIndex]; ok { rewards = rewards.Add(reward.Mul(d.getRocketPoolOperatorFactor(rpValidator))) continue } } rewards = rewards.Add(reward) } - smoothingPoolRewards := decimal.Zero - smoothingPoolDeposits := decimal.Zero - var smoothingPoolNodes [][]byte + + // Add up smoothing pool rewards if rpInfos != nil && protocolModes.RocketPool { - for node, nodeInfo := range rpInfos.Node { - for epoch, reward := range nodeInfo.SmoothingPoolReward { - if epoch >= epochStart && epoch <= epochEnd { - smoothingPoolRewards = smoothingPoolRewards.Add(reward) - nodeAddress, err := hexutil.Decode(node) - if err != nil { - return decimal.Zero, 0, decimal.Zero, 0, fmt.Errorf("error decoding node address: %w", err) - } - smoothingPoolNodes = append(smoothingPoolNodes, nodeAddress) + for validatorIndex := range validatorGroupMap { + for epoch, reward := range rpInfos.Minipool[validatorIndex].SmoothingPoolRewards { + if epoch >= uint64(epochStart) && epoch <= uint64(epochEnd) { + rewards = rewards.Add(reward) } } } } - nodeDeposits, err := d.getRocketPoolNodeDeposits(ctx, smoothingPoolNodes) - if err != nil { - return decimal.Zero, 0, decimal.Zero, 0, err - } - for _, deposit := range nodeDeposits { - smoothingPoolDeposits = smoothingPoolDeposits.Add(deposit) - } - - rewardsRatio := decimal.Zero if !deposits.IsZero() { - rewardsRatio = rewards.Div(deposits) - } - smoothingPoolRewardsRatio := decimal.Zero - if !smoothingPoolDeposits.IsZero() { - smoothingPoolRewardsRatio = smoothingPoolRewards.Div(smoothingPoolDeposits) + elAPR = rewards.Div(deposits).Div(decimal.NewFromInt(int64(aprDivisor))).Mul(decimal.NewFromInt(24 * 365 * 100)).InexactFloat64() } - elAPR = rewardsRatio.Add(smoothingPoolRewardsRatio).Div(decimal.NewFromInt(int64(aprDivisor))).Mul(decimal.NewFromInt(24 * 365 * 100)).InexactFloat64() if hours == -1 { elTotalDs := elDs. @@ -1046,21 +1045,24 @@ func (d *DataAccessService) internal_getElClAPR(ctx context.Context, dashboardId return decimal.Zero, 0, decimal.Zero, 0, err } + // Add up EL rewards rewards = decimal.Zero for _, row := range elRewardsResult { reward := row.Reward - if rpInfos != nil { - if rpValidator, ok := rpInfos.Minipool[row.ValidatorIndex]; ok && protocolModes.RocketPool { + if rpInfos != nil && protocolModes.RocketPool { + if rpValidator, ok := rpInfos.Minipool[row.ValidatorIndex]; ok { rewards = rewards.Add(reward.Mul(d.getRocketPoolOperatorFactor(rpValidator))) continue } } rewards = rewards.Add(reward) } + + // Add up smoothing pool rewards if rpInfos != nil && protocolModes.RocketPool { - for _, nodeInfo := range rpInfos.Node { - for epoch, reward := range nodeInfo.SmoothingPoolReward { - if epoch >= epochStart && epoch <= epochEnd { + for validatorIndex := range validatorGroupMap { + for epoch, reward := range rpInfos.Minipool[validatorIndex].SmoothingPoolRewards { + if epoch >= uint64(epochStart) && epoch <= uint64(epochEnd) { rewards = rewards.Add(reward) } } diff --git a/backend/pkg/api/data_access/vdb_withdrawals.go b/backend/pkg/api/data_access/vdb_withdrawals.go index 60bce0394..88beadb3a 100644 --- a/backend/pkg/api/data_access/vdb_withdrawals.go +++ b/backend/pkg/api/data_access/vdb_withdrawals.go @@ -239,8 +239,8 @@ func (d *DataAccessService) GetValidatorDashboardWithdrawals(ctx context.Context for i, withdrawal := range queryResult { address := hexutil.Encode(withdrawal.Address) amount := utils.GWeiToWei(big.NewInt(int64(withdrawal.Amount))) - if rpInfos != nil { - if rpValidator, ok := rpInfos.Minipool[withdrawal.ValidatorIndex]; ok && protocolModes.RocketPool { + if rpInfos != nil && protocolModes.RocketPool { + if rpValidator, ok := rpInfos.Minipool[withdrawal.ValidatorIndex]; ok { amount = amount.Mul(d.getRocketPoolOperatorFactor(rpValidator)) } } @@ -290,8 +290,8 @@ func (d *DataAccessService) GetValidatorDashboardWithdrawals(ctx context.Context // TODO integrate label/ens data for "next" row // nextData.Recipient.Ens = addressEns[string(nextData.Recipient.Hash)] - if rpInfos != nil { - if rpValidator, ok := rpInfos.Minipool[nextData.Index]; ok && protocolModes.RocketPool { + if rpInfos != nil && protocolModes.RocketPool { + if rpValidator, ok := rpInfos.Minipool[nextData.Index]; ok { nextData.Amount = nextData.Amount.Mul(d.getRocketPoolOperatorFactor(rpValidator)) } } @@ -529,8 +529,8 @@ func (d *DataAccessService) GetValidatorDashboardTotalWithdrawals(ctx context.Co for _, res := range queryResult { amount := utils.GWeiToWei(big.NewInt(res.Amount)) - if rpInfos != nil { - if rpValidator, ok := rpInfos.Minipool[res.ValidatorIndex]; ok && protocolModes.RocketPool { + if rpInfos != nil && protocolModes.RocketPool { + if rpValidator, ok := rpInfos.Minipool[res.ValidatorIndex]; ok { amount = amount.Mul(d.getRocketPoolOperatorFactor(rpValidator)) } } @@ -556,8 +556,8 @@ func (d *DataAccessService) GetValidatorDashboardTotalWithdrawals(ctx context.Co for _, res := range queryResult { amount := utils.GWeiToWei(big.NewInt(res.Amount)) - if rpInfos != nil { - if rpValidator, ok := rpInfos.Minipool[res.ValidatorIndex]; ok && protocolModes.RocketPool { + if rpInfos != nil && protocolModes.RocketPool { + if rpValidator, ok := rpInfos.Minipool[res.ValidatorIndex]; ok { amount = amount.Mul(d.getRocketPoolOperatorFactor(rpValidator)) } } diff --git a/backend/pkg/api/types/rocketpool.go b/backend/pkg/api/types/rocketpool.go index 9e4f006eb..f432e539a 100644 --- a/backend/pkg/api/types/rocketpool.go +++ b/backend/pkg/api/types/rocketpool.go @@ -10,14 +10,12 @@ type RPNetworkStats struct { } type RPInfo struct { - Node map[string]RPNodeInfo Minipool map[uint64]RPMinipoolInfo } -type RPNodeInfo struct { - SmoothingPoolReward map[uint64]decimal.Decimal -} + type RPMinipoolInfo struct { - NodeFee float64 - NodeDepositBalance decimal.Decimal - UserDepositBalance decimal.Decimal + NodeFee float64 + NodeDepositBalance decimal.Decimal + UserDepositBalance decimal.Decimal + SmoothingPoolRewards map[uint64]decimal.Decimal } From 3ec2a7622d8d65158ac6ca30bfe5a3b207de79e8 Mon Sep 17 00:00:00 2001 From: Stefan Pletka <124689083+Eisei24@users.noreply.github.com> Date: Tue, 19 Nov 2024 15:16:32 +0100 Subject: [PATCH 13/19] Excluded smoothing pool rewards from el queries --- backend/pkg/api/data_access/vdb_rewards.go | 34 ++++++++++++++++++++-- backend/pkg/api/data_access/vdb_summary.go | 14 +++++++++ 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/backend/pkg/api/data_access/vdb_rewards.go b/backend/pkg/api/data_access/vdb_rewards.go index fb68e0c8e..dd71a487e 100644 --- a/backend/pkg/api/data_access/vdb_rewards.go +++ b/backend/pkg/api/data_access/vdb_rewards.go @@ -117,6 +117,13 @@ func (d *DataAccessService) GetValidatorDashboardRewards(ctx context.Context, da ). GroupBy(goqu.L("b.proposer")) + if protocolModes.RocketPool { + // Exclude rewards that went to the smoothing pool + // TODO: Add smoothing pool address to the parameters + elDs = elDs. + Where(goqu.L("b.exec_fee_recipient != ? OR (rb.proposer_fee_recipient IS NOT NULL AND rb.proposer_fee_recipient != ?)", 1, 2)) + } + if dashboardId.Validators == nil { rewardsDs = rewardsDs. InnerJoin(goqu.L("validators v"), goqu.On(goqu.L("e.validator_index = v.validator_index"))). @@ -669,9 +676,15 @@ func (d *DataAccessService) GetValidatorDashboardGroupRewards(ctx context.Contex Where(goqu.L("b.epoch = ?", epoch)). GroupBy(goqu.L("b.proposer")) - // handle the case when we have a list of validators + if protocolModes.RocketPool { + // Exclude rewards that went to the smoothing pool + // TODO: Add smoothing pool address to the parameters + elDs = elDs. + Where(goqu.L("b.exec_fee_recipient != ? OR (rb.proposer_fee_recipient IS NOT NULL AND rb.proposer_fee_recipient != ?)", 1, 2)) + } if dashboardId.Validators == nil { + // handle the case when we have a dashboard id and an optional group id rewardsDs = rewardsDs. InnerJoin(goqu.L("validators v"), goqu.On(goqu.L("e.validator_index = v.validator_index"))). Where(goqu.L("e.validator_index IN (SELECT validator_index FROM validators)")) @@ -681,7 +694,8 @@ func (d *DataAccessService) GetValidatorDashboardGroupRewards(ctx context.Contex rewardsDs = rewardsDs.Where(goqu.L("v.group_id = ?", groupId)) elDs = elDs.Where(goqu.L("v.group_id = ?", groupId)) } - } else { // handle the case when we have a dashboard id and an optional group id + } else { + // handle the case when we have a list of validators rewardsDs = rewardsDs. Where(goqu.L("e.validator_index IN ?", dashboardId.Validators)) elDs = elDs. @@ -881,6 +895,13 @@ func (d *DataAccessService) GetValidatorDashboardRewardsChart(ctx context.Contex Where(goqu.L("b.epoch >= ?", startEpoch)). GroupBy(goqu.L("b.proposer")) + if protocolModes.RocketPool { + // Exclude rewards that went to the smoothing pool + // TODO: Add smoothing pool address to the parameters + elDs = elDs. + Where(goqu.L("b.exec_fee_recipient != ? OR (rb.proposer_fee_recipient IS NOT NULL AND rb.proposer_fee_recipient != ?)", 1, 2)) + } + if dashboardId.Validators == nil { rewardsDs = rewardsDs. InnerJoin(goqu.L("validators v"), goqu.On(goqu.L("e.validator_index = v.validator_index"))). @@ -1206,6 +1227,13 @@ func (d *DataAccessService) GetValidatorDashboardDuties(ctx context.Context, das Where(goqu.L("b.status = '1'")). GroupBy(goqu.L("b.proposer")) + if protocolModes.RocketPool { + // Exclude rewards that went to the smoothing pool + // TODO: Add smoothing pool address to the parameters + elDs = elDs. + Where(goqu.L("b.exec_fee_recipient != ? OR (rb.proposer_fee_recipient IS NOT NULL AND rb.proposer_fee_recipient != ?)", 1, 2)) + } + // ------------------------------------------------------------------------------------------------------------------ // Add further conditions if dashboardId.Validators == nil { @@ -1364,7 +1392,7 @@ func (d *DataAccessService) GetValidatorDashboardDuties(ctx context.Context, das for _, entry := range elQueryResult { reward := entry.ElRewards if rpInfos != nil && protocolModes.RocketPool { - if rpValidator, ok := rpInfos.Minipool[entry.Proposer]; ok /*TODO: && "Not a smoothing pool reward"*/ { + if rpValidator, ok := rpInfos.Minipool[entry.Proposer]; ok { reward = reward.Mul(d.getRocketPoolOperatorFactor(rpValidator)) } } diff --git a/backend/pkg/api/data_access/vdb_summary.go b/backend/pkg/api/data_access/vdb_summary.go index ea2fb048c..31afce965 100644 --- a/backend/pkg/api/data_access/vdb_summary.go +++ b/backend/pkg/api/data_access/vdb_summary.go @@ -260,6 +260,13 @@ func (d *DataAccessService) GetValidatorDashboardSummary(ctx context.Context, da Where(goqu.L("b.epoch >= ? AND b.epoch <= ? AND b.status = '1'", epochStart, epochEnd)). GroupBy(goqu.L("b.proposer")) + if protocolModes.RocketPool { + // Exclude rewards that went to the smoothing pool + // TODO: Add smoothing pool address to the parameters + ds = ds. + Where(goqu.L("b.exec_fee_recipient != ? OR (rb.proposer_fee_recipient IS NOT NULL AND rb.proposer_fee_recipient != ?)", 1, 2)) + } + if len(validators) > 0 { ds = ds. SelectAppend(goqu.L("?::smallint AS result_group_id", t.DefaultGroupId)). @@ -976,6 +983,13 @@ func (d *DataAccessService) internal_getElClAPR(ctx context.Context, dashboardId Where(goqu.L("b.status = '1'")). GroupBy("b.proposer") + if protocolModes.RocketPool { + // Exclude rewards that went to the smoothing pool + // TODO: Add smoothing pool address to the parameters + elDs = elDs. + Where(goqu.L("b.exec_fee_recipient != ? OR (rb.proposer_fee_recipient IS NOT NULL AND rb.proposer_fee_recipient != ?)", 1, 2)) + } + if len(dashboardId.Validators) > 0 { elDs = elDs. Where(goqu.L("b.proposer = ANY(?)", pq.Array(dashboardId.Validators))) From a279cffbf1f8d04b829a364f1fd4fc8df10e6427 Mon Sep 17 00:00:00 2001 From: Stefan Pletka <124689083+Eisei24@users.noreply.github.com> Date: Tue, 19 Nov 2024 16:50:36 +0100 Subject: [PATCH 14/19] Added smoothing pool logic for blocks view --- backend/pkg/api/data_access/vdb_blocks.go | 36 +++++++++++++++-------- 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/backend/pkg/api/data_access/vdb_blocks.go b/backend/pkg/api/data_access/vdb_blocks.go index adfd23153..b7c2194e3 100644 --- a/backend/pkg/api/data_access/vdb_blocks.go +++ b/backend/pkg/api/data_access/vdb_blocks.go @@ -175,6 +175,12 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das goqu.COALESCE(goqu.L("rb.value / 1e18"), goqu.I("ep.fee_recipient_reward")).As("el_reward"), ) + if protocolModes.RocketPool { + // TODO: Add smoothing pool address to the parameters + blocksDs = blocksDs. + SelectAppend(goqu.L("blocks.exec_fee_recipient = ? AND (rb.proposer_fee_recipient IS NULL OR rb.proposer_fee_recipient = ?) AS is_smoothing_pool", 1, 2)) + } + // 3. Sorting and pagination defaultColumns := []t.SortColumn{ {Column: enums.VDBBlocksColumns.Slot.ToExpr(), Desc: true, Offset: currentCursor.Slot}, @@ -272,6 +278,11 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das goqu.L("NULL::NUMERIC").As("el_reward"), ) + if protocolModes.RocketPool { + blocksDs = blocksDs. + SelectAppend(goqu.L("false").As("is_smoothing_pool")) + } + // We don't have access to exec_block_number and status for a WHERE without wrapping the query so if we sort by those get all the data if colSort.Column == enums.VDBBlocksColumns.Proposer || colSort.Column == enums.VDBBlocksColumns.Slot { scheduledDs = scheduledDs. @@ -307,16 +318,17 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das // ------------------------------------- // Execute query var proposals []struct { - Proposer t.VDBValidator `db:"validator_index"` - Group uint64 `db:"group_id"` - Epoch uint64 `db:"epoch"` - Slot uint64 `db:"slot"` - Status uint64 `db:"status"` - Block sql.NullInt64 `db:"exec_block_number"` - FeeRecipient []byte `db:"fee_recipient"` - ElReward decimal.NullDecimal `db:"el_reward"` - ClReward decimal.NullDecimal `db:"cl_reward"` - GraffitiText sql.NullString `db:"graffiti_text"` + Proposer t.VDBValidator `db:"validator_index"` + Group uint64 `db:"group_id"` + Epoch uint64 `db:"epoch"` + Slot uint64 `db:"slot"` + Status uint64 `db:"status"` + Block sql.NullInt64 `db:"exec_block_number"` + FeeRecipient []byte `db:"fee_recipient"` + ElReward decimal.NullDecimal `db:"el_reward"` + ClReward decimal.NullDecimal `db:"cl_reward"` + GraffitiText sql.NullString `db:"graffiti_text"` + IsSmoothingPool bool `db:"is_smoothing_pool"` // for cursor only Reward decimal.Decimal @@ -453,8 +465,8 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das TraceIdx: -1, }) reward.El = proposal.ElReward.Decimal.Mul(decimal.NewFromInt(1e18)) - if rpInfos != nil && protocolModes.RocketPool { - if rpValidator, ok := rpInfos.Minipool[proposal.Proposer]; ok /*TODO: && "Not a smoothing pool reward"*/ { + if rpInfos != nil && protocolModes.RocketPool && !proposal.IsSmoothingPool { + if rpValidator, ok := rpInfos.Minipool[proposal.Proposer]; ok { reward.El = reward.El.Mul(d.getRocketPoolOperatorFactor(rpValidator)) } } From 73917bf3b36eed62ff03740e2edc870bfed842ab Mon Sep 17 00:00:00 2001 From: Stefan Pletka <124689083+Eisei24@users.noreply.github.com> Date: Thu, 21 Nov 2024 15:14:46 +0100 Subject: [PATCH 15/19] Fixed endpoint call errors for dashboard ids --- backend/pkg/api/data_access/vdb_rewards.go | 10 +++++--- backend/pkg/api/data_access/vdb_summary.go | 25 ++++++++++--------- .../pkg/api/data_access/vdb_withdrawals.go | 2 +- 3 files changed, 20 insertions(+), 17 deletions(-) diff --git a/backend/pkg/api/data_access/vdb_rewards.go b/backend/pkg/api/data_access/vdb_rewards.go index 900bf47eb..d3cee4741 100644 --- a/backend/pkg/api/data_access/vdb_rewards.go +++ b/backend/pkg/api/data_access/vdb_rewards.go @@ -115,7 +115,7 @@ func (d *DataAccessService) GetValidatorDashboardRewards(ctx context.Context, da GroupBy("exec_block_hash")).As("rb"), goqu.On(goqu.L("rb.exec_block_hash = b.exec_block_hash")), ). - GroupBy(goqu.L("b.proposer")) + GroupBy(goqu.L("b.epoch"), goqu.L("b.proposer")) if protocolModes.RocketPool { // Exclude rewards that went to the smoothing pool @@ -245,7 +245,8 @@ func (d *DataAccessService) GetValidatorDashboardRewards(ctx context.Context, da rewardsDs = rewardsDs. SelectAppend(goqu.L("v.group_id AS result_group_id")) elDs = elDs. - SelectAppend(goqu.L("v.group_id AS result_group_id")) + SelectAppend(goqu.L("v.group_id AS result_group_id")). + GroupByAppend(goqu.L("result_group_id")) if isReverseDirection { rewardsDs = rewardsDs.Order(goqu.L("e.epoch").Desc(), goqu.L("result_group_id").Desc()) @@ -881,7 +882,7 @@ func (d *DataAccessService) GetValidatorDashboardRewardsChart(ctx context.Contex Select( goqu.L("e.validator_index"), goqu.L("e.epoch"), - goqu.L(`SUM(COALESCE(e.attestations_reward, 0) + COALESCE(e.blocks_cl_reward, 0) + COALESCE(e.sync_rewards, 0)) AS cl_rewards`)). + goqu.L(`(e.attestations_reward + e.blocks_cl_reward + e.sync_rewards) AS cl_rewards`)). From(goqu.L("validator_dashboard_data_epoch e")). With("validators", goqu.L("(SELECT validator_index as validator_index, group_id FROM users_val_dashboards_validators WHERE dashboard_id = ?)", dashboardId.Id)). Where(goqu.L("e.epoch_timestamp >= fromUnixTimestamp(?)", utils.EpochToTime(startEpoch).Unix())) @@ -905,7 +906,7 @@ func (d *DataAccessService) GetValidatorDashboardRewardsChart(ctx context.Contex goqu.On(goqu.L("rb.exec_block_hash = b.exec_block_hash")), ). Where(goqu.L("b.epoch >= ?", startEpoch)). - GroupBy(goqu.L("b.proposer")) + GroupBy(goqu.L("b.epoch"), goqu.L("b.proposer")) if protocolModes.RocketPool { // Exclude rewards that went to the smoothing pool @@ -934,6 +935,7 @@ func (d *DataAccessService) GetValidatorDashboardRewardsChart(ctx context.Contex Order(goqu.L("e.epoch").Asc(), goqu.L("result_group_id").Asc()) elDs = elDs. SelectAppend(goqu.L("v.group_id AS result_group_id")). + GroupByAppend(goqu.L("result_group_id")). Order(goqu.L("b.epoch").Asc(), goqu.L("result_group_id").Asc()) } } else { diff --git a/backend/pkg/api/data_access/vdb_summary.go b/backend/pkg/api/data_access/vdb_summary.go index faf01aa5f..e90f6ee47 100644 --- a/backend/pkg/api/data_access/vdb_summary.go +++ b/backend/pkg/api/data_access/vdb_summary.go @@ -118,18 +118,18 @@ func (d *DataAccessService) GetValidatorDashboardSummary(ctx context.Context, da From(goqu.L(fmt.Sprintf(`%s AS r FINAL`, clickhouseTable))). With("validators", goqu.L("(SELECT dashboard_id, group_id, validator_index FROM users_val_dashboards_validators WHERE dashboard_id = ?)", dashboardId.Id)). Select( - goqu.L("r.validator_index"), + goqu.L("r.validator_index AS validator_index"), goqu.L("(r.balance_end + r.withdrawals_amount - r.deposits_amount - r.balance_start) AS cl_rewards"), goqu.L("r.attestations_reward::decimal AS attestations_reward"), goqu.L("r.attestations_ideal_reward::decimal AS attestations_ideal_reward"), - goqu.L("r.attestations_executed"), - goqu.L("r.attestations_scheduled"), - goqu.L("r.blocks_proposed"), - goqu.L("r.blocks_scheduled"), - goqu.L("r.sync_executed"), - goqu.L("r.sync_scheduled"), - goqu.L("r.epoch_start"), - goqu.L("r.epoch_end")) + goqu.L("r.attestations_executed AS attestations_executed"), + goqu.L("r.attestations_scheduled AS attestations_scheduled"), + goqu.L("r.blocks_proposed AS blocks_proposed"), + goqu.L("r.blocks_scheduled AS blocks_scheduled"), + goqu.L("r.sync_executed AS sync_executed"), + goqu.L("r.sync_scheduled AS sync_scheduled"), + goqu.L("r.epoch_start AS epoch_start"), + goqu.L("r.epoch_end AS epoch_end")) if len(validators) > 0 { ds = ds. @@ -277,7 +277,8 @@ func (d *DataAccessService) GetValidatorDashboardSummary(ctx context.Context, da SelectAppend(goqu.L("?::smallint AS result_group_id", t.DefaultGroupId)) } else { ds = ds. - SelectAppend(goqu.L("v.group_id AS result_group_id")) + SelectAppend(goqu.L("v.group_id AS result_group_id")). + GroupByAppend(goqu.L("result_group_id")) } ds = ds. @@ -1123,7 +1124,7 @@ func (d *DataAccessService) internal_getElClAPR(ctx context.Context, dashboardId if rpInfos != nil && protocolModes.RocketPool { for validatorIndex := range validatorGroupMap { for epoch, reward := range rpInfos.Minipool[validatorIndex].SmoothingPoolRewards { - if epoch >= uint64(epochStart) && epoch <= uint64(epochEnd) { + if epoch >= epochStart && epoch <= epochEnd { rewards = rewards.Add(reward) } } @@ -1165,7 +1166,7 @@ func (d *DataAccessService) internal_getElClAPR(ctx context.Context, dashboardId if rpInfos != nil && protocolModes.RocketPool { for validatorIndex := range validatorGroupMap { for epoch, reward := range rpInfos.Minipool[validatorIndex].SmoothingPoolRewards { - if epoch >= uint64(epochStart) && epoch <= uint64(epochEnd) { + if epoch >= epochStart && epoch <= epochEnd { rewards = rewards.Add(reward) } } diff --git a/backend/pkg/api/data_access/vdb_withdrawals.go b/backend/pkg/api/data_access/vdb_withdrawals.go index 88beadb3a..4974e8571 100644 --- a/backend/pkg/api/data_access/vdb_withdrawals.go +++ b/backend/pkg/api/data_access/vdb_withdrawals.go @@ -138,7 +138,7 @@ func (d *DataAccessService) GetValidatorDashboardWithdrawals(ctx context.Context ). From(goqu.L("blocks_withdrawals AS w")). InnerJoin(goqu.L("blocks AS b"), goqu.On(goqu.L("w.block_slot = b.slot AND w.block_root = b.blockroot AND b.status = '1'"))). - Where(goqu.L("validatorindex = ANY(?)", validators)) + Where(goqu.L("validatorindex = ANY(?)", pq.Array(validators))) // Limit the query using sorting and the cursor sortColName := "" From 6c787978bde4f50ad931ff168e551857df2927f8 Mon Sep 17 00:00:00 2001 From: Stefan Pletka <124689083+Eisei24@users.noreply.github.com> Date: Thu, 21 Nov 2024 17:30:37 +0100 Subject: [PATCH 16/19] Added the smoothing pool address for the queries --- backend/pkg/api/data_access/vdb_blocks.go | 25 +++-- backend/pkg/api/data_access/vdb_helpers.go | 19 ++-- backend/pkg/api/data_access/vdb_rewards.go | 101 ++++++++++----------- backend/pkg/api/data_access/vdb_summary.go | 10 +- backend/pkg/api/types/rocketpool.go | 3 +- 5 files changed, 80 insertions(+), 78 deletions(-) diff --git a/backend/pkg/api/data_access/vdb_blocks.go b/backend/pkg/api/data_access/vdb_blocks.go index b7c2194e3..92ff09cae 100644 --- a/backend/pkg/api/data_access/vdb_blocks.go +++ b/backend/pkg/api/data_access/vdb_blocks.go @@ -42,6 +42,15 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das } } + // Get the rocketpool minipool infos + var rpInfos *t.RPInfo + if protocolModes.RocketPool { + rpInfos, err = d.getRocketPoolInfos(ctx, dashboardId, t.AllGroups) + if err != nil { + return nil, nil, err + } + } + searchPubkey := regexp.MustCompile(`^0x[0-9a-fA-F]{96}$`).MatchString(search) searchGroup := regexp.MustCompile(`^[a-zA-Z0-9_\-.\ ]+$`).MatchString(search) searchIndex := regexp.MustCompile(`^[0-9]+$`).MatchString(search) @@ -175,10 +184,9 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das goqu.COALESCE(goqu.L("rb.value / 1e18"), goqu.I("ep.fee_recipient_reward")).As("el_reward"), ) - if protocolModes.RocketPool { - // TODO: Add smoothing pool address to the parameters + if rpInfos != nil && protocolModes.RocketPool { blocksDs = blocksDs. - SelectAppend(goqu.L("blocks.exec_fee_recipient = ? AND (rb.proposer_fee_recipient IS NULL OR rb.proposer_fee_recipient = ?) AS is_smoothing_pool", 1, 2)) + SelectAppend(goqu.L("blocks.exec_fee_recipient = ? AND (rb.proposer_fee_recipient IS NULL OR rb.proposer_fee_recipient = ?) AS is_smoothing_pool", rpInfos.SmoothingPoolAddress, rpInfos.SmoothingPoolAddress)) } // 3. Sorting and pagination @@ -278,7 +286,7 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das goqu.L("NULL::NUMERIC").As("el_reward"), ) - if protocolModes.RocketPool { + if rpInfos != nil && protocolModes.RocketPool { blocksDs = blocksDs. SelectAppend(goqu.L("false").As("is_smoothing_pool")) } @@ -358,15 +366,6 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das slices.Reverse(proposals) } - // Get the rocketpool minipool infos - var rpInfos *t.RPInfo - if protocolModes.RocketPool { - rpInfos, err = d.getRocketPoolInfos(ctx, dashboardId, t.AllGroups) - if err != nil { - return nil, nil, err - } - } - slots := make([]uint64, len(proposals)) for i, proposal := range proposals { slots[i] = proposal.Slot diff --git a/backend/pkg/api/data_access/vdb_helpers.go b/backend/pkg/api/data_access/vdb_helpers.go index c942d56c5..267866406 100644 --- a/backend/pkg/api/data_access/vdb_helpers.go +++ b/backend/pkg/api/data_access/vdb_helpers.go @@ -138,13 +138,14 @@ func (d *DataAccessService) getRocketPoolInfos(ctx context.Context, dashboardId wg := errgroup.Group{} queryResult := []struct { - ValidatorIndex uint64 `db:"validatorindex"` - NodeAddress []byte `db:"node_address"` - NodeFee float64 `db:"node_fee"` - NodeDepositBalance decimal.Decimal `db:"node_deposit_balance"` - UserDepositBalance decimal.Decimal `db:"user_deposit_balance"` - EndTime sql.NullTime `db:"end_time"` - SmoothingPoolEth *decimal.Decimal `db:"smoothing_pool_eth"` + ValidatorIndex uint64 `db:"validatorindex"` + NodeAddress []byte `db:"node_address"` + NodeFee float64 `db:"node_fee"` + NodeDepositBalance decimal.Decimal `db:"node_deposit_balance"` + UserDepositBalance decimal.Decimal `db:"user_deposit_balance"` + EndTime sql.NullTime `db:"end_time"` + SmoothingPoolAddress []byte `db:"smoothing_pool_address"` + SmoothingPoolEth *decimal.Decimal `db:"smoothing_pool_eth"` }{} wg.Go(func() error { @@ -156,11 +157,13 @@ func (d *DataAccessService) getRocketPoolInfos(ctx context.Context, dashboardId goqu.L("rplm.node_deposit_balance"), goqu.L("rplm.user_deposit_balance"), goqu.L("rplrs.end_time"), + goqu.L("rploc.smoothing_pool_address"), goqu.L("rplrs.smoothing_pool_eth"), ). From(goqu.L("rocketpool_minipools AS rplm")). LeftJoin(goqu.L("validators AS v"), goqu.On(goqu.L("rplm.pubkey = v.pubkey"))). LeftJoin(goqu.L("rocketpool_rewards_summary AS rplrs"), goqu.On(goqu.L("rplm.node_address = rplrs.node_address"))). + LeftJoin(goqu.L("rocketpool_onchain_configs AS rploc"), goqu.On(goqu.L("rplm.rocketpool_storage_address = rploc.rocketpool_storage_address"))). Where(goqu.L("rplm.node_deposit_balance IS NOT NULL")). Where(goqu.L("rplm.user_deposit_balance IS NOT NULL")) @@ -223,6 +226,8 @@ func (d *DataAccessService) getRocketPoolInfos(ctx context.Context, dashboardId rpInfo := t.RPInfo{ Minipool: make(map[uint64]t.RPMinipoolInfo), + // Smoothing pool address is the same for all nodes on the network so take the first result + SmoothingPoolAddress: queryResult[0].SmoothingPoolAddress, } for _, res := range queryResult { diff --git a/backend/pkg/api/data_access/vdb_rewards.go b/backend/pkg/api/data_access/vdb_rewards.go index d3cee4741..2f9082219 100644 --- a/backend/pkg/api/data_access/vdb_rewards.go +++ b/backend/pkg/api/data_access/vdb_rewards.go @@ -77,6 +77,16 @@ func (d *DataAccessService) GetValidatorDashboardRewards(ctx context.Context, da groupIdSearchMap := make(map[uint64]bool, 0) + // ------------------------------------------------------------------------------------------------------------------ + // Get rocketpool minipool infos if needed + var rpInfos *t.RPInfo + if protocolModes.RocketPool { + rpInfos, err = d.getRocketPoolInfos(ctx, dashboardId, t.AllGroups) + if err != nil { + return nil, nil, err + } + } + // ------------------------------------------------------------------------------------------------------------------ // Build the main and EL rewards queries rewardsDs := goqu.Dialect("postgres"). @@ -117,11 +127,10 @@ func (d *DataAccessService) GetValidatorDashboardRewards(ctx context.Context, da ). GroupBy(goqu.L("b.epoch"), goqu.L("b.proposer")) - if protocolModes.RocketPool { + if rpInfos != nil && protocolModes.RocketPool { // Exclude rewards that went to the smoothing pool - // TODO: Add smoothing pool address to the parameters elDs = elDs. - Where(goqu.L("b.exec_fee_recipient != ? OR (rb.proposer_fee_recipient IS NOT NULL AND rb.proposer_fee_recipient != ?)", 1, 2)) + Where(goqu.L("b.exec_fee_recipient != ? OR (rb.proposer_fee_recipient IS NOT NULL AND rb.proposer_fee_recipient != ?)", rpInfos.SmoothingPoolAddress, rpInfos.SmoothingPoolAddress)) } if dashboardId.Validators == nil { @@ -296,16 +305,6 @@ func (d *DataAccessService) GetValidatorDashboardRewards(ctx context.Context, da } } - // ------------------------------------------------------------------------------------------------------------------ - // Get rocketpool minipool infos if needed - var rpInfos *t.RPInfo - if protocolModes.RocketPool { - rpInfos, err = d.getRocketPoolInfos(ctx, dashboardId, t.AllGroups) - if err != nil { - return nil, nil, err - } - } - // ------------------------------------------------------------------------------------------------------------------ // Build the main query and get the data @@ -634,12 +633,23 @@ func (d *DataAccessService) GetValidatorDashboardGroupRewards(ctx context.Contex ret := &t.VDBGroupRewardsData{} wg := errgroup.Group{} + var err error if dashboardId.AggregateGroups { // If we are aggregating groups then ignore the group id and sum up everything groupId = t.AllGroups } + // ------------------------------------------------------------------------------------------------------------------ + // Get rocketpool minipool infos if needed + var rpInfos *t.RPInfo + if protocolModes.RocketPool { + rpInfos, err = d.getRocketPoolInfos(ctx, dashboardId, groupId) + if err != nil { + return nil, err + } + } + // ------------------------------------------------------------------------------------------------------------------ // Build the main and EL rewards queries rewardsDs := goqu.Dialect("postgres"). @@ -689,11 +699,10 @@ func (d *DataAccessService) GetValidatorDashboardGroupRewards(ctx context.Contex Where(goqu.L("b.epoch = ?", epoch)). GroupBy(goqu.L("b.proposer")) - if protocolModes.RocketPool { + if rpInfos != nil && protocolModes.RocketPool { // Exclude rewards that went to the smoothing pool - // TODO: Add smoothing pool address to the parameters elDs = elDs. - Where(goqu.L("b.exec_fee_recipient != ? OR (rb.proposer_fee_recipient IS NOT NULL AND rb.proposer_fee_recipient != ?)", 1, 2)) + Where(goqu.L("b.exec_fee_recipient != ? OR (rb.proposer_fee_recipient IS NOT NULL AND rb.proposer_fee_recipient != ?)", rpInfos.SmoothingPoolAddress, rpInfos.SmoothingPoolAddress)) } if dashboardId.Validators == nil { @@ -785,21 +794,13 @@ func (d *DataAccessService) GetValidatorDashboardGroupRewards(ctx context.Contex return nil }) - err := wg.Wait() + err = wg.Wait() if err != nil { return nil, fmt.Errorf("error retrieving validator dashboard group rewards data: %w", err) } // ------------------------------------------------------------------------------------------------------------------ // Create the result - var rpInfos *t.RPInfo - if protocolModes.RocketPool { - rpInfos, err = d.getRocketPoolInfos(ctx, dashboardId, groupId) - if err != nil { - return nil, err - } - } - gWei := decimal.NewFromInt(1e9) for _, entry := range queryResult { @@ -876,6 +877,16 @@ func (d *DataAccessService) GetValidatorDashboardRewardsChart(ctx context.Contex startEpoch = latestFinalizedEpoch - epochLookBack } + // ------------------------------------------------------------------------------------------------------------------ + // Get rocketpool minipool infos if needed + var rpInfos *t.RPInfo + if protocolModes.RocketPool { + rpInfos, err = d.getRocketPoolInfos(ctx, dashboardId, t.AllGroups) + if err != nil { + return nil, err + } + } + // ------------------------------------------------------------------------------------------------------------------ // Build the main and EL rewards queries rewardsDs := goqu.Dialect("postgres"). @@ -908,11 +919,10 @@ func (d *DataAccessService) GetValidatorDashboardRewardsChart(ctx context.Contex Where(goqu.L("b.epoch >= ?", startEpoch)). GroupBy(goqu.L("b.epoch"), goqu.L("b.proposer")) - if protocolModes.RocketPool { + if rpInfos != nil && protocolModes.RocketPool { // Exclude rewards that went to the smoothing pool - // TODO: Add smoothing pool address to the parameters elDs = elDs. - Where(goqu.L("b.exec_fee_recipient != ? OR (rb.proposer_fee_recipient IS NOT NULL AND rb.proposer_fee_recipient != ?)", 1, 2)) + Where(goqu.L("b.exec_fee_recipient != ? OR (rb.proposer_fee_recipient IS NOT NULL AND rb.proposer_fee_recipient != ?)", rpInfos.SmoothingPoolAddress, rpInfos.SmoothingPoolAddress)) } if dashboardId.Validators == nil { @@ -950,16 +960,6 @@ func (d *DataAccessService) GetValidatorDashboardRewardsChart(ctx context.Contex Order(goqu.L("b.epoch").Asc()) } - // ------------------------------------------------------------------------------------------------------------------ - // Get rocketpool minipool infos if needed - var rpInfos *t.RPInfo - if protocolModes.RocketPool { - rpInfos, err = d.getRocketPoolInfos(ctx, dashboardId, t.AllGroups) - if err != nil { - return nil, err - } - } - // ------------------------------------------------------------------------------------------------------------------ // Build the main query and get the data type QueryResultSum struct { @@ -1202,6 +1202,16 @@ func (d *DataAccessService) GetValidatorDashboardDuties(ctx context.Context, das } } + // ------------------------------------------------------------------------------------------------------------------ + // Get rocketpool minipool infos if needed + var rpInfos *t.RPInfo + if protocolModes.RocketPool { + rpInfos, err = d.getRocketPoolInfos(ctx, dashboardId, groupId) + if err != nil { + return nil, nil, err + } + } + // ------------------------------------------------------------------------------------------------------------------ // Build the main and EL rewards queries rewardsDs := goqu.Dialect("postgres"). @@ -1253,11 +1263,10 @@ func (d *DataAccessService) GetValidatorDashboardDuties(ctx context.Context, das Where(goqu.L("b.status = '1'")). GroupBy(goqu.L("b.proposer")) - if protocolModes.RocketPool { + if rpInfos != nil && protocolModes.RocketPool { // Exclude rewards that went to the smoothing pool - // TODO: Add smoothing pool address to the parameters elDs = elDs. - Where(goqu.L("b.exec_fee_recipient != ? OR (rb.proposer_fee_recipient IS NOT NULL AND rb.proposer_fee_recipient != ?)", 1, 2)) + Where(goqu.L("b.exec_fee_recipient != ? OR (rb.proposer_fee_recipient IS NOT NULL AND rb.proposer_fee_recipient != ?)", rpInfos.SmoothingPoolAddress, rpInfos.SmoothingPoolAddress)) } // ------------------------------------------------------------------------------------------------------------------ @@ -1296,16 +1305,6 @@ func (d *DataAccessService) GetValidatorDashboardDuties(ctx context.Context, das elDs = elDs.Where(goqu.L("b.proposer = ANY(?)", pq.Array(validators))) } - // ------------------------------------------------------------------------------------------------------------------ - // Get rocketpool minipool infos if needed - var rpInfos *t.RPInfo - if protocolModes.RocketPool { - rpInfos, err = d.getRocketPoolInfos(ctx, dashboardId, groupId) - if err != nil { - return nil, nil, err - } - } - // ------------------------------------------------------------------------------------------------------------------ // Get the main data type QueryResultBase struct { diff --git a/backend/pkg/api/data_access/vdb_summary.go b/backend/pkg/api/data_access/vdb_summary.go index e90f6ee47..3ca289b4c 100644 --- a/backend/pkg/api/data_access/vdb_summary.go +++ b/backend/pkg/api/data_access/vdb_summary.go @@ -260,11 +260,10 @@ func (d *DataAccessService) GetValidatorDashboardSummary(ctx context.Context, da Where(goqu.L("b.epoch >= ? AND b.epoch <= ? AND b.status = '1'", epochStart, epochEnd)). GroupBy(goqu.L("b.proposer")) - if protocolModes.RocketPool { + if rpInfos != nil && protocolModes.RocketPool { // Exclude rewards that went to the smoothing pool - // TODO: Add smoothing pool address to the parameters ds = ds. - Where(goqu.L("b.exec_fee_recipient != ? OR (rb.proposer_fee_recipient IS NOT NULL AND rb.proposer_fee_recipient != ?)", 1, 2)) + Where(goqu.L("b.exec_fee_recipient != ? OR (rb.proposer_fee_recipient IS NOT NULL AND rb.proposer_fee_recipient != ?)", rpInfos.SmoothingPoolAddress, rpInfos.SmoothingPoolAddress)) } if len(validators) > 0 { @@ -1073,11 +1072,10 @@ func (d *DataAccessService) internal_getElClAPR(ctx context.Context, dashboardId Where(goqu.L("b.status = '1'")). GroupBy("b.proposer") - if protocolModes.RocketPool { + if rpInfos != nil && protocolModes.RocketPool { // Exclude rewards that went to the smoothing pool - // TODO: Add smoothing pool address to the parameters elDs = elDs. - Where(goqu.L("b.exec_fee_recipient != ? OR (rb.proposer_fee_recipient IS NOT NULL AND rb.proposer_fee_recipient != ?)", 1, 2)) + Where(goqu.L("b.exec_fee_recipient != ? OR (rb.proposer_fee_recipient IS NOT NULL AND rb.proposer_fee_recipient != ?)", rpInfos.SmoothingPoolAddress, rpInfos.SmoothingPoolAddress)) } if len(dashboardId.Validators) > 0 { diff --git a/backend/pkg/api/types/rocketpool.go b/backend/pkg/api/types/rocketpool.go index f432e539a..d34324e5f 100644 --- a/backend/pkg/api/types/rocketpool.go +++ b/backend/pkg/api/types/rocketpool.go @@ -10,7 +10,8 @@ type RPNetworkStats struct { } type RPInfo struct { - Minipool map[uint64]RPMinipoolInfo + Minipool map[uint64]RPMinipoolInfo + SmoothingPoolAddress []byte } type RPMinipoolInfo struct { From 78db420eabd1e6a9a7cbe5ae0bd052e1a3643637 Mon Sep 17 00:00:00 2001 From: Stefan Pletka <124689083+Eisei24@users.noreply.github.com> Date: Tue, 26 Nov 2024 13:17:30 +0100 Subject: [PATCH 17/19] Fixed various bugs --- backend/pkg/api/data_access/vdb_blocks.go | 6 ++--- backend/pkg/api/data_access/vdb_helpers.go | 4 ++++ backend/pkg/api/data_access/vdb_management.go | 6 ++++- backend/pkg/api/data_access/vdb_rewards.go | 23 ++++++++++--------- backend/pkg/api/data_access/vdb_summary.go | 15 ++++++------ 5 files changed, 31 insertions(+), 23 deletions(-) diff --git a/backend/pkg/api/data_access/vdb_blocks.go b/backend/pkg/api/data_access/vdb_blocks.go index 92ff09cae..ce47063b4 100644 --- a/backend/pkg/api/data_access/vdb_blocks.go +++ b/backend/pkg/api/data_access/vdb_blocks.go @@ -24,8 +24,6 @@ import ( ) func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, dashboardId t.VDBId, cursor string, colSort t.Sort[enums.VDBBlocksColumn], search string, limit uint64, protocolModes t.VDBProtocolModes) ([]t.VDBBlocksTableRow, *t.Paging, error) { - // @DATA-ACCESS incorporate protocolModes - // ------------------------------------- // Setup var err error @@ -186,7 +184,7 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das if rpInfos != nil && protocolModes.RocketPool { blocksDs = blocksDs. - SelectAppend(goqu.L("blocks.exec_fee_recipient = ? AND (rb.proposer_fee_recipient IS NULL OR rb.proposer_fee_recipient = ?) AS is_smoothing_pool", rpInfos.SmoothingPoolAddress, rpInfos.SmoothingPoolAddress)) + SelectAppend(goqu.L("(blocks.exec_fee_recipient = ? AND (rb.proposer_fee_recipient IS NULL OR rb.proposer_fee_recipient = ?)) AS is_smoothing_pool", rpInfos.SmoothingPoolAddress, rpInfos.SmoothingPoolAddress)) } // 3. Sorting and pagination @@ -287,7 +285,7 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das ) if rpInfos != nil && protocolModes.RocketPool { - blocksDs = blocksDs. + scheduledDs = scheduledDs. SelectAppend(goqu.L("false").As("is_smoothing_pool")) } diff --git a/backend/pkg/api/data_access/vdb_helpers.go b/backend/pkg/api/data_access/vdb_helpers.go index 267866406..b336059c6 100644 --- a/backend/pkg/api/data_access/vdb_helpers.go +++ b/backend/pkg/api/data_access/vdb_helpers.go @@ -224,6 +224,10 @@ func (d *DataAccessService) getRocketPoolInfos(ctx context.Context, dashboardId return nil, err } + if len(queryResult) == 0 { + return nil, nil + } + rpInfo := t.RPInfo{ Minipool: make(map[uint64]t.RPMinipoolInfo), // Smoothing pool address is the same for all nodes on the network so take the first result diff --git a/backend/pkg/api/data_access/vdb_management.go b/backend/pkg/api/data_access/vdb_management.go index 2328cb85a..223d7a6d8 100644 --- a/backend/pkg/api/data_access/vdb_management.go +++ b/backend/pkg/api/data_access/vdb_management.go @@ -375,7 +375,11 @@ func (d *DataAccessService) GetValidatorDashboardOverview(ctx context.Context, d validatorBalance := utils.GWeiToWei(big.NewInt(int64(metadata.Balance))) effectiveBalance := utils.GWeiToWei(big.NewInt(int64(metadata.EffectiveBalance))) - if rpValidator, ok := rpInfos.Minipool[validator]; ok { + if rpInfos == nil { + data.Balances.Total = data.Balances.Total.Add(validatorBalance) + + nonRpDashboardId.Validators = append(nonRpDashboardId.Validators, validator) + } else if rpValidator, ok := rpInfos.Minipool[validator]; ok { if protocolModes.RocketPool { // Calculate the balance of the operator fullDeposit := rpValidator.UserDepositBalance.Add(rpValidator.NodeDepositBalance) diff --git a/backend/pkg/api/data_access/vdb_rewards.go b/backend/pkg/api/data_access/vdb_rewards.go index 2f9082219..4fe80fdbb 100644 --- a/backend/pkg/api/data_access/vdb_rewards.go +++ b/backend/pkg/api/data_access/vdb_rewards.go @@ -21,7 +21,6 @@ import ( ) func (d *DataAccessService) GetValidatorDashboardRewards(ctx context.Context, dashboardId t.VDBId, cursor string, colSort t.Sort[enums.VDBRewardsColumn], search string, limit uint64, protocolModes t.VDBProtocolModes) ([]t.VDBRewardsTableRow, *t.Paging, error) { - // @DATA-ACCESS incorporate protocolModes result := make([]t.VDBRewardsTableRow, 0) var paging t.Paging @@ -120,9 +119,10 @@ func (d *DataAccessService) GetValidatorDashboardRewards(ctx context.Context, da From("relays_blocks"). Select( goqu.L("exec_block_hash"), + goqu.L("proposer_fee_recipient"), goqu.MAX("value").As("value")). Where(goqu.L("relays_blocks.exec_block_hash = b.exec_block_hash")). - GroupBy("exec_block_hash")).As("rb"), + GroupBy("exec_block_hash", "proposer_fee_recipient")).As("rb"), goqu.On(goqu.L("rb.exec_block_hash = b.exec_block_hash")), ). GroupBy(goqu.L("b.epoch"), goqu.L("b.proposer")) @@ -130,7 +130,7 @@ func (d *DataAccessService) GetValidatorDashboardRewards(ctx context.Context, da if rpInfos != nil && protocolModes.RocketPool { // Exclude rewards that went to the smoothing pool elDs = elDs. - Where(goqu.L("b.exec_fee_recipient != ? OR (rb.proposer_fee_recipient IS NOT NULL AND rb.proposer_fee_recipient != ?)", rpInfos.SmoothingPoolAddress, rpInfos.SmoothingPoolAddress)) + Where(goqu.L("(b.exec_fee_recipient != ? OR (rb.proposer_fee_recipient IS NOT NULL AND rb.proposer_fee_recipient != ?))", rpInfos.SmoothingPoolAddress, rpInfos.SmoothingPoolAddress)) } if dashboardId.Validators == nil { @@ -629,7 +629,6 @@ func (d *DataAccessService) GetValidatorDashboardRewards(ctx context.Context, da } func (d *DataAccessService) GetValidatorDashboardGroupRewards(ctx context.Context, dashboardId t.VDBId, groupId int64, epoch uint64, protocolModes t.VDBProtocolModes) (*t.VDBGroupRewardsData, error) { - // @DATA-ACCESS incorporate protocolModes ret := &t.VDBGroupRewardsData{} wg := errgroup.Group{} @@ -691,9 +690,10 @@ func (d *DataAccessService) GetValidatorDashboardGroupRewards(ctx context.Contex From("relays_blocks"). Select( goqu.L("exec_block_hash"), + goqu.L("proposer_fee_recipient"), goqu.MAX("value").As("value")). Where(goqu.L("relays_blocks.exec_block_hash = b.exec_block_hash")). - GroupBy("exec_block_hash")).As("rb"), + GroupBy("exec_block_hash", "proposer_fee_recipient")).As("rb"), goqu.On(goqu.L("rb.exec_block_hash = b.exec_block_hash")), ). Where(goqu.L("b.epoch = ?", epoch)). @@ -702,7 +702,7 @@ func (d *DataAccessService) GetValidatorDashboardGroupRewards(ctx context.Contex if rpInfos != nil && protocolModes.RocketPool { // Exclude rewards that went to the smoothing pool elDs = elDs. - Where(goqu.L("b.exec_fee_recipient != ? OR (rb.proposer_fee_recipient IS NOT NULL AND rb.proposer_fee_recipient != ?)", rpInfos.SmoothingPoolAddress, rpInfos.SmoothingPoolAddress)) + Where(goqu.L("(b.exec_fee_recipient != ? OR (rb.proposer_fee_recipient IS NOT NULL AND rb.proposer_fee_recipient != ?))", rpInfos.SmoothingPoolAddress, rpInfos.SmoothingPoolAddress)) } if dashboardId.Validators == nil { @@ -862,7 +862,6 @@ func (d *DataAccessService) GetValidatorDashboardGroupRewards(ctx context.Contex } func (d *DataAccessService) GetValidatorDashboardRewardsChart(ctx context.Context, dashboardId t.VDBId, protocolModes t.VDBProtocolModes) (*t.ChartData[int, decimal.Decimal], error) { - // @DATA-ACCESS incorporate protocolModes // bar chart for the CL and EL rewards for each group for each epoch. // NO series for all groups combined except if AggregateGroups is true. // series id is group id, series property is 'cl' or 'el' @@ -911,9 +910,10 @@ func (d *DataAccessService) GetValidatorDashboardRewardsChart(ctx context.Contex From("relays_blocks"). Select( goqu.L("exec_block_hash"), + goqu.L("proposer_fee_recipient"), goqu.MAX("value").As("value")). Where(goqu.L("relays_blocks.exec_block_hash = b.exec_block_hash")). - GroupBy("exec_block_hash")).As("rb"), + GroupBy("exec_block_hash", "proposer_fee_recipient")).As("rb"), goqu.On(goqu.L("rb.exec_block_hash = b.exec_block_hash")), ). Where(goqu.L("b.epoch >= ?", startEpoch)). @@ -922,7 +922,7 @@ func (d *DataAccessService) GetValidatorDashboardRewardsChart(ctx context.Contex if rpInfos != nil && protocolModes.RocketPool { // Exclude rewards that went to the smoothing pool elDs = elDs. - Where(goqu.L("b.exec_fee_recipient != ? OR (rb.proposer_fee_recipient IS NOT NULL AND rb.proposer_fee_recipient != ?)", rpInfos.SmoothingPoolAddress, rpInfos.SmoothingPoolAddress)) + Where(goqu.L("(b.exec_fee_recipient != ? OR (rb.proposer_fee_recipient IS NOT NULL AND rb.proposer_fee_recipient != ?))", rpInfos.SmoothingPoolAddress, rpInfos.SmoothingPoolAddress)) } if dashboardId.Validators == nil { @@ -1254,9 +1254,10 @@ func (d *DataAccessService) GetValidatorDashboardDuties(ctx context.Context, das From("relays_blocks"). Select( goqu.L("exec_block_hash"), + goqu.L("proposer_fee_recipient"), goqu.MAX("value").As("value")). Where(goqu.L("relays_blocks.exec_block_hash = b.exec_block_hash")). - GroupBy("exec_block_hash")).As("rb"), + GroupBy("exec_block_hash", "proposer_fee_recipient")).As("rb"), goqu.On(goqu.L("rb.exec_block_hash = b.exec_block_hash")), ). Where(goqu.L("b.epoch = ?", epoch)). @@ -1266,7 +1267,7 @@ func (d *DataAccessService) GetValidatorDashboardDuties(ctx context.Context, das if rpInfos != nil && protocolModes.RocketPool { // Exclude rewards that went to the smoothing pool elDs = elDs. - Where(goqu.L("b.exec_fee_recipient != ? OR (rb.proposer_fee_recipient IS NOT NULL AND rb.proposer_fee_recipient != ?)", rpInfos.SmoothingPoolAddress, rpInfos.SmoothingPoolAddress)) + Where(goqu.L("(b.exec_fee_recipient != ? OR (rb.proposer_fee_recipient IS NOT NULL AND rb.proposer_fee_recipient != ?))", rpInfos.SmoothingPoolAddress, rpInfos.SmoothingPoolAddress)) } // ------------------------------------------------------------------------------------------------------------------ diff --git a/backend/pkg/api/data_access/vdb_summary.go b/backend/pkg/api/data_access/vdb_summary.go index 3ca289b4c..d49a9bb9c 100644 --- a/backend/pkg/api/data_access/vdb_summary.go +++ b/backend/pkg/api/data_access/vdb_summary.go @@ -30,7 +30,6 @@ import ( ) func (d *DataAccessService) GetValidatorDashboardSummary(ctx context.Context, dashboardId t.VDBId, period enums.TimePeriod, cursor string, colSort t.Sort[enums.VDBSummaryColumn], search string, limit uint64, protocolModes t.VDBProtocolModes) ([]t.VDBSummaryTableRow, *t.Paging, error) { - // @DATA-ACCESS incorporate protocolModes result := make([]t.VDBSummaryTableRow, 0) var paging t.Paging @@ -235,6 +234,7 @@ func (d *DataAccessService) GetValidatorDashboardSummary(ctx context.Context, da } } groupSum.ClRewards = groupSum.ClRewards.Add(clRewardWei) + queryResultSumMap[row.GroupId] = groupSum } queryResultSum := slices.Collect(maps.Values(queryResultSumMap)) @@ -252,9 +252,10 @@ func (d *DataAccessService) GetValidatorDashboardSummary(ctx context.Context, da From("relays_blocks"). Select( goqu.L("exec_block_hash"), + goqu.L("proposer_fee_recipient"), goqu.MAX("value").As("value")). Where(goqu.L("relays_blocks.exec_block_hash = b.exec_block_hash")). - GroupBy("exec_block_hash")).As("rb"), + GroupBy("exec_block_hash", "proposer_fee_recipient")).As("rb"), goqu.On(goqu.L("rb.exec_block_hash = b.exec_block_hash")), ). Where(goqu.L("b.epoch >= ? AND b.epoch <= ? AND b.status = '1'", epochStart, epochEnd)). @@ -263,7 +264,7 @@ func (d *DataAccessService) GetValidatorDashboardSummary(ctx context.Context, da if rpInfos != nil && protocolModes.RocketPool { // Exclude rewards that went to the smoothing pool ds = ds. - Where(goqu.L("b.exec_fee_recipient != ? OR (rb.proposer_fee_recipient IS NOT NULL AND rb.proposer_fee_recipient != ?)", rpInfos.SmoothingPoolAddress, rpInfos.SmoothingPoolAddress)) + Where(goqu.L("(b.exec_fee_recipient != ? OR (rb.proposer_fee_recipient IS NOT NULL AND rb.proposer_fee_recipient != ?))", rpInfos.SmoothingPoolAddress, rpInfos.SmoothingPoolAddress)) } if len(validators) > 0 { @@ -583,7 +584,6 @@ func (d *DataAccessService) GetValidatorDashboardGroupSummary(ctx context.Contex // TODO: implement data retrieval for the following new field // Fetch validator list for user dashboard from the dashboard table when querying the past sync committees as the rolling table might miss exited validators // TotalMissedRewards - // @DATA-ACCESS incorporate protocolModes // @DATA-ACCESS implement data retrieval for Rocket Pool stats (if present) var err error @@ -1064,18 +1064,19 @@ func (d *DataAccessService) internal_getElClAPR(ctx context.Context, dashboardId From("relays_blocks"). Select( goqu.L("exec_block_hash"), + goqu.L("proposer_fee_recipient"), goqu.MAX("value").As("value")). Where(goqu.L("relays_blocks.exec_block_hash = b.exec_block_hash")). - GroupBy("exec_block_hash")).As("rb"), + GroupBy("exec_block_hash", "proposer_fee_recipient")).As("rb"), goqu.On(goqu.L("rb.exec_block_hash = b.exec_block_hash")), ). Where(goqu.L("b.status = '1'")). - GroupBy("b.proposer") + GroupBy(goqu.L("b.proposer")) if rpInfos != nil && protocolModes.RocketPool { // Exclude rewards that went to the smoothing pool elDs = elDs. - Where(goqu.L("b.exec_fee_recipient != ? OR (rb.proposer_fee_recipient IS NOT NULL AND rb.proposer_fee_recipient != ?)", rpInfos.SmoothingPoolAddress, rpInfos.SmoothingPoolAddress)) + Where(goqu.L("(b.exec_fee_recipient != ? OR (rb.proposer_fee_recipient IS NOT NULL AND rb.proposer_fee_recipient != ?))", rpInfos.SmoothingPoolAddress, rpInfos.SmoothingPoolAddress)) } if len(dashboardId.Validators) > 0 { From 7dc5572fe870e247dd766692023ab8119a99312f Mon Sep 17 00:00:00 2001 From: Stefan Pletka <124689083+Eisei24@users.noreply.github.com> Date: Tue, 26 Nov 2024 16:06:57 +0100 Subject: [PATCH 18/19] Fixed missing addresses for proposals --- backend/pkg/api/data_access/vdb_blocks.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/pkg/api/data_access/vdb_blocks.go b/backend/pkg/api/data_access/vdb_blocks.go index ce47063b4..d27baa5a2 100644 --- a/backend/pkg/api/data_access/vdb_blocks.go +++ b/backend/pkg/api/data_access/vdb_blocks.go @@ -286,7 +286,7 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das if rpInfos != nil && protocolModes.RocketPool { scheduledDs = scheduledDs. - SelectAppend(goqu.L("false").As("is_smoothing_pool")) + SelectAppend(goqu.L("NULL::BOOL").As("is_smoothing_pool")) } // We don't have access to exec_block_number and status for a WHERE without wrapping the query so if we sort by those get all the data @@ -334,7 +334,7 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das ElReward decimal.NullDecimal `db:"el_reward"` ClReward decimal.NullDecimal `db:"cl_reward"` GraffitiText sql.NullString `db:"graffiti_text"` - IsSmoothingPool bool `db:"is_smoothing_pool"` + IsSmoothingPool sql.NullBool `db:"is_smoothing_pool"` // for cursor only Reward decimal.Decimal @@ -462,7 +462,7 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das TraceIdx: -1, }) reward.El = proposal.ElReward.Decimal.Mul(decimal.NewFromInt(1e18)) - if rpInfos != nil && protocolModes.RocketPool && !proposal.IsSmoothingPool { + if rpInfos != nil && protocolModes.RocketPool && !(proposal.IsSmoothingPool.Valid && proposal.IsSmoothingPool.Bool) { if rpValidator, ok := rpInfos.Minipool[proposal.Proposer]; ok { reward.El = reward.El.Mul(d.getRocketPoolOperatorFactor(rpValidator)) } From 316e42e00d1f274d2f3b437b4182aefa00054fd6 Mon Sep 17 00:00:00 2001 From: Stefan Pletka <124689083+Eisei24@users.noreply.github.com> Date: Tue, 26 Nov 2024 16:24:57 +0100 Subject: [PATCH 19/19] Removed obsolete RPL struct --- backend/pkg/api/types/validator_dashboard.go | 5 ----- frontend/types/api/validator_dashboard.ts | 4 ---- 2 files changed, 9 deletions(-) diff --git a/backend/pkg/api/types/validator_dashboard.go b/backend/pkg/api/types/validator_dashboard.go index db3886072..ac38ba94c 100644 --- a/backend/pkg/api/types/validator_dashboard.go +++ b/backend/pkg/api/types/validator_dashboard.go @@ -98,11 +98,6 @@ type VDBGroupSummaryData struct { Apr ClElValue[float64] `json:"apr"` Luck Luck `json:"luck"` - - RocketPool struct { - Minipools uint64 `json:"minipools"` - Collateral float64 `json:"collateral"` - } `json:"rocket_pool,omitempty"` } type GetValidatorDashboardGroupSummaryResponse ApiDataResponse[VDBGroupSummaryData] diff --git a/frontend/types/api/validator_dashboard.ts b/frontend/types/api/validator_dashboard.ts index e85eac671..ac4546c0e 100644 --- a/frontend/types/api/validator_dashboard.ts +++ b/frontend/types/api/validator_dashboard.ts @@ -81,10 +81,6 @@ export interface VDBGroupSummaryData { missed_rewards: VDBGroupSummaryMissedRewards; apr: ClElValue; luck: Luck; - rocket_pool?: { - minipools: number /* uint64 */; - collateral: number /* float64 */; - }; } export type GetValidatorDashboardGroupSummaryResponse = ApiDataResponse; export type GetValidatorDashboardSummaryChartResponse = ApiDataResponse>; // line chart, series id is group id