From 9f673c260946b2b97b544d9a9b9624389fc1bdc0 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Tue, 11 Jun 2024 12:55:53 +0200 Subject: [PATCH] prevent exceeding the dashboard validator limit during batch inserts --- backend/pkg/api/data_access/vdb_management.go | 145 ++++++++++++++++-- 1 file changed, 136 insertions(+), 9 deletions(-) diff --git a/backend/pkg/api/data_access/vdb_management.go b/backend/pkg/api/data_access/vdb_management.go index 38c4a129c..60bcbfe2a 100644 --- a/backend/pkg/api/data_access/vdb_management.go +++ b/backend/pkg/api/data_access/vdb_management.go @@ -12,6 +12,7 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/gobitfly/beaconchain/pkg/api/enums" t "github.com/gobitfly/beaconchain/pkg/api/types" + "github.com/gobitfly/beaconchain/pkg/commons/log" "github.com/gobitfly/beaconchain/pkg/commons/utils" constypes "github.com/gobitfly/beaconchain/pkg/consapi/types" "github.com/lib/pq" @@ -790,13 +791,55 @@ func (d *DataAccessService) AddValidatorDashboardValidatorsByDepositAddress(dash if len(addressParsed) != 20 { return nil, fmt.Errorf("invalid deposit address: %s", address) } - var validatorIndices []uint64 - err = d.readerDb.Select(&validatorIndices, "SELECT validatorindex FROM validators WHERE pubkey IN (SELECT publickey FROM eth1_deposits WHERE from_address = $1) ORDER BY validatorindex LIMIT $2;", addressParsed, limit) + var validatorIndicesToAdd []uint64 + err = d.readerDb.Select(&validatorIndicesToAdd, "SELECT validatorindex FROM validators WHERE pubkey IN (SELECT publickey FROM eth1_deposits WHERE from_address = $1) ORDER BY validatorindex LIMIT $2;", addressParsed, limit) if err != nil { return nil, err } - // AddValidatorDashboardValidators will handle updating groups for validators already in the dashboard + // retrieve the existing validators + var existingValidators []uint64 + err = d.alloyWriter.Select(&existingValidators, "SELECT validator_index FROM users_val_dashboards_validators WHERE dashboard_id = $1", dashboardId) + if err != nil { + return nil, err + } + existingValidatorsMap := make(map[uint64]bool, len(existingValidators)) + for _, validatorIndex := range existingValidators { + existingValidatorsMap[validatorIndex] = true + } + + // filter out the validators that are already in the dashboard + var validatorIndicesToUpdate []uint64 + var validatorIndicesToInsert []uint64 + for _, validatorIndex := range validatorIndicesToAdd { + if _, ok := existingValidatorsMap[validatorIndex]; ok { + validatorIndicesToUpdate = append(validatorIndicesToUpdate, validatorIndex) + } else { + validatorIndicesToInsert = append(validatorIndicesToInsert, validatorIndex) + } + } + + // update the group for all existing validators + validatorIndices := make([]uint64, 0, int(limit)) + validatorIndices = append(validatorIndices, validatorIndicesToUpdate...) + + // insert the new validators up to the allowed user max limit taking into account how many validators are already in the dashboard + if len(validatorIndicesToInsert) > 0 { + freeSpace := int(limit) - len(existingValidators) + if freeSpace > 0 { + if len(validatorIndicesToInsert) > freeSpace { // cap inserts to the amount of free space available + log.Infof("limiting the number of validators to insert to %d", freeSpace) + validatorIndicesToInsert = validatorIndicesToInsert[:freeSpace] + } + validatorIndices = append(validatorIndices, validatorIndicesToInsert...) + } + } + + if len(validatorIndices) == 0 { + // no validators to add + return []t.VDBPostValidatorsData{}, nil + } + log.Infof("inserting %d new validators and updating %d validators of dashboard %d, limit is %d", len(validatorIndicesToInsert), len(validatorIndicesToUpdate), dashboardId, limit) return d.AddValidatorDashboardValidators(dashboardId, groupId, validatorIndices) } @@ -807,26 +850,110 @@ func (d *DataAccessService) AddValidatorDashboardValidatorsByWithdrawalAddress(d if err != nil { return nil, err } - var validatorIndices []uint64 - err = d.readerDb.Select(&validatorIndices, "SELECT validatorindex FROM validators WHERE withdrawalcredentials = $1 ORDER BY validatorindex LIMIT $2;", addressParsed, limit) + var validatorIndicesToAdd []uint64 + err = d.readerDb.Select(&validatorIndicesToAdd, "SELECT validatorindex FROM validators WHERE withdrawalcredentials = $1 ORDER BY validatorindex LIMIT $2;", addressParsed, limit) if err != nil { return nil, err } - // AddValidatorDashboardValidators will handle updating groups for validators already in the dashboard + // retrieve the existing validators + var existingValidators []uint64 + err = d.alloyWriter.Select(&existingValidators, "SELECT validator_index FROM users_val_dashboards_validators WHERE dashboard_id = $1", dashboardId) + if err != nil { + return nil, err + } + existingValidatorsMap := make(map[uint64]bool, len(existingValidators)) + for _, validatorIndex := range existingValidators { + existingValidatorsMap[validatorIndex] = true + } + + // filter out the validators that are already in the dashboard + var validatorIndicesToUpdate []uint64 + var validatorIndicesToInsert []uint64 + for _, validatorIndex := range validatorIndicesToAdd { + if _, ok := existingValidatorsMap[validatorIndex]; ok { + validatorIndicesToUpdate = append(validatorIndicesToUpdate, validatorIndex) + } else { + validatorIndicesToInsert = append(validatorIndicesToInsert, validatorIndex) + } + } + + // update the group for all existing validators + validatorIndices := make([]uint64, 0, int(limit)) + validatorIndices = append(validatorIndices, validatorIndicesToUpdate...) + + // insert the new validators up to the allowed user max limit taking into account how many validators are already in the dashboard + if len(validatorIndicesToInsert) > 0 { + freeSpace := int(limit) - len(existingValidators) + if freeSpace > 0 { + if len(validatorIndicesToInsert) > freeSpace { // cap inserts to the amount of free space available + log.Infof("limiting the number of validators to insert to %d", freeSpace) + validatorIndicesToInsert = validatorIndicesToInsert[:freeSpace] + } + validatorIndices = append(validatorIndices, validatorIndicesToInsert...) + } + } + + if len(validatorIndices) == 0 { + // no validators to add + return []t.VDBPostValidatorsData{}, nil + } + log.Infof("inserting %d new validators and updating %d validators of dashboard %d, limit is %d", len(validatorIndicesToInsert), len(validatorIndicesToUpdate), dashboardId, limit) return d.AddValidatorDashboardValidators(dashboardId, groupId, validatorIndices) } func (d *DataAccessService) AddValidatorDashboardValidatorsByGraffiti(dashboardId t.VDBIdPrimary, groupId uint64, graffiti string, limit uint64) ([]t.VDBPostValidatorsData, error) { // for all validators already in the dashboard that are associated with the graffiti (by produced block), update the group // then add no more than `limit` validators associated with the deposit address to the dashboard - var validatorIndices []uint64 - err := d.readerDb.Select(&validatorIndices, "SELECT DISTINCT proposer FROM blocks WHERE graffiti_text = $1 ORDER BY proposer LIMIT $2;", graffiti, limit) + var validatorIndicesToAdd []uint64 + err := d.readerDb.Select(&validatorIndicesToAdd, "SELECT DISTINCT proposer FROM blocks WHERE graffiti_text = $1 ORDER BY proposer LIMIT $2;", graffiti, limit) + if err != nil { + return nil, err + } + + // retrieve the existing validators + var existingValidators []uint64 + err = d.alloyWriter.Select(&existingValidators, "SELECT validator_index FROM users_val_dashboards_validators WHERE dashboard_id = $1", dashboardId) if err != nil { return nil, err } + existingValidatorsMap := make(map[uint64]bool, len(existingValidators)) + for _, validatorIndex := range existingValidators { + existingValidatorsMap[validatorIndex] = true + } + + // filter out the validators that are already in the dashboard + var validatorIndicesToUpdate []uint64 + var validatorIndicesToInsert []uint64 + for _, validatorIndex := range validatorIndicesToAdd { + if _, ok := existingValidatorsMap[validatorIndex]; ok { + validatorIndicesToUpdate = append(validatorIndicesToUpdate, validatorIndex) + } else { + validatorIndicesToInsert = append(validatorIndicesToInsert, validatorIndex) + } + } + + // update the group for all existing validators + validatorIndices := make([]uint64, 0, int(limit)) + validatorIndices = append(validatorIndices, validatorIndicesToUpdate...) - // AddValidatorDashboardValidators will handle updating groups for validators already in the dashboard + // insert the new validators up to the allowed user max limit taking into account how many validators are already in the dashboard + if len(validatorIndicesToInsert) > 0 { + freeSpace := int(limit) - len(existingValidators) + if freeSpace > 0 { + if len(validatorIndicesToInsert) > freeSpace { // cap inserts to the amount of free space available + log.Infof("limiting the number of validators to insert to %d", freeSpace) + validatorIndicesToInsert = validatorIndicesToInsert[:freeSpace] + } + validatorIndices = append(validatorIndices, validatorIndicesToInsert...) + } + } + + if len(validatorIndices) == 0 { + // no validators to add + return []t.VDBPostValidatorsData{}, nil + } + log.Infof("inserting %d new validators and updating %d validators of dashboard %d, limit is %d", len(validatorIndicesToInsert), len(validatorIndicesToUpdate), dashboardId, limit) return d.AddValidatorDashboardValidators(dashboardId, groupId, validatorIndices) }