From 52db45f80c3c6ff54a71c87a104c19454cf52e09 Mon Sep 17 00:00:00 2001 From: remoterami <142154971+remoterami@users.noreply.github.com> Date: Wed, 16 Oct 2024 13:33:43 +0200 Subject: [PATCH 01/82] scheduled blocks retrieval (WIP) --- backend/pkg/api/data_access/vdb_blocks.go | 213 +++++++++++++--------- 1 file changed, 126 insertions(+), 87 deletions(-) diff --git a/backend/pkg/api/data_access/vdb_blocks.go b/backend/pkg/api/data_access/vdb_blocks.go index ca881c0fb..4ab744b00 100644 --- a/backend/pkg/api/data_access/vdb_blocks.go +++ b/backend/pkg/api/data_access/vdb_blocks.go @@ -9,6 +9,7 @@ import ( "strings" "time" + "github.com/doug-martin/goqu/v9" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/gobitfly/beaconchain/pkg/api/enums" t "github.com/gobitfly/beaconchain/pkg/api/types" @@ -17,13 +18,38 @@ import ( "github.com/gobitfly/beaconchain/pkg/commons/log" "github.com/gobitfly/beaconchain/pkg/commons/types" "github.com/gobitfly/beaconchain/pkg/commons/utils" + "github.com/lib/pq" "github.com/shopspring/decimal" ) +type table string + +// Stringer interface +func (t table) String() string { + return string(t) +} + +//func (t table) C(column string) exp.IdentifierExpression { +// return goqu.I(string(t) + "." + column) +//} + +func (t table) C(column string) string { + return string(t) + "." + column +} + func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, dashboardId t.VDBId, cursor string, colSort t.Sort[enums.VDBBlocksColumn], search string, limit uint64, protocolModes t.VDBProtocolModes) ([]t.VDBBlocksTableRow, *t.Paging, error) { // @DATA-ACCESS incorporate protocolModes + + // ------------------------------------- + // Setup var err error var currentCursor t.BlocksCursor + validatorMapping, err := d.services.GetCurrentValidatorMapping() + if err != nil { + return nil, nil, err + } + validators := table("validators") + groups := table("goups") // TODO @LuccaBitfly move validation to handler? if cursor != "" { @@ -32,83 +58,127 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das } } - // regexes taken from api handler common.go searchPubkey := regexp.MustCompile(`^0x[0-9a-fA-F]{96}$`).MatchString(search) searchGroup := regexp.MustCompile(`^[a-zA-Z0-9_\-.\ ]+$`).MatchString(search) searchIndex := regexp.MustCompile(`^[0-9]+$`).MatchString(search) - validatorMap := make(map[t.VDBValidator]bool) - params := []interface{}{} - filteredValidatorsQuery := "" - validatorMapping, err := d.services.GetCurrentValidatorMapping() - if err != nil { - return nil, nil, err + // ------------------------------------- + // Goqu Query: Determine validators filtered by search + type validatorGroup struct { + Validator t.VDBValidator `db:"validator_index"` + Group uint64 `db:"group_id"` } - - // determine validators of interest first + var filteredValidators []validatorGroup + validatorsDs := goqu.Dialect("postgres"). + From( + goqu.T("users_val_dashboards_validators").As(validators), + ). + Select( + validators.C("validator_index"), + ) if dashboardId.Validators == nil { - // could also optimize this for the average and/or the whale case; will go with some middle-ground, needs testing - // (query validators twice: once without search applied (fast) to pre-filter scheduled proposals (which are sent to db, want to minimize), - // again for blocks query with search applied to not having to send potentially huge validator-list) - startTime := time.Now() - valis, err := d.getDashboardValidators(ctx, dashboardId, nil) - log.Debugf("=== getting validators took %s", time.Since(startTime)) - if err != nil { - return nil, nil, err - } - for _, v := range valis { - validatorMap[v] = true - } + validatorsDs = validatorsDs. + Select( + // TODO mustn't be here, can be done further down + validators.C("group_id"), + ). + Where(goqu.Ex{validators.C("dashboard_id"): dashboardId.Id}) - // create a subquery to get the (potentially filtered) validators and their groups for later - params = append(params, dashboardId.Id) - selectStr := `SELECT validator_index, group_id ` - from := `FROM users_val_dashboards_validators validators ` - where := `WHERE validators.dashboard_id = $1` - extraConds := make([]string, 0, 3) + // apply search filters if searchIndex { - params = append(params, search) - extraConds = append(extraConds, fmt.Sprintf(`validator_index = $%d`, len(params))) + validatorsDs = validatorsDs.Where(goqu.Ex{validators.C("validator_index"): search}) } if searchGroup { - from += `INNER JOIN users_val_dashboards_groups groups ON validators.dashboard_id = groups.dashboard_id AND validators.group_id = groups.id ` - // escape the psql single character wildcard "_"; apply prefix-search - params = append(params, strings.Replace(search, "_", "\\_", -1)+"%") - extraConds = append(extraConds, fmt.Sprintf(`LOWER(name) LIKE LOWER($%d)`, len(params))) + validatorsDs = validatorsDs. + InnerJoin(goqu.T("users_val_dashboards_groups").As(groups), goqu.On( + goqu.Ex{validators.C("dashboard_id"): groups.C("dashboard_id")}, + goqu.Ex{validators.C("group_id"): groups.C("id")}, + )). + Where( + goqu.L("LOWER(?)", groups.C("name")).Like(strings.Replace(search, "_", "\\_", -1) + "%"), + ) } if searchPubkey { index, ok := validatorMapping.ValidatorIndices[search] - if !ok && len(extraConds) == 0 { - // don't even need to query + if !ok && !searchGroup && !searchIndex { + // searched pubkey doesn't exist, don't even need to query anything return make([]t.VDBBlocksTableRow, 0), &t.Paging{}, nil } - params = append(params, index) - extraConds = append(extraConds, fmt.Sprintf(`validator_index = $%d`, len(params))) - } - if len(extraConds) > 0 { - where += ` AND (` + strings.Join(extraConds, ` OR `) + `)` - } - filteredValidatorsQuery = selectStr + from + where + validatorsDs = validatorsDs. + Where(goqu.Ex{validators.C("validator_index"): index}) + } } else { - validators := make([]t.VDBValidator, 0, len(dashboardId.Validators)) for _, validator := range dashboardId.Validators { if searchIndex && fmt.Sprint(validator) != search || searchPubkey && validator != validatorMapping.ValidatorIndices[search] { continue } - validatorMap[validator] = true - validators = append(validators, validator) + filteredValidators = append(filteredValidators, validatorGroup{ + Validator: validator, + Group: t.DefaultGroupId, + }) if searchIndex || searchPubkey { break } } - if len(validators) == 0 { - return make([]t.VDBBlocksTableRow, 0), &t.Paging{}, nil + validatorsDs = validatorsDs. + Where(goqu.L( + validators.C("validator_index")+" = ANY(?)", pq.Array(filteredValidators)), + ) + } + + if dashboardId.Validators == nil { + validatorsQuery, validatorsArgs, err := validatorsDs.Prepared(true).ToSQL() + if err != nil { + return nil, nil, err + } + if err = d.alloyReader.SelectContext(ctx, &filteredValidators, validatorsQuery, validatorsArgs...); err != nil { + return nil, nil, err + } + } + if len(filteredValidators) == 0 { + return make([]t.VDBBlocksTableRow, 0), &t.Paging{}, nil + } + + // ------------------------------------- + // Gather scheduled blocks + // found in dutiesInfo; pass results to final query later and let db do the sorting etc + validatorSet := make(map[t.VDBValidator]bool) + for _, v := range filteredValidators { + validatorSet[v.Validator] = true + } + var scheduledProposers []t.VDBValidator + var scheduledEpochs []uint64 + var scheduledSlots []uint64 + // don't need if requested slots are in the past + latestSlot := cache.LatestSlot.Get() + onlyPrimarySort := colSort.Column == enums.VDBBlockSlot || colSort.Column == enums.VDBBlockBlock + if !onlyPrimarySort || !currentCursor.IsValid() || + currentCursor.Slot > latestSlot+1 && currentCursor.Reverse != colSort.Desc || + currentCursor.Slot < latestSlot+1 && currentCursor.Reverse == colSort.Desc { + dutiesInfo, err := d.services.GetCurrentDutiesInfo() + if err == nil { + for slot, vali := range dutiesInfo.PropAssignmentsForSlot { + // only gather scheduled slots + if _, ok := dutiesInfo.SlotStatus[slot]; ok { + continue + } + // only gather slots scheduled for our validators + if _, ok := validatorSet[vali]; !ok { + continue + } + scheduledProposers = append(scheduledProposers, dutiesInfo.PropAssignmentsForSlot[slot]) + scheduledEpochs = append(scheduledEpochs, slot/utils.Config.Chain.ClConfig.SlotsPerEpoch) + scheduledSlots = append(scheduledSlots, slot) + } + } else { + log.Debugf("duties info not available, skipping scheduled slots: %s", err) } - params = append(params, validators) } + // WIP + var proposals []struct { Proposer t.VDBValidator `db:"proposer"` Group uint64 `db:"group_id"` @@ -126,26 +196,26 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das } // handle sorting + params := make([]any, 0) where := `` orderBy := `ORDER BY ` sortOrder := ` ASC` if colSort.Desc { sortOrder = ` DESC` } - var val any + var offset any sortColName := `slot` switch colSort.Column { case enums.VDBBlockProposer: sortColName = `proposer` - val = currentCursor.Proposer + offset = currentCursor.Proposer case enums.VDBBlockStatus: sortColName = `status` - val = currentCursor.Status + offset = currentCursor.Status case enums.VDBBlockProposerReward: sortColName = `el_reward + cl_reward` - val = currentCursor.Reward + offset = currentCursor.Reward } - onlyPrimarySort := sortColName == `slot` if currentCursor.IsValid() { sign := ` > ` if colSort.Desc && !currentCursor.IsReverse() || !colSort.Desc && currentCursor.IsReverse() { @@ -163,7 +233,7 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das if onlyPrimarySort { where += `slot` + sign + fmt.Sprintf(`$%d`, len(params)) } else { - params = append(params, val) + params = append(params, offset) secSign := ` < ` if currentCursor.IsReverse() { secSign = ` > ` @@ -190,36 +260,6 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das orderBy += `, slot ` + secSort } - // Get scheduled blocks. They aren't written to blocks table, get from duties - // Will just pass scheduled proposals to query and let db do the sorting etc - var scheduledProposers []t.VDBValidator - var scheduledEpochs []uint64 - var scheduledSlots []uint64 - // don't need to query if requested slots are in the past - latestSlot := cache.LatestSlot.Get() - if !onlyPrimarySort || !currentCursor.IsValid() || - currentCursor.Slot > latestSlot+1 && currentCursor.Reverse != colSort.Desc || - currentCursor.Slot < latestSlot+1 && currentCursor.Reverse == colSort.Desc { - dutiesInfo, err := d.services.GetCurrentDutiesInfo() - if err == nil { - for slot, vali := range dutiesInfo.PropAssignmentsForSlot { - // only gather scheduled slots - if _, ok := dutiesInfo.SlotStatus[slot]; ok { - continue - } - // only gather slots scheduled for our validators - if _, ok := validatorMap[vali]; !ok { - continue - } - scheduledProposers = append(scheduledProposers, dutiesInfo.PropAssignmentsForSlot[slot]) - scheduledEpochs = append(scheduledEpochs, slot/utils.Config.Chain.ClConfig.SlotsPerEpoch) - scheduledSlots = append(scheduledSlots, slot) - } - } else { - log.Debugf("duties info not available, skipping scheduled slots: %s", err) - } - } - groupIdCol := "group_id" if dashboardId.Validators != nil { groupIdCol = fmt.Sprintf("%d AS %s", t.DefaultGroupId, groupIdCol) @@ -256,9 +296,8 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das } query += `) as u `*/ if dashboardId.Validators == nil { - cte += fmt.Sprintf(` - INNER JOIN (%s) validators ON validators.validator_index = proposer - `, filteredValidatorsQuery) + //cte += fmt.Sprintf(` + //INNER JOIN (%s) validators ON validators.validator_index = proposer`, filteredValidatorsQuery) } else { if len(where) == 0 { where += `WHERE ` From ee680640335b128eb076b1c3e2e7cd7b411a9ff8 Mon Sep 17 00:00:00 2001 From: remoterami <142154971+remoterami@users.noreply.github.com> Date: Thu, 17 Oct 2024 14:07:45 +0200 Subject: [PATCH 02/82] base query (WIP) --- backend/pkg/api/data_access/vdb_blocks.go | 213 ++++++++---------- .../api/enums/validator_dashboard_enums.go | 17 ++ backend/pkg/api/types/data_access.go | 5 +- 3 files changed, 110 insertions(+), 125 deletions(-) diff --git a/backend/pkg/api/data_access/vdb_blocks.go b/backend/pkg/api/data_access/vdb_blocks.go index b5de790af..409099ebc 100644 --- a/backend/pkg/api/data_access/vdb_blocks.go +++ b/backend/pkg/api/data_access/vdb_blocks.go @@ -49,6 +49,7 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das return nil, nil, err } validators := table("validators") + blocks := table("blocks") groups := table("goups") // TODO @LuccaBitfly move validation to handler? @@ -70,18 +71,18 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das } var filteredValidators []validatorGroup validatorsDs := goqu.Dialect("postgres"). - From( - goqu.T("users_val_dashboards_validators").As(validators), - ). Select( validators.C("validator_index"), ) if dashboardId.Validators == nil { validatorsDs = validatorsDs. - Select( + From( + goqu.T("users_val_dashboards_validators").As(validators), + ). + /*Select( // TODO mustn't be here, can be done further down validators.C("group_id"), - ). + ).*/ Where(goqu.Ex{validators.C("dashboard_id"): dashboardId.Id}) // apply search filters @@ -123,9 +124,9 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das } } validatorsDs = validatorsDs. - Where(goqu.L( - validators.C("validator_index")+" = ANY(?)", pq.Array(filteredValidators)), - ) + From( + goqu.L("unnest(?)", pq.Array(filteredValidators)).As("validator_index"), + ).As(string(validators)) } if dashboardId.Validators == nil { @@ -177,124 +178,84 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das } } - // WIP - - var proposals []struct { - Proposer t.VDBValidator `db:"proposer"` - Group uint64 `db:"group_id"` - Epoch uint64 `db:"epoch"` - Slot uint64 `db:"slot"` - Status uint64 `db:"status"` - Block sql.NullInt64 `db:"exec_block_number"` - FeeRecipient []byte `db:"fee_recipient"` - ElReward decimal.NullDecimal `db:"el_reward"` - ClReward decimal.NullDecimal `db:"cl_reward"` - GraffitiText string `db:"graffiti_text"` - - // for cursor only - Reward decimal.Decimal - } - - // handle sorting - params := make([]any, 0) - where := `` - orderBy := `ORDER BY ` - sortOrder := ` ASC` - if colSort.Desc { - sortOrder = ` DESC` + // Sorting and pagination if cursor is present + defaultColumns := []t.SortColumn{ + {Column: enums.VDBBlocksColumns.Slot.ToString(), Desc: true, Offset: currentCursor.Slot}, } var offset any - sortColName := `slot` switch colSort.Column { - case enums.VDBBlockProposer: - sortColName = `proposer` + case enums.VDBBlocksColumns.Proposer: offset = currentCursor.Proposer - case enums.VDBBlockStatus: - sortColName = `status` - offset = currentCursor.Status - case enums.VDBBlockProposerReward: - sortColName = `el_reward + cl_reward` + case enums.VDBBlocksColumns.Block: + offset = currentCursor.Block + case enums.VDBBlocksColumns.Status: + offset = fmt.Sprintf("%d", currentCursor.Status) // type of 'status' column is text for some reason + case enums.VDBBlocksColumns.ProposerReward: offset = currentCursor.Reward } - if currentCursor.IsValid() { - sign := ` > ` - if colSort.Desc && !currentCursor.IsReverse() || !colSort.Desc && currentCursor.IsReverse() { - sign = ` < ` - } - if currentCursor.IsReverse() { - if sortOrder == ` ASC` { - sortOrder = ` DESC` - } else { - sortOrder = ` ASC` - } - } - params = append(params, currentCursor.Slot) - where += `WHERE (` - if onlyPrimarySort { - where += `slot` + sign + fmt.Sprintf(`$%d`, len(params)) - } else { - params = append(params, offset) - secSign := ` < ` - if currentCursor.IsReverse() { - secSign = ` > ` - } - if sortColName == "status" { - // explicit cast to int because type of 'status' column is text for some reason - sortColName += "::int" - } - where += fmt.Sprintf(`(slot`+secSign+`$%d AND `+sortColName+` = $%d) OR `+sortColName+sign+`$%d`, len(params)-1, len(params), len(params)) - } - where += `) ` + + order, directions := applySortAndPagination(defaultColumns, t.SortColumn{Column: colSort.Column.ToString(), Desc: colSort.Desc, Offset: offset}, currentCursor.GenericCursor) + validatorsDs = validatorsDs.Order(order...) + if directions != nil { + validatorsDs = validatorsDs.Where(directions) } - if sortOrder == ` ASC` { - sortOrder += ` NULLS FIRST` + + // group id + if dashboardId.Validators == nil { + validatorsDs = validatorsDs.Select( + validators.C("group_id"), + ) } else { - sortOrder += ` NULLS LAST` - } - orderBy += sortColName + sortOrder - secSort := `DESC` - if !onlyPrimarySort { - if currentCursor.IsReverse() { - secSort = `ASC` - } - orderBy += `, slot ` + secSort + validatorsDs = validatorsDs.Select( + goqu.L("?", t.DefaultGroupId).As("group_id"), + ) } - groupIdCol := "group_id" - if dashboardId.Validators != nil { - groupIdCol = fmt.Sprintf("%d AS %s", t.DefaultGroupId, groupIdCol) - } - selectFields := fmt.Sprintf(` - blocks.proposer, - blocks.epoch, - blocks.slot, - %s, - blocks.status, - exec_block_number, - COALESCE(rb.proposer_fee_recipient, blocks.exec_fee_recipient) AS fee_recipient, - COALESCE(rb.value / 1e18, ep.fee_recipient_reward) AS el_reward, - cp.cl_attestations_reward / 1e9 + cp.cl_sync_aggregate_reward / 1e9 + cp.cl_slashing_inclusion_reward / 1e9 as cl_reward, - blocks.graffiti_text`, groupIdCol) + validatorsDs = validatorsDs. + Select( + blocks.C("proposer"), + blocks.C("epoch"), + blocks.C("slot"), + blocks.C("status"), + blocks.C("exec_block_number"), + blocks.C("graffiti_text"), + ). + LeftJoin(goqu.T("consensus_payloads").As("cp"), goqu.On( + goqu.Ex{blocks.C("slot"): goqu.I("cp.slot")}, + )). + LeftJoin(goqu.T("execution_payloads").As("ep"), goqu.On( + goqu.Ex{blocks.C("exec_block_hash"): goqu.I("ep.block_hash")}, + )). + LeftJoin( + // relay bribe deduplication; select most likely (=max) relay bribe value for each block + goqu.Lateral(goqu.Dialect("postgres"). + From(goqu.T("relays_blocks")). + Select( + goqu.I("relays_blocks.exec_block_hash"), + goqu.MAX(goqu.I("relays_blocks.value")).As("value")). + // needed? TODO test + // Where(goqu.L("relays_blocks.exec_block_hash = blocks.exec_block_hash")). + GroupBy("exec_block_hash")).As("rb"), + goqu.On( + goqu.Ex{"rb.exec_block_hash": blocks.C("exec_block_hash")}, + ), + ). + Select( + goqu.COALESCE(goqu.I("rb.proposer_fee_recipient"), blocks.C("exec_fee_recipient")).As("fee_recipient"), + goqu.COALESCE(goqu.L("rb.value / 1e18"), goqu.I("ep.fee_recipient_reward")).As("el_reward"), + goqu.L("cp.cl_attestations_reward / 1e9 + cp.cl_sync_aggregate_reward / 1e9 + cp.cl_slashing_inclusion_reward / 1e9").As("cl_reward"), + ) + + // union scheduled blocks if present + // WIP + + params := make([]any, 0) + selectFields, where, orderBy, groupIdCol, sortColName := "", "", "", "", "" cte := fmt.Sprintf(`WITH past_blocks AS (SELECT %s FROM blocks `, selectFields) - /*if dashboardId.Validators == nil { - query += ` - LEFT JOIN cached_proposal_rewards ON cached_proposal_rewards.dashboard_id = $1 AND blocks.slot = cached_proposal_rewards.slot - ` - } else { - query += ` - LEFT JOIN execution_payloads ep ON ep.block_hash = blocks.exec_block_hash - LEFT JOIN relays_blocks rb ON rb.exec_block_hash = blocks.exec_block_hash - ` - } - // shrink selection to our filtered validators - if len(scheduledProposers) > 0 { - query += `)` - } - query += `) as u `*/ if dashboardId.Validators == nil { //cte += fmt.Sprintf(` //INNER JOIN (%s) validators ON validators.validator_index = proposer`, filteredValidatorsQuery) @@ -311,17 +272,6 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das limitStr := fmt.Sprintf(` LIMIT $%d `, len(params)) - // relay bribe deduplication; select most likely (=max) relay bribe value for each block - cte += ` - LEFT JOIN consensus_payloads cp on blocks.slot = cp.slot - LEFT JOIN execution_payloads ep ON ep.block_hash = blocks.exec_block_hash - LEFT JOIN LATERAL (SELECT exec_block_hash, proposer_fee_recipient, max(value) as value - FROM relays_blocks - WHERE relays_blocks.exec_block_hash = blocks.exec_block_hash - GROUP BY exec_block_hash, proposer_fee_recipient - ) rb ON rb.exec_block_hash = blocks.exec_block_hash - ) - ` from := `past_blocks ` selectStr := `SELECT * FROM ` @@ -375,7 +325,26 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das query = selectStr + from + where + orderBy + limitStr } + var proposals []struct { + Proposer t.VDBValidator `db:"proposer"` + Group uint64 `db:"group_id"` + Epoch uint64 `db:"epoch"` + Slot uint64 `db:"slot"` + Status uint64 `db:"status"` + Block sql.NullInt64 `db:"exec_block_number"` + FeeRecipient []byte `db:"fee_recipient"` + ElReward decimal.NullDecimal `db:"el_reward"` + ClReward decimal.NullDecimal `db:"cl_reward"` + GraffitiText string `db:"graffiti_text"` + + // for cursor only + Reward decimal.Decimal + } startTime := time.Now() + _, _, err = validatorsDs.Prepared(true).ToSQL() + if err != nil { + return nil, nil, err + } err = d.alloyReader.SelectContext(ctx, &proposals, cte+query, params...) log.Debugf("=== getting past blocks took %s", time.Since(startTime)) if err != nil { diff --git a/backend/pkg/api/enums/validator_dashboard_enums.go b/backend/pkg/api/enums/validator_dashboard_enums.go index 58e87f337..2646244e8 100644 --- a/backend/pkg/api/enums/validator_dashboard_enums.go +++ b/backend/pkg/api/enums/validator_dashboard_enums.go @@ -156,6 +156,23 @@ func (VDBBlocksColumn) NewFromString(s string) VDBBlocksColumn { } } +func (c VDBBlocksColumn) ToString() string { + switch c { + case VDBBlockProposer: + return "proposer" + case VDBBlockSlot: + return "slot" + case VDBBlockBlock: + return "block" + case VDBBlockStatus: + return "status" + case VDBBlockProposerReward: + return "reward" + default: + return "" + } +} + var VDBBlocksColumns = struct { Proposer VDBBlocksColumn Slot VDBBlocksColumn diff --git a/backend/pkg/api/types/data_access.go b/backend/pkg/api/types/data_access.go index db35871fb..3b1d77c16 100644 --- a/backend/pkg/api/types/data_access.go +++ b/backend/pkg/api/types/data_access.go @@ -164,11 +164,10 @@ type UserCredentialInfo struct { type BlocksCursor struct { GenericCursor - Slot uint64 // basically the same as Block, Epoch, Age; mandatory, used to index - // optional, max one of those (for now) Proposer uint64 - Group uint64 + Slot uint64 // same as Age + Block uint64 Status uint64 Reward decimal.Decimal } From cf997c42bd18f72054a4cdd86765b30c93befbdb Mon Sep 17 00:00:00 2001 From: remoterami <142154971+remoterami@users.noreply.github.com> Date: Fri, 18 Oct 2024 14:44:09 +0200 Subject: [PATCH 03/82] union past and scheduled blocks, syntax fixed --- backend/pkg/api/data_access/general.go | 13 +- backend/pkg/api/data_access/vdb_blocks.go | 224 +++++++----------- .../api/enums/validator_dashboard_enums.go | 2 +- backend/pkg/api/types/data_access.go | 13 +- .../dashboard/table/DashboardTableBlocks.vue | 2 +- 5 files changed, 108 insertions(+), 146 deletions(-) diff --git a/backend/pkg/api/data_access/general.go b/backend/pkg/api/data_access/general.go index 3c94db8a1..26d051dc9 100644 --- a/backend/pkg/api/data_access/general.go +++ b/backend/pkg/api/data_access/general.go @@ -62,6 +62,9 @@ func applySortAndPagination(defaultColumns []types.SortColumn, primary types.Sor if primary.Offset == nil { queryOrderColumns[0].Offset = column.Offset } + if len(primary.Table) == 0 { + queryOrderColumns[0].Table = column.Table + } continue } queryOrderColumns = append(queryOrderColumns, column) @@ -74,9 +77,9 @@ func applySortAndPagination(defaultColumns []types.SortColumn, primary types.Sor if cursor.IsReverse() { column.Desc = !column.Desc } - colOrder := goqu.C(column.Column).Asc() + colOrder := column.Expr().Asc() if column.Desc { - colOrder = goqu.C(column.Column).Desc() + colOrder = column.Expr().Desc() } queryOrder = append(queryOrder, colOrder) } @@ -87,15 +90,15 @@ func applySortAndPagination(defaultColumns []types.SortColumn, primary types.Sor // reverse order to nest conditions for i := len(queryOrderColumns) - 1; i >= 0; i-- { column := queryOrderColumns[i] - colWhere := goqu.C(column.Column).Gt(column.Offset) + colWhere := column.Expr().Gt(column.Offset) if column.Desc { - colWhere = goqu.C(column.Column).Lt(column.Offset) + colWhere = column.Expr().Lt(column.Offset) } if queryWhere == nil { queryWhere = colWhere } else { - queryWhere = goqu.And(goqu.C(column.Column).Eq(column.Offset), queryWhere) + queryWhere = goqu.And(column.Expr().Eq(column.Offset), queryWhere) queryWhere = goqu.Or(colWhere, queryWhere) } } diff --git a/backend/pkg/api/data_access/vdb_blocks.go b/backend/pkg/api/data_access/vdb_blocks.go index 409099ebc..f45ec2ac1 100644 --- a/backend/pkg/api/data_access/vdb_blocks.go +++ b/backend/pkg/api/data_access/vdb_blocks.go @@ -22,21 +22,6 @@ import ( "github.com/shopspring/decimal" ) -type table string - -// Stringer interface -func (t table) String() string { - return string(t) -} - -//func (t table) C(column string) exp.IdentifierExpression { -// return goqu.I(string(t) + "." + column) -//} - -func (t table) C(column string) string { - return string(t) + "." + column -} - func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, dashboardId t.VDBId, cursor string, colSort t.Sort[enums.VDBBlocksColumn], search string, limit uint64, protocolModes t.VDBProtocolModes) ([]t.VDBBlocksTableRow, *t.Paging, error) { // @DATA-ACCESS incorporate protocolModes @@ -48,9 +33,9 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das if err != nil { return nil, nil, err } - validators := table("validators") - blocks := table("blocks") - groups := table("goups") + validators := goqu.T("users_val_dashboards_validators").As("validators") + blocks := goqu.T("blocks") + groups := goqu.T("goups") // TODO @LuccaBitfly move validation to handler? if cursor != "" { @@ -72,31 +57,24 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das var filteredValidators []validatorGroup validatorsDs := goqu.Dialect("postgres"). Select( - validators.C("validator_index"), + "validator_index", ) if dashboardId.Validators == nil { validatorsDs = validatorsDs. - From( - goqu.T("users_val_dashboards_validators").As(validators), - ). - /*Select( - // TODO mustn't be here, can be done further down - validators.C("group_id"), - ).*/ - Where(goqu.Ex{validators.C("dashboard_id"): dashboardId.Id}) - + From(validators). + Where(validators.Col("dashboard_id").Eq(dashboardId.Id)) // apply search filters if searchIndex { - validatorsDs = validatorsDs.Where(goqu.Ex{validators.C("validator_index"): search}) + validatorsDs = validatorsDs.Where(validators.Col("validator_index").Eq(search)) } if searchGroup { validatorsDs = validatorsDs. InnerJoin(goqu.T("users_val_dashboards_groups").As(groups), goqu.On( - goqu.Ex{validators.C("dashboard_id"): groups.C("dashboard_id")}, - goqu.Ex{validators.C("group_id"): groups.C("id")}, + validators.Col("group_id").Eq(groups.Col("id")), + validators.Col("dashboard_id").Eq(groups.Col("dashboard_id")), )). Where( - goqu.L("LOWER(?)", groups.C("name")).Like(strings.Replace(search, "_", "\\_", -1) + "%"), + goqu.L("LOWER(?)", groups.Col("name")).Like(strings.Replace(search, "_", "\\_", -1) + "%"), ) } if searchPubkey { @@ -107,7 +85,7 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das } validatorsDs = validatorsDs. - Where(goqu.Ex{validators.C("validator_index"): index}) + Where(validators.Col("validator_index").Eq(index)) } } else { for _, validator := range dashboardId.Validators { @@ -126,7 +104,7 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das validatorsDs = validatorsDs. From( goqu.L("unnest(?)", pq.Array(filteredValidators)).As("validator_index"), - ).As(string(validators)) + ).As("validators") // TODO ? } if dashboardId.Validators == nil { @@ -180,51 +158,37 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das // Sorting and pagination if cursor is present defaultColumns := []t.SortColumn{ - {Column: enums.VDBBlocksColumns.Slot.ToString(), Desc: true, Offset: currentCursor.Slot}, + {Column: enums.VDBBlocksColumns.Slot.ToString(), Table: blocks.GetTable(), Desc: true, Offset: currentCursor.Slot}, } var offset any + var table string switch colSort.Column { case enums.VDBBlocksColumns.Proposer: offset = currentCursor.Proposer case enums.VDBBlocksColumns.Block: offset = currentCursor.Block + table = blocks.GetTable() case enums.VDBBlocksColumns.Status: offset = fmt.Sprintf("%d", currentCursor.Status) // type of 'status' column is text for some reason case enums.VDBBlocksColumns.ProposerReward: offset = currentCursor.Reward } - order, directions := applySortAndPagination(defaultColumns, t.SortColumn{Column: colSort.Column.ToString(), Desc: colSort.Desc, Offset: offset}, currentCursor.GenericCursor) + order, directions := applySortAndPagination(defaultColumns, t.SortColumn{Column: colSort.Column.ToString(), Table: table, Desc: colSort.Desc, Offset: offset}, currentCursor.GenericCursor) validatorsDs = validatorsDs.Order(order...) if directions != nil { validatorsDs = validatorsDs.Where(directions) } - // group id - if dashboardId.Validators == nil { - validatorsDs = validatorsDs.Select( - validators.C("group_id"), - ) - } else { - validatorsDs = validatorsDs.Select( - goqu.L("?", t.DefaultGroupId).As("group_id"), - ) - } - validatorsDs = validatorsDs. - Select( - blocks.C("proposer"), - blocks.C("epoch"), - blocks.C("slot"), - blocks.C("status"), - blocks.C("exec_block_number"), - blocks.C("graffiti_text"), - ). + InnerJoin(blocks, goqu.On( + blocks.Col("proposer").Eq(validators.Col("validator_index")), + )). LeftJoin(goqu.T("consensus_payloads").As("cp"), goqu.On( - goqu.Ex{blocks.C("slot"): goqu.I("cp.slot")}, + blocks.Col("slot").Eq(goqu.I("cp.slot")), )). LeftJoin(goqu.T("execution_payloads").As("ep"), goqu.On( - goqu.Ex{blocks.C("exec_block_hash"): goqu.I("ep.block_hash")}, + blocks.Col("exec_block_hash").Eq(goqu.I("ep.block_hash")), )). LeftJoin( // relay bribe deduplication; select most likely (=max) relay bribe value for each block @@ -232,101 +196,85 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das From(goqu.T("relays_blocks")). Select( goqu.I("relays_blocks.exec_block_hash"), + goqu.I("relays_blocks.proposer_fee_recipient"), goqu.MAX(goqu.I("relays_blocks.value")).As("value")). - // needed? TODO test - // Where(goqu.L("relays_blocks.exec_block_hash = blocks.exec_block_hash")). - GroupBy("exec_block_hash")).As("rb"), + GroupBy( + "exec_block_hash", + "proposer_fee_recipient", + )).As("rb"), goqu.On( - goqu.Ex{"rb.exec_block_hash": blocks.C("exec_block_hash")}, + goqu.I("rb.exec_block_hash").Eq(blocks.Col("exec_block_hash")), ), ). - Select( - goqu.COALESCE(goqu.I("rb.proposer_fee_recipient"), blocks.C("exec_fee_recipient")).As("fee_recipient"), + SelectAppend( + blocks.Col("epoch"), + blocks.Col("slot"), + blocks.Col("status"), + blocks.Col("exec_block_number"), + blocks.Col("graffiti_text"), + goqu.COALESCE(goqu.I("rb.proposer_fee_recipient"), blocks.Col("exec_fee_recipient")).As("fee_recipient"), goqu.COALESCE(goqu.L("rb.value / 1e18"), goqu.I("ep.fee_recipient_reward")).As("el_reward"), goqu.L("cp.cl_attestations_reward / 1e9 + cp.cl_sync_aggregate_reward / 1e9 + cp.cl_slashing_inclusion_reward / 1e9").As("cl_reward"), - ) - - // union scheduled blocks if present - // WIP - - params := make([]any, 0) - selectFields, where, orderBy, groupIdCol, sortColName := "", "", "", "", "" - cte := fmt.Sprintf(`WITH past_blocks AS (SELECT - %s - FROM blocks - `, selectFields) + ). + Limit(uint(limit + 1)) - if dashboardId.Validators == nil { - //cte += fmt.Sprintf(` - //INNER JOIN (%s) validators ON validators.validator_index = proposer`, filteredValidatorsQuery) - } else { - if len(where) == 0 { - where += `WHERE ` - } else { - where += `AND ` - } - where += `proposer = ANY($1) ` + // Group id + groupId := validators.Col("group_id") + if dashboardId.Validators != nil { + groupId = goqu.V(t.DefaultGroupId).As("group_id").GetAs() } + validatorsDs = validatorsDs.SelectAppend(groupId) - params = append(params, limit+1) - limitStr := fmt.Sprintf(` - LIMIT $%d - `, len(params)) + /* + if dashboardId.Validators == nil { + validatorsDs = validatorsDs.Select( + validators.Col("group_id"), + ) + } else { + validatorsDs = validatorsDs.Select( + goqu.L("?", t.DefaultGroupId).As("group_id"), + ) + }*/ - from := `past_blocks ` - selectStr := `SELECT * FROM ` + // union scheduled blocks if present + // WIP - query := selectStr + from + where + orderBy + limitStr - // supply scheduled proposals, if any + finalDs := validatorsDs if len(scheduledProposers) > 0 { - // distinct to filter out duplicates in an edge case (if dutiesInfo didn't update yet after a block was proposed, but the blocks table was) + scheduledDs := goqu.Dialect("postgres"). + From( + goqu.L("unnest(?, ?, ?) AS prov(validator_index, epoch, slot)", pq.Array(scheduledProposers), pq.Array(scheduledEpochs), pq.Array(scheduledSlots)), + ). + Select( + goqu.C("validator_index"), + goqu.C("epoch"), + goqu.C("slot"), + goqu.V("0").As("status"), + goqu.V(nil).As("exec_block_number"), + goqu.V(nil).As("fee_recipient"), + goqu.V(nil).As("el_reward"), + goqu.V(nil).As("cl_reward"), + goqu.V(nil).As("graffiti_text"), + ). + As("scheduled_blocks") + + // distinct + block number ordering to filter out duplicates in an edge case (if dutiesInfo didn't update yet after a block was proposed, but the blocks table was) // might be possible to remove this once the TODO in service_slot_viz.go:startSlotVizDataService is resolved - params = append(params, scheduledProposers) - params = append(params, scheduledEpochs) - params = append(params, scheduledSlots) - cte += fmt.Sprintf(`, - scheduled_blocks as ( - SELECT - prov.proposer, - prov.epoch, - prov.slot, - %s, - '0'::text AS status, - NULL::int AS exec_block_number, - ''::bytea AS fee_recipient, - NULL::float AS el_reward, - NULL::float AS cl_reward, - ''::text AS graffiti_text - FROM unnest($%d::int[], $%d::int[], $%d::int[]) AS prov(proposer, epoch, slot) - `, groupIdCol, len(params)-2, len(params)-1, len(params)) - if dashboardId.Validators == nil { - // add group id - cte += fmt.Sprintf(`INNER JOIN users_val_dashboards_validators validators - ON validators.dashboard_id = $1 - AND validators.validator_index = ANY($%d::int[]) - `, len(params)-2) - } - cte += `) ` - distinct := "slot" + finalDs = validatorsDs. + Union(scheduledDs). + Where(directions). + Order(order...). + OrderAppend(goqu.C("exec_block_number").Desc().NullsLast()). + Limit(uint(limit + 1)). + Distinct(blocks.Col("slot")) if !onlyPrimarySort { - distinct = sortColName + ", " + distinct + finalDs = finalDs. + Distinct(blocks.Col("slot"), blocks.Col("exec_block_number")) } - // keep all ordering, sorting etc - selectStr = `SELECT DISTINCT ON (` + distinct + `) * FROM ` - // encapsulate past blocks query to ensure performance - from = `( - ( ` + query + ` ) - UNION ALL - SELECT * FROM scheduled_blocks - ) as combined - ` - // make sure the distinct clause filters out the correct duplicated row (e.g. block=nil) - orderBy += `, exec_block_number NULLS LAST` - query = selectStr + from + where + orderBy + limitStr } var proposals []struct { - Proposer t.VDBValidator `db:"proposer"` + Proposer t.VDBValidator `db:"validator_index"` Group uint64 `db:"group_id"` Epoch uint64 `db:"epoch"` Slot uint64 `db:"slot"` @@ -341,11 +289,11 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das Reward decimal.Decimal } startTime := time.Now() - _, _, err = validatorsDs.Prepared(true).ToSQL() + query, args, err := finalDs.Prepared(true).ToSQL() if err != nil { return nil, nil, err } - err = d.alloyReader.SelectContext(ctx, &proposals, cte+query, params...) + err = d.alloyReader.SelectContext(ctx, &proposals, query, args...) log.Debugf("=== getting past blocks took %s", time.Since(startTime)) if err != nil { return nil, nil, err @@ -389,11 +337,11 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das } graffiti := proposal.GraffitiText data[i].Graffiti = &graffiti + block := uint64(proposal.Block.Int64) + data[i].Block = &block if proposal.Status == 3 { continue } - block := uint64(proposal.Block.Int64) - data[i].Block = &block var reward t.ClElValue[decimal.Decimal] if proposal.ElReward.Valid { rewardRecp := t.Address{ diff --git a/backend/pkg/api/enums/validator_dashboard_enums.go b/backend/pkg/api/enums/validator_dashboard_enums.go index 2646244e8..228b224dc 100644 --- a/backend/pkg/api/enums/validator_dashboard_enums.go +++ b/backend/pkg/api/enums/validator_dashboard_enums.go @@ -163,7 +163,7 @@ func (c VDBBlocksColumn) ToString() string { case VDBBlockSlot: return "slot" case VDBBlockBlock: - return "block" + return "exec_block_number" case VDBBlockStatus: return "status" case VDBBlockProposerReward: diff --git a/backend/pkg/api/types/data_access.go b/backend/pkg/api/types/data_access.go index 3b1d77c16..70961c5f2 100644 --- a/backend/pkg/api/types/data_access.go +++ b/backend/pkg/api/types/data_access.go @@ -1,8 +1,11 @@ package types import ( + "database/sql" "time" + "github.com/doug-martin/goqu/v9" + "github.com/doug-martin/goqu/v9/exp" "github.com/gobitfly/beaconchain/pkg/api/enums" "github.com/gobitfly/beaconchain/pkg/consapi/types" "github.com/gobitfly/beaconchain/pkg/monitoring/constants" @@ -24,11 +27,19 @@ type Sort[T enums.Enum] struct { type SortColumn struct { Column string + Table string // optional Desc bool // represents value from cursor Offset any } +func (s SortColumn) Expr() exp.IdentifierExpression { + if s.Table != "" { + return goqu.T(s.Table).Col(s.Column) + } + return goqu.C(s.Column) +} + type VDBIdPrimary int type VDBIdPublic string type VDBIdValidatorSet []VDBValidator @@ -167,7 +178,7 @@ type BlocksCursor struct { Proposer uint64 Slot uint64 // same as Age - Block uint64 + Block sql.NullInt64 Status uint64 Reward decimal.Decimal } diff --git a/frontend/components/dashboard/table/DashboardTableBlocks.vue b/frontend/components/dashboard/table/DashboardTableBlocks.vue index daa3a6e2f..01227c95f 100644 --- a/frontend/components/dashboard/table/DashboardTableBlocks.vue +++ b/frontend/components/dashboard/table/DashboardTableBlocks.vue @@ -52,7 +52,7 @@ const loadData = (query?: TableQueryParams) => { if (!query) { query = { limit: pageSize.value, - sort: 'block:desc', + sort: 'slot:desc', } } setQuery(query, true, true) From 327a88e4c8b90275a8b72d355269d17281f83cc3 Mon Sep 17 00:00:00 2001 From: remoterami <142154971+remoterami@users.noreply.github.com> Date: Mon, 21 Oct 2024 13:47:15 +0200 Subject: [PATCH 04/82] extended sorting/paging features, fixed blocks query --- backend/pkg/api/data_access/general.go | 19 +- backend/pkg/api/data_access/notifications.go | 38 +-- backend/pkg/api/data_access/vdb_blocks.go | 282 +++++++++--------- backend/pkg/api/enums/notifications_enums.go | 50 ++-- .../api/enums/validator_dashboard_enums.go | 25 +- backend/pkg/api/types/data_access.go | 16 +- 6 files changed, 223 insertions(+), 207 deletions(-) diff --git a/backend/pkg/api/data_access/general.go b/backend/pkg/api/data_access/general.go index 26d051dc9..d629d52df 100644 --- a/backend/pkg/api/data_access/general.go +++ b/backend/pkg/api/data_access/general.go @@ -62,9 +62,6 @@ func applySortAndPagination(defaultColumns []types.SortColumn, primary types.Sor if primary.Offset == nil { queryOrderColumns[0].Offset = column.Offset } - if len(primary.Table) == 0 { - queryOrderColumns[0].Table = column.Table - } continue } queryOrderColumns = append(queryOrderColumns, column) @@ -77,9 +74,9 @@ func applySortAndPagination(defaultColumns []types.SortColumn, primary types.Sor if cursor.IsReverse() { column.Desc = !column.Desc } - colOrder := column.Expr().Asc() + colOrder := column.Column.Asc() if column.Desc { - colOrder = column.Expr().Desc() + colOrder = column.Column.Desc() } queryOrder = append(queryOrder, colOrder) } @@ -90,15 +87,21 @@ func applySortAndPagination(defaultColumns []types.SortColumn, primary types.Sor // reverse order to nest conditions for i := len(queryOrderColumns) - 1; i >= 0; i-- { column := queryOrderColumns[i] - colWhere := column.Expr().Gt(column.Offset) + var colWhere exp.Expression + + // current convention is the psql default (ASC: nulls last, DESC: nulls first) + colWhere = goqu.Or(column.Column.Gt(column.Offset), column.Column.IsNull()) if column.Desc { - colWhere = column.Expr().Lt(column.Offset) + colWhere = column.Column.Lt(column.Offset) + if column.Offset == nil { + colWhere = goqu.Or(colWhere, column.Column.IsNull()) + } } if queryWhere == nil { queryWhere = colWhere } else { - queryWhere = goqu.And(column.Expr().Eq(column.Offset), queryWhere) + queryWhere = goqu.And(column.Column.Eq(column.Offset), queryWhere) queryWhere = goqu.Or(colWhere, queryWhere) } } diff --git a/backend/pkg/api/data_access/notifications.go b/backend/pkg/api/data_access/notifications.go index 776308329..7351de0c6 100644 --- a/backend/pkg/api/data_access/notifications.go +++ b/backend/pkg/api/data_access/notifications.go @@ -332,14 +332,14 @@ func (d *DataAccessService) GetDashboardNotifications(ctx context.Context, userI // sorting defaultColumns := []t.SortColumn{ - {Column: enums.NotificationsDashboardsColumns.Timestamp.ToString(), Desc: true, Offset: currentCursor.Epoch}, - {Column: enums.NotificationsDashboardsColumns.DashboardName.ToString(), Desc: false, Offset: currentCursor.DashboardName}, - {Column: enums.NotificationsDashboardsColumns.DashboardId.ToString(), Desc: false, Offset: currentCursor.DashboardId}, - {Column: enums.NotificationsDashboardsColumns.GroupName.ToString(), Desc: false, Offset: currentCursor.GroupName}, - {Column: enums.NotificationsDashboardsColumns.GroupId.ToString(), Desc: false, Offset: currentCursor.GroupId}, - {Column: enums.NotificationsDashboardsColumns.ChainId.ToString(), Desc: true, Offset: currentCursor.ChainId}, - } - order, directions := applySortAndPagination(defaultColumns, t.SortColumn{Column: colSort.Column.ToString(), Desc: colSort.Desc}, currentCursor.GenericCursor) + {Column: enums.NotificationsDashboardsColumns.Timestamp.ToExpr(), Desc: true, Offset: currentCursor.Epoch}, + {Column: enums.NotificationsDashboardsColumns.DashboardName.ToExpr(), Desc: false, Offset: currentCursor.DashboardName}, + {Column: enums.NotificationsDashboardsColumns.DashboardId.ToExpr(), Desc: false, Offset: currentCursor.DashboardId}, + {Column: enums.NotificationsDashboardsColumns.GroupName.ToExpr(), Desc: false, Offset: currentCursor.GroupName}, + {Column: enums.NotificationsDashboardsColumns.GroupId.ToExpr(), Desc: false, Offset: currentCursor.GroupId}, + {Column: enums.NotificationsDashboardsColumns.ChainId.ToExpr(), Desc: true, Offset: currentCursor.ChainId}, + } + order, directions := applySortAndPagination(defaultColumns, t.SortColumn{Column: colSort.Column.ToExpr(), Desc: colSort.Desc}, currentCursor.GenericCursor) unionQuery = unionQuery.Order(order...) if directions != nil { unionQuery = unionQuery.Where(directions) @@ -659,9 +659,9 @@ func (d *DataAccessService) GetMachineNotifications(ctx context.Context, userId // Sorting and limiting if cursor is present defaultColumns := []t.SortColumn{ - {Column: enums.NotificationsMachinesColumns.Timestamp.ToString(), Desc: true, Offset: currentCursor.Epoch}, - {Column: enums.NotificationsMachinesColumns.MachineId.ToString(), Desc: false, Offset: currentCursor.MachineId}, - {Column: enums.NotificationsMachinesColumns.EventType.ToString(), Desc: false, Offset: currentCursor.EventType}, + {Column: enums.NotificationsMachinesColumns.Timestamp.ToExpr(), Desc: true, Offset: currentCursor.Epoch}, + {Column: enums.NotificationsMachinesColumns.MachineId.ToExpr(), Desc: false, Offset: currentCursor.MachineId}, + {Column: enums.NotificationsMachinesColumns.EventType.ToExpr(), Desc: false, Offset: currentCursor.EventType}, } var offset interface{} switch colSort.Column { @@ -671,7 +671,7 @@ func (d *DataAccessService) GetMachineNotifications(ctx context.Context, userId offset = currentCursor.EventThreshold } - order, directions := applySortAndPagination(defaultColumns, t.SortColumn{Column: colSort.Column.ToString(), Desc: colSort.Desc, Offset: offset}, currentCursor.GenericCursor) + order, directions := applySortAndPagination(defaultColumns, t.SortColumn{Column: colSort.Column.ToExpr(), Desc: colSort.Desc, Offset: offset}, currentCursor.GenericCursor) ds = ds.Order(order...) if directions != nil { ds = ds.Where(directions) @@ -780,10 +780,10 @@ func (d *DataAccessService) GetClientNotifications(ctx context.Context, userId u // Sorting and limiting if cursor is present // Rows can be uniquely identified by (epoch, client) defaultColumns := []t.SortColumn{ - {Column: enums.NotificationsClientsColumns.Timestamp.ToString(), Desc: true, Offset: currentCursor.Epoch}, - {Column: enums.NotificationsClientsColumns.ClientName.ToString(), Desc: false, Offset: currentCursor.Client}, + {Column: enums.NotificationsClientsColumns.Timestamp.ToExpr(), Desc: true, Offset: currentCursor.Epoch}, + {Column: enums.NotificationsClientsColumns.ClientName.ToExpr(), Desc: false, Offset: currentCursor.Client}, } - order, directions := applySortAndPagination(defaultColumns, t.SortColumn{Column: colSort.Column.ToString(), Desc: colSort.Desc}, currentCursor.GenericCursor) + order, directions := applySortAndPagination(defaultColumns, t.SortColumn{Column: colSort.Column.ToExpr(), Desc: colSort.Desc}, currentCursor.GenericCursor) ds = ds.Order(order...) if directions != nil { ds = ds.Where(directions) @@ -1071,11 +1071,11 @@ func (d *DataAccessService) GetNetworkNotifications(ctx context.Context, userId // Sorting and limiting if cursor is present // Rows can be uniquely identified by (epoch, network, event_type) defaultColumns := []t.SortColumn{ - {Column: enums.NotificationNetworksColumns.Timestamp.ToString(), Desc: true, Offset: currentCursor.Epoch}, - {Column: enums.NotificationNetworksColumns.Network.ToString(), Desc: false, Offset: currentCursor.Network}, - {Column: enums.NotificationNetworksColumns.EventType.ToString(), Desc: false, Offset: currentCursor.EventType}, + {Column: enums.NotificationNetworksColumns.Timestamp.ToExpr(), Desc: true, Offset: currentCursor.Epoch}, + {Column: enums.NotificationNetworksColumns.Network.ToExpr(), Desc: false, Offset: currentCursor.Network}, + {Column: enums.NotificationNetworksColumns.EventType.ToExpr(), Desc: false, Offset: currentCursor.EventType}, } - order, directions := applySortAndPagination(defaultColumns, t.SortColumn{Column: colSort.Column.ToString(), Desc: colSort.Desc}, currentCursor.GenericCursor) + order, directions := applySortAndPagination(defaultColumns, t.SortColumn{Column: colSort.Column.ToExpr(), Desc: colSort.Desc}, currentCursor.GenericCursor) ds = ds.Order(order...) if directions != nil { ds = ds.Where(directions) diff --git a/backend/pkg/api/data_access/vdb_blocks.go b/backend/pkg/api/data_access/vdb_blocks.go index f45ec2ac1..cb9e6ef86 100644 --- a/backend/pkg/api/data_access/vdb_blocks.go +++ b/backend/pkg/api/data_access/vdb_blocks.go @@ -33,9 +33,6 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das if err != nil { return nil, nil, err } - validators := goqu.T("users_val_dashboards_validators").As("validators") - blocks := goqu.T("blocks") - groups := goqu.T("goups") // TODO @LuccaBitfly move validation to handler? if cursor != "" { @@ -48,27 +45,34 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das searchGroup := regexp.MustCompile(`^[a-zA-Z0-9_\-.\ ]+$`).MatchString(search) searchIndex := regexp.MustCompile(`^[0-9]+$`).MatchString(search) - // ------------------------------------- - // Goqu Query: Determine validators filtered by search + validators := goqu.T("users_val_dashboards_validators").As("validators") + blocks := goqu.T("blocks") + groups := goqu.T("groups") + type validatorGroup struct { Validator t.VDBValidator `db:"validator_index"` Group uint64 `db:"group_id"` } + + // ------------------------------------- + // Goqu Query to determine validators filtered by search + var filteredValidatorsDs *goqu.SelectDataset var filteredValidators []validatorGroup - validatorsDs := goqu.Dialect("postgres"). + + filteredValidatorsDs = goqu.Dialect("postgres"). Select( "validator_index", ) if dashboardId.Validators == nil { - validatorsDs = validatorsDs. + filteredValidatorsDs = filteredValidatorsDs. From(validators). Where(validators.Col("dashboard_id").Eq(dashboardId.Id)) // apply search filters if searchIndex { - validatorsDs = validatorsDs.Where(validators.Col("validator_index").Eq(search)) + filteredValidatorsDs = filteredValidatorsDs.Where(validators.Col("validator_index").Eq(search)) } if searchGroup { - validatorsDs = validatorsDs. + filteredValidatorsDs = filteredValidatorsDs. InnerJoin(goqu.T("users_val_dashboards_groups").As(groups), goqu.On( validators.Col("group_id").Eq(groups.Col("id")), validators.Col("dashboard_id").Eq(groups.Col("dashboard_id")), @@ -84,7 +88,7 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das return make([]t.VDBBlocksTableRow, 0), &t.Paging{}, nil } - validatorsDs = validatorsDs. + filteredValidatorsDs = filteredValidatorsDs. Where(validators.Col("validator_index").Eq(index)) } } else { @@ -101,86 +105,18 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das break } } - validatorsDs = validatorsDs. + filteredValidatorsDs = filteredValidatorsDs. From( goqu.L("unnest(?)", pq.Array(filteredValidators)).As("validator_index"), ).As("validators") // TODO ? } - if dashboardId.Validators == nil { - validatorsQuery, validatorsArgs, err := validatorsDs.Prepared(true).ToSQL() - if err != nil { - return nil, nil, err - } - if err = d.alloyReader.SelectContext(ctx, &filteredValidators, validatorsQuery, validatorsArgs...); err != nil { - return nil, nil, err - } - } - if len(filteredValidators) == 0 { - return make([]t.VDBBlocksTableRow, 0), &t.Paging{}, nil - } - // ------------------------------------- - // Gather scheduled blocks - // found in dutiesInfo; pass results to final query later and let db do the sorting etc - validatorSet := make(map[t.VDBValidator]bool) - for _, v := range filteredValidators { - validatorSet[v.Validator] = true - } - var scheduledProposers []t.VDBValidator - var scheduledEpochs []uint64 - var scheduledSlots []uint64 - // don't need if requested slots are in the past - latestSlot := cache.LatestSlot.Get() - onlyPrimarySort := colSort.Column == enums.VDBBlockSlot || colSort.Column == enums.VDBBlockBlock - if !onlyPrimarySort || !currentCursor.IsValid() || - currentCursor.Slot > latestSlot+1 && currentCursor.Reverse != colSort.Desc || - currentCursor.Slot < latestSlot+1 && currentCursor.Reverse == colSort.Desc { - dutiesInfo, err := d.services.GetCurrentDutiesInfo() - if err == nil { - for slot, vali := range dutiesInfo.PropAssignmentsForSlot { - // only gather scheduled slots - if _, ok := dutiesInfo.SlotStatus[slot]; ok { - continue - } - // only gather slots scheduled for our validators - if _, ok := validatorSet[vali]; !ok { - continue - } - scheduledProposers = append(scheduledProposers, dutiesInfo.PropAssignmentsForSlot[slot]) - scheduledEpochs = append(scheduledEpochs, slot/utils.Config.Chain.ClConfig.SlotsPerEpoch) - scheduledSlots = append(scheduledSlots, slot) - } - } else { - log.Debugf("duties info not available, skipping scheduled slots: %s", err) - } - } - - // Sorting and pagination if cursor is present - defaultColumns := []t.SortColumn{ - {Column: enums.VDBBlocksColumns.Slot.ToString(), Table: blocks.GetTable(), Desc: true, Offset: currentCursor.Slot}, - } - var offset any - var table string - switch colSort.Column { - case enums.VDBBlocksColumns.Proposer: - offset = currentCursor.Proposer - case enums.VDBBlocksColumns.Block: - offset = currentCursor.Block - table = blocks.GetTable() - case enums.VDBBlocksColumns.Status: - offset = fmt.Sprintf("%d", currentCursor.Status) // type of 'status' column is text for some reason - case enums.VDBBlocksColumns.ProposerReward: - offset = currentCursor.Reward - } + // Constuct final query + var blocksDs *goqu.SelectDataset - order, directions := applySortAndPagination(defaultColumns, t.SortColumn{Column: colSort.Column.ToString(), Table: table, Desc: colSort.Desc, Offset: offset}, currentCursor.GenericCursor) - validatorsDs = validatorsDs.Order(order...) - if directions != nil { - validatorsDs = validatorsDs.Where(directions) - } - - validatorsDs = validatorsDs. + // 1. Tables + blocksDs = filteredValidatorsDs. InnerJoin(blocks, goqu.On( blocks.Col("proposer").Eq(validators.Col("validator_index")), )). @@ -205,7 +141,10 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das goqu.On( goqu.I("rb.exec_block_hash").Eq(blocks.Col("exec_block_hash")), ), - ). + ) + + // 2. Selects + blocksDs = blocksDs. SelectAppend( blocks.Col("epoch"), blocks.Col("slot"), @@ -215,64 +154,128 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das goqu.COALESCE(goqu.I("rb.proposer_fee_recipient"), blocks.Col("exec_fee_recipient")).As("fee_recipient"), goqu.COALESCE(goqu.L("rb.value / 1e18"), goqu.I("ep.fee_recipient_reward")).As("el_reward"), goqu.L("cp.cl_attestations_reward / 1e9 + cp.cl_sync_aggregate_reward / 1e9 + cp.cl_slashing_inclusion_reward / 1e9").As("cl_reward"), - ). - Limit(uint(limit + 1)) + ) - // Group id groupId := validators.Col("group_id") if dashboardId.Validators != nil { groupId = goqu.V(t.DefaultGroupId).As("group_id").GetAs() } - validatorsDs = validatorsDs.SelectAppend(groupId) + blocksDs = blocksDs.SelectAppend(groupId) - /* - if dashboardId.Validators == nil { - validatorsDs = validatorsDs.Select( - validators.Col("group_id"), - ) - } else { - validatorsDs = validatorsDs.Select( - goqu.L("?", t.DefaultGroupId).As("group_id"), - ) - }*/ + // 3. Sorting and pagination + defaultColumns := []t.SortColumn{ + {Column: enums.VDBBlocksColumns.Slot.ToExpr(), Desc: true, Offset: currentCursor.Slot}, + } + var offset any + switch colSort.Column { + case enums.VDBBlocksColumns.Proposer: + offset = currentCursor.Proposer + case enums.VDBBlocksColumns.Block: + offset = currentCursor.Block + if !currentCursor.Block.Valid { + offset = nil + } + case enums.VDBBlocksColumns.Status: + offset = fmt.Sprintf("%d", currentCursor.Status) // type of 'status' column is text for some reason + case enums.VDBBlocksColumns.ProposerReward: + offset = currentCursor.Reward + } - // union scheduled blocks if present - // WIP + order, directions := applySortAndPagination(defaultColumns, t.SortColumn{Column: colSort.Column.ToExpr(), Desc: colSort.Desc, Offset: offset}, currentCursor.GenericCursor) + blocksDs = goqu.From(blocksDs). // encapsulate so we can use selected fields + Order(order...) + if directions != nil { + blocksDs = blocksDs.Where(directions) + } - finalDs := validatorsDs - if len(scheduledProposers) > 0 { - scheduledDs := goqu.Dialect("postgres"). - From( - goqu.L("unnest(?, ?, ?) AS prov(validator_index, epoch, slot)", pq.Array(scheduledProposers), pq.Array(scheduledEpochs), pq.Array(scheduledSlots)), - ). - Select( - goqu.C("validator_index"), - goqu.C("epoch"), - goqu.C("slot"), - goqu.V("0").As("status"), - goqu.V(nil).As("exec_block_number"), - goqu.V(nil).As("fee_recipient"), - goqu.V(nil).As("el_reward"), - goqu.V(nil).As("cl_reward"), - goqu.V(nil).As("graffiti_text"), - ). - As("scheduled_blocks") + // 4. Limit + blocksDs = blocksDs.Limit(uint(limit + 1)) + + // 5. Gather and supply scheduled blocks to let db do the sorting etc + latestSlot := cache.LatestSlot.Get() + onlyPrimarySort := colSort.Column == enums.VDBBlockSlot + if !(onlyPrimarySort || colSort.Column == enums.VDBBlockBlock) || !currentCursor.IsValid() || + currentCursor.Slot > latestSlot+1 && currentCursor.Reverse != colSort.Desc || + currentCursor.Slot < latestSlot+1 && currentCursor.Reverse == colSort.Desc { + dutiesInfo, err := d.services.GetCurrentDutiesInfo() + if err == nil { + if dashboardId.Validators == nil { + // fetch filtered validators if not done yet + validatorsQuery, validatorsArgs, err := filteredValidatorsDs.Prepared(true).ToSQL() + if err != nil { + return nil, nil, err + } + if err = d.alloyReader.SelectContext(ctx, &filteredValidators, validatorsQuery, validatorsArgs...); err != nil { + return nil, nil, err + } + } + if len(filteredValidators) == 0 { + return make([]t.VDBBlocksTableRow, 0), &t.Paging{}, nil + } + + validatorSet := make(map[t.VDBValidator]bool) + for _, v := range filteredValidators { + validatorSet[v.Validator] = true + } + var scheduledProposers []t.VDBValidator + var scheduledEpochs []uint64 + var scheduledSlots []uint64 + // don't need if requested slots are in the past + for slot, vali := range dutiesInfo.PropAssignmentsForSlot { + // only gather scheduled slots + if _, ok := dutiesInfo.SlotStatus[slot]; ok { + continue + } + // only gather slots scheduled for our validators + if _, ok := validatorSet[vali]; !ok { + continue + } + scheduledProposers = append(scheduledProposers, dutiesInfo.PropAssignmentsForSlot[slot]) + scheduledEpochs = append(scheduledEpochs, slot/utils.Config.Chain.ClConfig.SlotsPerEpoch) + scheduledSlots = append(scheduledSlots, slot) + } - // distinct + block number ordering to filter out duplicates in an edge case (if dutiesInfo didn't update yet after a block was proposed, but the blocks table was) - // might be possible to remove this once the TODO in service_slot_viz.go:startSlotVizDataService is resolved - finalDs = validatorsDs. - Union(scheduledDs). - Where(directions). - Order(order...). - OrderAppend(goqu.C("exec_block_number").Desc().NullsLast()). - Limit(uint(limit + 1)). - Distinct(blocks.Col("slot")) - if !onlyPrimarySort { - finalDs = finalDs. - Distinct(blocks.Col("slot"), blocks.Col("exec_block_number")) + scheduledDs := goqu.Dialect("postgres"). + From( + goqu.L("unnest(?::int[], ?::int[], ?::int[]) AS prov(validator_index, epoch, slot)", pq.Array(scheduledProposers), pq.Array(scheduledEpochs), pq.Array(scheduledSlots)), + ). + Select( + goqu.C("validator_index"), + goqu.C("epoch"), + goqu.C("slot"), + goqu.V("0").As("status"), + goqu.V(nil).As("exec_block_number"), + goqu.V(nil).As("fee_recipient"), + goqu.V(nil).As("el_reward"), + goqu.V(nil).As("cl_reward"), + goqu.V(nil).As("graffiti_text"), + goqu.V(t.DefaultGroupId).As("group_id"), + ). + As("scheduled_blocks") + + // Supply to result query + // distinct + block number ordering to filter out duplicates in an edge case (if dutiesInfo didn't update yet after a block was proposed, but the blocks table was) + // might be possible to remove this once the TODO in service_slot_viz.go:startSlotVizDataService is resolved + blocksDs = goqu.Dialect("Postgres"). + From(blocksDs.Union(scheduledDs)). // wrap union to apply order + Order(order...). + OrderAppend(goqu.C("exec_block_number").Desc().NullsLast()). + Limit(uint(limit + 1)). + Distinct(enums.VDBBlocksColumns.Slot.ToExpr()) + if directions != nil { + blocksDs = blocksDs.Where(directions) + } + if !onlyPrimarySort { + blocksDs = blocksDs. + Distinct(colSort.Column.ToExpr(), enums.VDBBlocksColumns.Slot.ToExpr()) + } + } else { + log.Warnf("Error getting scheduled proposals, DutiesInfo not available in Redis: %s", err) } } + // ------------------------------------- + // Execute query var proposals []struct { Proposer t.VDBValidator `db:"validator_index"` Group uint64 `db:"group_id"` @@ -283,13 +286,13 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das FeeRecipient []byte `db:"fee_recipient"` ElReward decimal.NullDecimal `db:"el_reward"` ClReward decimal.NullDecimal `db:"cl_reward"` - GraffitiText string `db:"graffiti_text"` + GraffitiText sql.NullString `db:"graffiti_text"` // for cursor only Reward decimal.Decimal } startTime := time.Now() - query, args, err := finalDs.Prepared(true).ToSQL() + query, args, err := blocksDs.Prepared(true).ToSQL() if err != nil { return nil, nil, err } @@ -301,6 +304,9 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das if len(proposals) == 0 { return make([]t.VDBBlocksTableRow, 0), &t.Paging{}, nil } + + // ------------------------------------- + // Prepare result moreDataFlag := len(proposals) > int(limit) if moreDataFlag { proposals = proposals[:len(proposals)-1] @@ -335,10 +341,14 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das if proposal.Status == 0 || proposal.Status == 2 { continue } - graffiti := proposal.GraffitiText - data[i].Graffiti = &graffiti - block := uint64(proposal.Block.Int64) - data[i].Block = &block + if proposal.GraffitiText.Valid { + graffiti := proposal.GraffitiText.String + data[i].Graffiti = &graffiti + } + if proposal.Block.Valid { + block := uint64(proposal.Block.Int64) + data[i].Block = &block + } if proposal.Status == 3 { continue } diff --git a/backend/pkg/api/enums/notifications_enums.go b/backend/pkg/api/enums/notifications_enums.go index 1fb78529e..9af65cb1a 100644 --- a/backend/pkg/api/enums/notifications_enums.go +++ b/backend/pkg/api/enums/notifications_enums.go @@ -1,5 +1,7 @@ package enums +import "github.com/doug-martin/goqu/v9" + // ------------------------------------------------------------ // Notifications Dashboard Table Columns @@ -34,22 +36,22 @@ func (NotificationDashboardsColumn) NewFromString(s string) NotificationDashboar } // internal use, used to map to query column names -func (c NotificationDashboardsColumn) ToString() string { +func (c NotificationDashboardsColumn) ToExpr() OrderableSortable { switch c { case NotificationDashboardChainId: - return "chain_id" + return goqu.C("chain_id") case NotificationDashboardEpoch: - return "epoch" + return goqu.C("epoch") case NotificationDashboardDashboardName: - return "dashboard_name" + return goqu.C("dashboard_name") case NotificationDashboardDashboardId: - return "dashboard_id" + return goqu.C("dashboard_id") case NotificationDashboardGroupName: - return "group_name" + return goqu.C("group_name") case NotificationDashboardGroupId: - return "group_id" + return goqu.C("group_id") default: - return "" + return nil } } @@ -104,20 +106,20 @@ func (NotificationMachinesColumn) NewFromString(s string) NotificationMachinesCo } // internal use, used to map to query column names -func (c NotificationMachinesColumn) ToString() string { +func (c NotificationMachinesColumn) ToExpr() OrderableSortable { switch c { case NotificationMachineId: - return "machine_id" + return goqu.C("machine_id") case NotificationMachineName: - return "machine_name" + return goqu.C("machine_name") case NotificationMachineThreshold: - return "threshold" + return goqu.C("threshold") case NotificationMachineEventType: - return "event_type" + return goqu.C("event_type") case NotificationMachineTimestamp: - return "epoch" + return goqu.C("epoch") default: - return "" + return nil } } @@ -163,14 +165,14 @@ func (NotificationClientsColumn) NewFromString(s string) NotificationClientsColu } // internal use, used to map to query column names -func (c NotificationClientsColumn) ToString() string { +func (c NotificationClientsColumn) ToExpr() OrderableSortable { switch c { case NotificationClientName: - return "client_name" + return goqu.C("client_name") case NotificationClientTimestamp: - return "epoch" + return goqu.C("epoch") default: - return "" + return nil } } @@ -251,16 +253,16 @@ func (NotificationNetworksColumn) NewFromString(s string) NotificationNetworksCo } // internal use, used to map to query column names -func (c NotificationNetworksColumn) ToString() string { +func (c NotificationNetworksColumn) ToExpr() OrderableSortable { switch c { case NotificationNetworkTimestamp: - return "epoch" + return goqu.C("epoch") case NotificationNetworkNetwork: - return "network" + return goqu.C("network") case NotificationNetworkEventType: - return "event_type" + return goqu.C("event_type") default: - return "" + return nil } } diff --git a/backend/pkg/api/enums/validator_dashboard_enums.go b/backend/pkg/api/enums/validator_dashboard_enums.go index 228b224dc..c241ef98e 100644 --- a/backend/pkg/api/enums/validator_dashboard_enums.go +++ b/backend/pkg/api/enums/validator_dashboard_enums.go @@ -1,5 +1,10 @@ package enums +import ( + "github.com/doug-martin/goqu/v9" + "github.com/doug-martin/goqu/v9/exp" +) + // ---------------- // Validator Dashboard Summary Table @@ -156,20 +161,26 @@ func (VDBBlocksColumn) NewFromString(s string) VDBBlocksColumn { } } -func (c VDBBlocksColumn) ToString() string { +type OrderableSortable interface { + exp.Orderable + exp.Comparable + exp.Isable +} + +func (c VDBBlocksColumn) ToExpr() OrderableSortable { switch c { case VDBBlockProposer: - return "proposer" + return goqu.C("validator_index") case VDBBlockSlot: - return "slot" + return goqu.C("slot") case VDBBlockBlock: - return "exec_block_number" + return goqu.C("exec_block_number") case VDBBlockStatus: - return "status" + return goqu.C("status") case VDBBlockProposerReward: - return "reward" + return goqu.L("el_reward + cl_reward") default: - return "" + return nil } } diff --git a/backend/pkg/api/types/data_access.go b/backend/pkg/api/types/data_access.go index 70961c5f2..6bcf30472 100644 --- a/backend/pkg/api/types/data_access.go +++ b/backend/pkg/api/types/data_access.go @@ -4,8 +4,6 @@ import ( "database/sql" "time" - "github.com/doug-martin/goqu/v9" - "github.com/doug-martin/goqu/v9/exp" "github.com/gobitfly/beaconchain/pkg/api/enums" "github.com/gobitfly/beaconchain/pkg/consapi/types" "github.com/gobitfly/beaconchain/pkg/monitoring/constants" @@ -26,18 +24,10 @@ type Sort[T enums.Enum] struct { } type SortColumn struct { - Column string - Table string // optional + // defaults + Column enums.OrderableSortable Desc bool - // represents value from cursor - Offset any -} - -func (s SortColumn) Expr() exp.IdentifierExpression { - if s.Table != "" { - return goqu.T(s.Table).Col(s.Column) - } - return goqu.C(s.Column) + Offset any // nil to indicate null value } type VDBIdPrimary int From 5013243850ad320ec6db4f7e1f62cfe76c15f214 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Mon, 21 Oct 2024 12:00:06 +0000 Subject: [PATCH 05/82] feat(notifications): add webhook config retrieval --- backend/pkg/notification/queuing.go | 99 +++++++++++++++++++++++------ 1 file changed, 79 insertions(+), 20 deletions(-) diff --git a/backend/pkg/notification/queuing.go b/backend/pkg/notification/queuing.go index 90cb78bca..7a86e0ea2 100644 --- a/backend/pkg/notification/queuing.go +++ b/backend/pkg/notification/queuing.go @@ -19,6 +19,7 @@ import ( "github.com/gobitfly/beaconchain/pkg/commons/types" "github.com/gobitfly/beaconchain/pkg/commons/utils" "github.com/jmoiron/sqlx" + "github.com/lib/pq" "golang.org/x/text/cases" "golang.org/x/text/language" ) @@ -583,29 +584,87 @@ func QueuePushNotification(epoch uint64, notificationsByUserID types.Notificatio } func QueueWebhookNotifications(notificationsByUserID types.NotificationsPerUserId, tx *sqlx.Tx) error { + var webhooks []types.UserWebhook + userIds := slices.Collect(maps.Keys(notificationsByUserID)) + err := db.FrontendWriterDB.Select(&webhooks, ` + SELECT + id, + user_id, + url, + retries, + event_names, + last_sent, + destination + FROM + users_webhooks + WHERE + user_id = $1 AND user_id NOT IN (SELECT user_id from users_notification_channels WHERE active = false and channel = $2) + `, pq.Array(userIds), types.WebhookNotificationChannel) + + if err != nil { + return fmt.Errorf("error quering users_webhooks, err: %w", err) + } + webhooksMap := make(map[uint64][]types.UserWebhook) + for _, w := range webhooks { + if _, exists := webhooksMap[w.UserID]; !exists { + webhooksMap[w.UserID] = make([]types.UserWebhook, 0) + } + webhooksMap[w.UserID] = append(webhooksMap[w.UserID], w) + } + + // now fetch the webhooks for each dashboard config + var dashboardWebhooks []struct { + UserId types.UserId `db:"user_id"` + DashboardID types.DashboardId `db:"dashboard_id"` + GroupId types.DashboardGroupId `db:"id"` + WebhookTarget string `db:"webhook_target"` + WebhookFormat string `db:"webhook_format"` + WebhookRetries uint64 `db:"webhook_retries"` + } + + err = db.ReaderDb.Select(&dashboardWebhooks, ` + SELECT + users_val_dashboards_groups.id, + dashboard_id, + webhook_target, + webhook_format, + webhook_retries + FROM users_val_dashboards_groups + LEFT JOIN users_val_dashboards ON users_val_dashboards_groups.dashboard_id = users_val_dashboards.id + WHERE users_val_dashboards.user_id = ANY($1) + AND webhook_target IS NOT NULL + AND webhook_format IS NOT NULL; + `, pq.Array(userIds)) + if err != nil { + return fmt.Errorf("error quering users_val_dashboards_groups, err: %w", err) + } + dashboardWebhookMap := make(map[types.UserId]map[types.DashboardId]map[types.DashboardGroupId]types.UserWebhook) + for _, w := range dashboardWebhooks { + if _, exists := dashboardWebhookMap[w.UserId]; !exists { + dashboardWebhookMap[w.UserId] = make(map[types.DashboardId]map[types.DashboardGroupId]types.UserWebhook) + } + if _, exists := dashboardWebhookMap[w.UserId][w.DashboardID]; !exists { + dashboardWebhookMap[w.UserId][w.DashboardID] = make(map[types.DashboardGroupId]types.UserWebhook) + } + + uw := types.UserWebhook{ + UserID: uint64(w.UserId), + Url: w.WebhookTarget, + Retries: w.WebhookRetries, + } + if w.WebhookFormat == "discord" { + uw.Destination = sql.NullString{String: "webhook_discord", Valid: true} + } else { + uw.Destination = sql.NullString{String: "webhook", Valid: true} + } + dashboardWebhookMap[w.UserId][w.DashboardID][w.GroupId] = uw + } + for userID, userNotifications := range notificationsByUserID { - var webhooks []types.UserWebhook - err := db.FrontendWriterDB.Select(&webhooks, ` - SELECT - id, - user_id, - url, - retries, - event_names, - last_sent, - destination - FROM - users_webhooks - WHERE - user_id = $1 AND user_id NOT IN (SELECT user_id from users_notification_channels WHERE active = false and channel = $2) - `, userID, types.WebhookNotificationChannel) - // continue if the user does not have a webhook - if err == sql.ErrNoRows { + webhooks, exists := webhooksMap[uint64(userID)] + if !exists { continue } - if err != nil { - return fmt.Errorf("error quering users_webhooks, err: %w", err) - } // webhook => [] notifications discordNotifMap := make(map[uint64][]types.TransitDiscordContent) notifs := make([]types.TransitWebhook, 0) From 3d4b61409466bfe9428f4b3299a373dc74ef9913 Mon Sep 17 00:00:00 2001 From: remoterami <142154971+remoterami@users.noreply.github.com> Date: Mon, 21 Oct 2024 14:11:46 +0200 Subject: [PATCH 06/82] goqu quirck workaround --- backend/pkg/api/data_access/vdb_blocks.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/pkg/api/data_access/vdb_blocks.go b/backend/pkg/api/data_access/vdb_blocks.go index cb9e6ef86..00495e265 100644 --- a/backend/pkg/api/data_access/vdb_blocks.go +++ b/backend/pkg/api/data_access/vdb_blocks.go @@ -162,7 +162,10 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das } blocksDs = blocksDs.SelectAppend(groupId) - // 3. Sorting and pagination + // 3. Limit + blocksDs = blocksDs.Limit(uint(limit + 1)) + + // 4. Sorting and pagination defaultColumns := []t.SortColumn{ {Column: enums.VDBBlocksColumns.Slot.ToExpr(), Desc: true, Offset: currentCursor.Slot}, } @@ -188,9 +191,6 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das blocksDs = blocksDs.Where(directions) } - // 4. Limit - blocksDs = blocksDs.Limit(uint(limit + 1)) - // 5. Gather and supply scheduled blocks to let db do the sorting etc latestSlot := cache.LatestSlot.Get() onlyPrimarySort := colSort.Column == enums.VDBBlockSlot From 7a208e1f98f87c7f44638931404bdd7c5a10ba1d Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Tue, 22 Oct 2024 07:22:14 +0000 Subject: [PATCH 07/82] feat(notifications): improve webhook handling --- backend/pkg/api/data_access/notifications.go | 2 +- backend/pkg/commons/types/frontend.go | 20 +- backend/pkg/notification/queuing.go | 206 +++++++++---------- 3 files changed, 111 insertions(+), 117 deletions(-) diff --git a/backend/pkg/api/data_access/notifications.go b/backend/pkg/api/data_access/notifications.go index 2bf6d619f..5532c27d7 100644 --- a/backend/pkg/api/data_access/notifications.go +++ b/backend/pkg/api/data_access/notifications.go @@ -81,7 +81,7 @@ const ( ValidatorDashboardEventPrefix string = "vdb" AccountDashboardEventPrefix string = "adb" - DiscordWebhookFormat string = "discord" + DiscordWebhookFormat string = "webhook_discord" GroupOfflineThresholdDefault float64 = 0.1 MaxCollateralThresholdDefault float64 = 1.0 diff --git a/backend/pkg/commons/types/frontend.go b/backend/pkg/commons/types/frontend.go index 0ef4bde69..bb51cf0e6 100644 --- a/backend/pkg/commons/types/frontend.go +++ b/backend/pkg/commons/types/frontend.go @@ -636,15 +636,17 @@ type Email struct { } type UserWebhook struct { - ID uint64 `db:"id" json:"id"` - UserID uint64 `db:"user_id" json:"-"` - Url string `db:"url" json:"url"` - Retries uint64 `db:"retries" json:"retries"` - LastSent sql.NullTime `db:"last_sent" json:"lastRetry"` - Response sql.NullString `db:"response" json:"response"` - Request sql.NullString `db:"request" json:"request"` - Destination sql.NullString `db:"destination" json:"destination"` - EventNames pq.StringArray `db:"event_names" json:"-"` + ID uint64 `db:"id" json:"id"` + UserID uint64 `db:"user_id" json:"-"` + Url string `db:"url" json:"url"` + Retries uint64 `db:"retries" json:"retries"` + LastSent sql.NullTime `db:"last_sent" json:"lastRetry"` + Response sql.NullString `db:"response" json:"response"` + Request sql.NullString `db:"request" json:"request"` + Destination sql.NullString `db:"destination" json:"destination"` + EventNames pq.StringArray `db:"event_names" json:"-"` + DashboardId uint64 `db:"dashboard_id" json:"dashboardId"` + DashboardGroupId uint64 `db:"dashboard_group_id" json:"dashboardGroupId"` } type UserWebhookSubscriptions struct { diff --git a/backend/pkg/notification/queuing.go b/backend/pkg/notification/queuing.go index 7a86e0ea2..091acdfbb 100644 --- a/backend/pkg/notification/queuing.go +++ b/backend/pkg/notification/queuing.go @@ -3,7 +3,6 @@ package notification import ( "bytes" "compress/gzip" - "database/sql" "encoding/gob" "fmt" "html/template" @@ -613,22 +612,14 @@ func QueueWebhookNotifications(notificationsByUserID types.NotificationsPerUserI } // now fetch the webhooks for each dashboard config - var dashboardWebhooks []struct { - UserId types.UserId `db:"user_id"` - DashboardID types.DashboardId `db:"dashboard_id"` - GroupId types.DashboardGroupId `db:"id"` - WebhookTarget string `db:"webhook_target"` - WebhookFormat string `db:"webhook_format"` - WebhookRetries uint64 `db:"webhook_retries"` - } - - err = db.ReaderDb.Select(&dashboardWebhooks, ` + err = db.ReaderDb.Select(&webhooks, ` SELECT - users_val_dashboards_groups.id, - dashboard_id, - webhook_target, - webhook_format, - webhook_retries + users_val_dashboards_groups.id AS dashboard_group_id, + dashboard_id AS dashboard_id, + webhook_target AS url, + COALESCE(webhook_format, "webhook") AS destination, + webhook_retries AS retries, + webhook_last_sent AS last_sent FROM users_val_dashboards_groups LEFT JOIN users_val_dashboards ON users_val_dashboards_groups.dashboard_id = users_val_dashboards.id WHERE users_val_dashboards.user_id = ANY($1) @@ -639,25 +630,15 @@ func QueueWebhookNotifications(notificationsByUserID types.NotificationsPerUserI return fmt.Errorf("error quering users_val_dashboards_groups, err: %w", err) } dashboardWebhookMap := make(map[types.UserId]map[types.DashboardId]map[types.DashboardGroupId]types.UserWebhook) - for _, w := range dashboardWebhooks { - if _, exists := dashboardWebhookMap[w.UserId]; !exists { - dashboardWebhookMap[w.UserId] = make(map[types.DashboardId]map[types.DashboardGroupId]types.UserWebhook) + for _, w := range webhooks { + if _, exists := dashboardWebhookMap[types.UserId(w.UserID)]; !exists { + dashboardWebhookMap[types.UserId(w.UserID)] = make(map[types.DashboardId]map[types.DashboardGroupId]types.UserWebhook) } - if _, exists := dashboardWebhookMap[w.UserId][w.DashboardID]; !exists { - dashboardWebhookMap[w.UserId][w.DashboardID] = make(map[types.DashboardGroupId]types.UserWebhook) + if _, exists := dashboardWebhookMap[types.UserId(w.UserID)][types.DashboardId(w.DashboardId)]; !exists { + dashboardWebhookMap[types.UserId(w.UserID)][types.DashboardId(w.DashboardId)] = make(map[types.DashboardGroupId]types.UserWebhook) } - uw := types.UserWebhook{ - UserID: uint64(w.UserId), - Url: w.WebhookTarget, - Retries: w.WebhookRetries, - } - if w.WebhookFormat == "discord" { - uw.Destination = sql.NullString{String: "webhook_discord", Valid: true} - } else { - uw.Destination = sql.NullString{String: "webhook", Valid: true} - } - dashboardWebhookMap[w.UserId][w.DashboardID][w.GroupId] = uw + dashboardWebhookMap[types.UserId(w.UserID)][types.DashboardId(w.DashboardId)][types.DashboardGroupId(w.DashboardGroupId)] = w } for userID, userNotifications := range notificationsByUserID { @@ -671,90 +652,101 @@ func QueueWebhookNotifications(notificationsByUserID types.NotificationsPerUserI // send the notifications to each registered webhook for _, w := range webhooks { for dashboardId, notificationsPerDashboard := range userNotifications { - if dashboardId != 0 { // disable webhooks for dashboard notifications for now - continue - } for _, notificationsPerGroup := range notificationsPerDashboard { - for event, notifications := range notificationsPerGroup { - // check if the webhook is subscribed to the type of event - eventSubscribed := slices.Contains(w.EventNames, string(event)) - - if eventSubscribed { - if len(notifications) > 0 { - // reset Retries - if w.Retries > 5 && w.LastSent.Valid && w.LastSent.Time.Add(time.Hour).Before(time.Now()) { - _, err = db.FrontendWriterDB.Exec(`UPDATE users_webhooks SET retries = 0 WHERE id = $1;`, w.ID) - if err != nil { - log.Error(err, "error updating users_webhooks table; setting retries to zero", 0) + if dashboardId != 0 { // disable webhooks for dashboard notifications for now + // retrieve the associated webhook config from the map + if _, exists := dashboardWebhookMap[types.UserId(userID)]; !exists { + continue + } + if _, exists := dashboardWebhookMap[types.UserId(userID)][types.DashboardId(dashboardId)]; !exists { + continue + } + if _, exists := dashboardWebhookMap[types.UserId(userID)][types.DashboardId(dashboardId)][0]; !exists { + continue + } + w = dashboardWebhookMap[types.UserId(userID)][types.DashboardId(dashboardId)][0] + } else { + for event, notifications := range notificationsPerGroup { + // check if the webhook is subscribed to the type of event + eventSubscribed := slices.Contains(w.EventNames, string(event)) + + if eventSubscribed { + if len(notifications) > 0 { + // reset Retries + if w.Retries > 5 && w.LastSent.Valid && w.LastSent.Time.Add(time.Hour).Before(time.Now()) { + _, err = db.FrontendWriterDB.Exec(`UPDATE users_webhooks SET retries = 0 WHERE id = $1;`, w.ID) + if err != nil { + log.Error(err, "error updating users_webhooks table; setting retries to zero", 0) + continue + } + } else if w.Retries > 5 && !w.LastSent.Valid { + log.Warnf("webhook '%v' has more than 5 retries and does not have a valid last_sent timestamp", w.Url) continue } - } else if w.Retries > 5 && !w.LastSent.Valid { - log.Warnf("webhook '%v' has more than 5 retries and does not have a valid last_sent timestamp", w.Url) - continue - } - - if w.Retries >= 5 { - // early return - continue - } - } - for _, n := range notifications { - if w.Destination.Valid && w.Destination.String == "webhook_discord" { - if _, exists := discordNotifMap[w.ID]; !exists { - discordNotifMap[w.ID] = make([]types.TransitDiscordContent, 0) - } - l_notifs := len(discordNotifMap[w.ID]) - if l_notifs == 0 || len(discordNotifMap[w.ID][l_notifs-1].DiscordRequest.Embeds) >= 10 { - discordNotifMap[w.ID] = append(discordNotifMap[w.ID], types.TransitDiscordContent{ - Webhook: w, - DiscordRequest: types.DiscordReq{ - Username: utils.Config.Frontend.SiteDomain, - }, - UserId: userID, - }) - l_notifs++ + if w.Retries >= 5 { + // early return + continue } + } - fields := []types.DiscordEmbedField{ - { - Name: "Epoch", - Value: fmt.Sprintf("[%[1]v](https://%[2]s/%[1]v)", n.GetEpoch(), utils.Config.Frontend.SiteDomain+"/epoch"), - Inline: false, - }, - } + for _, n := range notifications { + if w.Destination.Valid && w.Destination.String == "webhook_discord" { + if _, exists := discordNotifMap[w.ID]; !exists { + discordNotifMap[w.ID] = make([]types.TransitDiscordContent, 0) + } + l_notifs := len(discordNotifMap[w.ID]) + if l_notifs == 0 || len(discordNotifMap[w.ID][l_notifs-1].DiscordRequest.Embeds) >= 10 { + discordNotifMap[w.ID] = append(discordNotifMap[w.ID], types.TransitDiscordContent{ + Webhook: w, + DiscordRequest: types.DiscordReq{ + Username: utils.Config.Frontend.SiteDomain, + }, + UserId: userID, + }) + l_notifs++ + } - if strings.HasPrefix(string(n.GetEventName()), "monitoring") || n.GetEventName() == types.EthClientUpdateEventName || n.GetEventName() == types.RocketpoolCollateralMaxReachedEventName || n.GetEventName() == types.RocketpoolCollateralMinReachedEventName { - fields = append(fields, - types.DiscordEmbedField{ - Name: "Target", - Value: fmt.Sprintf("%v", n.GetEventFilter()), + fields := []types.DiscordEmbedField{ + { + Name: "Epoch", + Value: fmt.Sprintf("[%[1]v](https://%[2]s/%[1]v)", n.GetEpoch(), utils.Config.Frontend.SiteDomain+"/epoch"), Inline: false, - }) - } - discordNotifMap[w.ID][l_notifs-1].DiscordRequest.Embeds = append(discordNotifMap[w.ID][l_notifs-1].DiscordRequest.Embeds, types.DiscordEmbed{ - Type: "rich", - Color: "16745472", - Description: n.GetLegacyInfo(), - Title: n.GetLegacyTitle(), - Fields: fields, - }) - } else { - notifs = append(notifs, types.TransitWebhook{ - Channel: w.Destination.String, - Content: types.TransitWebhookContent{ - Webhook: w, - Event: types.WebhookEvent{ - Network: utils.GetNetwork(), - Name: string(n.GetEventName()), - Title: n.GetLegacyTitle(), - Description: n.GetLegacyInfo(), - Epoch: n.GetEpoch(), - Target: n.GetEventFilter(), }, - UserId: userID, - }, - }) + } + + if strings.HasPrefix(string(n.GetEventName()), "monitoring") || n.GetEventName() == types.EthClientUpdateEventName || n.GetEventName() == types.RocketpoolCollateralMaxReachedEventName || n.GetEventName() == types.RocketpoolCollateralMinReachedEventName { + fields = append(fields, + types.DiscordEmbedField{ + Name: "Target", + Value: fmt.Sprintf("%v", n.GetEventFilter()), + Inline: false, + }) + } + discordNotifMap[w.ID][l_notifs-1].DiscordRequest.Embeds = append(discordNotifMap[w.ID][l_notifs-1].DiscordRequest.Embeds, types.DiscordEmbed{ + Type: "rich", + Color: "16745472", + Description: n.GetLegacyInfo(), + Title: n.GetLegacyTitle(), + Fields: fields, + }) + } else { + notifs = append(notifs, types.TransitWebhook{ + Channel: w.Destination.String, + Content: types.TransitWebhookContent{ + Webhook: w, + Event: types.WebhookEvent{ + Network: utils.GetNetwork(), + Name: string(n.GetEventName()), + Title: n.GetLegacyTitle(), + Description: n.GetLegacyInfo(), + Epoch: n.GetEpoch(), + Target: n.GetEventFilter(), + }, + UserId: userID, + }, + }) + } } } } From 85d7259da5ffab4cb9799e78389a41d167aa60fd Mon Sep 17 00:00:00 2001 From: Stefan Pletka <124689083+Eisei24@users.noreply.github.com> Date: Tue, 22 Oct 2024 12:40:12 +0200 Subject: [PATCH 08/82] Adjusted discord webhook handling --- backend/pkg/api/data_access/notifications.go | 66 +++++++++++++------- 1 file changed, 42 insertions(+), 24 deletions(-) diff --git a/backend/pkg/api/data_access/notifications.go b/backend/pkg/api/data_access/notifications.go index 01d54b860..b84296825 100644 --- a/backend/pkg/api/data_access/notifications.go +++ b/backend/pkg/api/data_access/notifications.go @@ -81,8 +81,6 @@ const ( ValidatorDashboardEventPrefix string = "vdb" AccountDashboardEventPrefix string = "adb" - DiscordWebhookFormat string = "discord" - GroupOfflineThresholdDefault float64 = 0.1 MaxCollateralThresholdDefault float64 = 1.0 MinCollateralThresholdDefault float64 = 0.2 @@ -1765,14 +1763,14 @@ func (d *DataAccessService) GetNotificationSettingsDashboards(ctx context.Contex // ------------------------------------- // Get the validator dashboards valDashboards := []struct { - DashboardId uint64 `db:"dashboard_id"` - DashboardName string `db:"dashboard_name"` - GroupId uint64 `db:"group_id"` - GroupName string `db:"group_name"` - Network uint64 `db:"network"` - WebhookUrl sql.NullString `db:"webhook_target"` - IsWebhookDiscordEnabled sql.NullBool `db:"discord_webhook"` - IsRealTimeModeEnabled sql.NullBool `db:"realtime_notifications"` + DashboardId uint64 `db:"dashboard_id"` + DashboardName string `db:"dashboard_name"` + GroupId uint64 `db:"group_id"` + GroupName string `db:"group_name"` + Network uint64 `db:"network"` + WebhookUrl sql.NullString `db:"webhook_target"` + WebhookFormat sql.NullString `db:"webhook_format"` + IsRealTimeModeEnabled sql.NullBool `db:"realtime_notifications"` }{} wg.Go(func() error { err := d.alloyReader.SelectContext(ctx, &valDashboards, ` @@ -1783,11 +1781,11 @@ func (d *DataAccessService) GetNotificationSettingsDashboards(ctx context.Contex g.name AS group_name, d.network, g.webhook_target, - (g.webhook_format = $1) AS discord_webhook, + g.webhook_format, g.realtime_notifications FROM users_val_dashboards d INNER JOIN users_val_dashboards_groups g ON d.id = g.dashboard_id - WHERE d.user_id = $2`, DiscordWebhookFormat, userId) + WHERE d.user_id = $1`, userId) if err != nil { return fmt.Errorf(`error retrieving data for validator dashboard notifications: %w`, err) } @@ -1803,7 +1801,7 @@ func (d *DataAccessService) GetNotificationSettingsDashboards(ctx context.Contex GroupId uint64 `db:"group_id"` GroupName string `db:"group_name"` WebhookUrl sql.NullString `db:"webhook_target"` - IsWebhookDiscordEnabled sql.NullBool `db:"discord_webhook"` + WebhookFormat sql.NullString `db:"webhook_format"` IsIgnoreSpamTransactionsEnabled bool `db:"ignore_spam_transactions"` SubscribedChainIds []uint64 `db:"subscribed_chain_ids"` }{} @@ -1816,12 +1814,12 @@ func (d *DataAccessService) GetNotificationSettingsDashboards(ctx context.Contex // g.id AS group_id, // g.name AS group_name, // g.webhook_target, - // (g.webhook_format = $1) AS discord_webhook, + // g.webhook_format, // g.ignore_spam_transactions, // g.subscribed_chain_ids // FROM users_acc_dashboards d // INNER JOIN users_acc_dashboards_groups g ON d.id = g.dashboard_id - // WHERE d.user_id = $2`, DiscordWebhookFormat, userId) + // WHERE d.user_id = $1`, userId) // if err != nil { // return fmt.Errorf(`error retrieving data for validator dashboard notifications: %w`, err) // } @@ -1944,7 +1942,8 @@ func (d *DataAccessService) GetNotificationSettingsDashboards(ctx context.Contex // Set the settings if valSettings, ok := resultMap[key].Settings.(*t.NotificationSettingsValidatorDashboard); ok { valSettings.WebhookUrl = valDashboard.WebhookUrl.String - valSettings.IsWebhookDiscordEnabled = valDashboard.IsWebhookDiscordEnabled.Bool + valSettings.IsWebhookDiscordEnabled = valDashboard.WebhookFormat.Valid && + types.NotificationChannel(valDashboard.WebhookFormat.String) == types.WebhookDiscordNotificationChannel valSettings.IsRealTimeModeEnabled = valDashboard.IsRealTimeModeEnabled.Bool } } @@ -1972,7 +1971,8 @@ func (d *DataAccessService) GetNotificationSettingsDashboards(ctx context.Contex // Set the settings if accSettings, ok := resultMap[key].Settings.(*t.NotificationSettingsAccountDashboard); ok { accSettings.WebhookUrl = accDashboard.WebhookUrl.String - accSettings.IsWebhookDiscordEnabled = accDashboard.IsWebhookDiscordEnabled.Bool + accSettings.IsWebhookDiscordEnabled = accDashboard.WebhookFormat.Valid && + types.NotificationChannel(accDashboard.WebhookFormat.String) == types.WebhookDiscordNotificationChannel accSettings.IsIgnoreSpamTransactionsEnabled = accDashboard.IsIgnoreSpamTransactionsEnabled accSettings.SubscribedChainIds = accDashboard.SubscribedChainIds } @@ -2167,13 +2167,22 @@ func (d *DataAccessService) UpdateNotificationSettingsValidatorDashboard(ctx con } // Set non-event settings + var webhookFormat sql.NullString + if settings.WebhookUrl != "" { + webhookFormat.String = string(types.WebhookNotificationChannel) + webhookFormat.Valid = true + if settings.IsWebhookDiscordEnabled { + webhookFormat.String = string(types.WebhookDiscordNotificationChannel) + } + } + _, err = d.alloyWriter.ExecContext(ctx, ` UPDATE users_val_dashboards_groups SET webhook_target = NULLIF($1, ''), - webhook_format = CASE WHEN $2 THEN $3 ELSE NULL END, - realtime_notifications = CASE WHEN $4 THEN TRUE ELSE NULL END - WHERE dashboard_id = $5 AND id = $6`, settings.WebhookUrl, settings.IsWebhookDiscordEnabled, DiscordWebhookFormat, settings.IsRealTimeModeEnabled, dashboardId, groupId) + webhook_format = $2, + realtime_notifications = CASE WHEN $3 THEN TRUE ELSE NULL END + WHERE dashboard_id = $4 AND id = $5`, settings.WebhookUrl, webhookFormat, settings.IsRealTimeModeEnabled, dashboardId, groupId) if err != nil { return err } @@ -2247,14 +2256,23 @@ func (d *DataAccessService) UpdateNotificationSettingsAccountDashboard(ctx conte // } // // Set non-event settings + // var webhookFormat sql.NullString + // if settings.WebhookUrl != "" { + // webhookFormat.String = string(types.WebhookNotificationChannel) + // webhookFormat.Valid = true + // if settings.IsWebhookDiscordEnabled { + // webhookFormat.String = string(types.WebhookDiscordNotificationChannel) + // } + // } + // _, err = d.alloyWriter.ExecContext(ctx, ` // UPDATE users_acc_dashboards_groups // SET // webhook_target = NULLIF($1, ''), - // webhook_format = CASE WHEN $2 THEN $3 ELSE NULL END, - // ignore_spam_transactions = $4, - // subscribed_chain_ids = $5 - // WHERE dashboard_id = $6 AND id = $7`, settings.WebhookUrl, settings.IsWebhookDiscordEnabled, DiscordWebhookFormat, settings.IsIgnoreSpamTransactionsEnabled, settings.SubscribedChainIds, dashboardId, groupId) + // webhook_format = $2, + // ignore_spam_transactions = $3, + // subscribed_chain_ids = $4 + // WHERE dashboard_id = $5 AND id = $6`, settings.WebhookUrl, webhookFormat, settings.IsIgnoreSpamTransactionsEnabled, settings.SubscribedChainIds, dashboardId, groupId) // if err != nil { // return err // } From cfcc538c68d04733b7ea78acb3205c56e2856486 Mon Sep 17 00:00:00 2001 From: Stefan Pletka <124689083+Eisei24@users.noreply.github.com> Date: Tue, 22 Oct 2024 12:59:43 +0200 Subject: [PATCH 09/82] Fixed pointer type check --- backend/pkg/api/data_access/notifications.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/backend/pkg/api/data_access/notifications.go b/backend/pkg/api/data_access/notifications.go index b84296825..a2dd67fb0 100644 --- a/backend/pkg/api/data_access/notifications.go +++ b/backend/pkg/api/data_access/notifications.go @@ -1940,11 +1940,13 @@ func (d *DataAccessService) GetNotificationSettingsDashboards(ctx context.Contex resultMap[key].ChainIds = []uint64{valDashboard.Network} // Set the settings - if valSettings, ok := resultMap[key].Settings.(*t.NotificationSettingsValidatorDashboard); ok { + if valSettings, ok := resultMap[key].Settings.(t.NotificationSettingsValidatorDashboard); ok { valSettings.WebhookUrl = valDashboard.WebhookUrl.String valSettings.IsWebhookDiscordEnabled = valDashboard.WebhookFormat.Valid && types.NotificationChannel(valDashboard.WebhookFormat.String) == types.WebhookDiscordNotificationChannel valSettings.IsRealTimeModeEnabled = valDashboard.IsRealTimeModeEnabled.Bool + + resultMap[key].Settings = valSettings } } @@ -1969,12 +1971,14 @@ func (d *DataAccessService) GetNotificationSettingsDashboards(ctx context.Contex resultMap[key].ChainIds = accDashboard.SubscribedChainIds // Set the settings - if accSettings, ok := resultMap[key].Settings.(*t.NotificationSettingsAccountDashboard); ok { + if accSettings, ok := resultMap[key].Settings.(t.NotificationSettingsAccountDashboard); ok { accSettings.WebhookUrl = accDashboard.WebhookUrl.String accSettings.IsWebhookDiscordEnabled = accDashboard.WebhookFormat.Valid && types.NotificationChannel(accDashboard.WebhookFormat.String) == types.WebhookDiscordNotificationChannel accSettings.IsIgnoreSpamTransactionsEnabled = accDashboard.IsIgnoreSpamTransactionsEnabled accSettings.SubscribedChainIds = accDashboard.SubscribedChainIds + + resultMap[key].Settings = accSettings } } From 76f315a9f4f717672f149e7eb521a8f4ebbee2bc Mon Sep 17 00:00:00 2001 From: Lucca Dukic <109136188+LuccaBitfly@users.noreply.github.com> Date: Thu, 24 Oct 2024 08:02:26 +0200 Subject: [PATCH 10/82] fix: consistent naming for webhook notification parameters See: BEDS-94 --- backend/pkg/api/handlers/public.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/pkg/api/handlers/public.go b/backend/pkg/api/handlers/public.go index 9c7f5d5e2..b35b5ec5f 100644 --- a/backend/pkg/api/handlers/public.go +++ b/backend/pkg/api/handlers/public.go @@ -2703,7 +2703,7 @@ func (h *HandlerService) PublicPostUserNotificationsTestWebhook(w http.ResponseW } type request struct { WebhookUrl string `json:"webhook_url"` - IsDiscordWebhookEnabled bool `json:"is_discord_webhook_enabled,omitempty"` + IsWebhookDiscordEnabled bool `json:"is_webhook_discord_enabled,omitempty"` } var req request if err := v.checkBody(&req, r); err != nil { @@ -2714,7 +2714,7 @@ func (h *HandlerService) PublicPostUserNotificationsTestWebhook(w http.ResponseW handleErr(w, r, v) return } - err = h.getDataAccessor(r).QueueTestWebhookNotification(r.Context(), userId, req.WebhookUrl, req.IsDiscordWebhookEnabled) + err = h.getDataAccessor(r).QueueTestWebhookNotification(r.Context(), userId, req.WebhookUrl, req.IsWebhookDiscordEnabled) if err != nil { handleErr(w, r, err) return From 98044f451487c75ea3f52e9a6f7ef7cfff398515 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Thu, 24 Oct 2024 06:20:00 +0000 Subject: [PATCH 11/82] feat(notifications): implement test notifications --- backend/pkg/api/data_access/notifications.go | 10 +-- backend/pkg/commons/mail/mail.go | 10 +-- backend/pkg/notification/db.go | 5 +- backend/pkg/notification/queuing.go | 44 ++++++++++- backend/pkg/notification/sending.go | 79 +++++++++++++++++++- 5 files changed, 130 insertions(+), 18 deletions(-) diff --git a/backend/pkg/api/data_access/notifications.go b/backend/pkg/api/data_access/notifications.go index 868efb171..7ae5c2515 100644 --- a/backend/pkg/api/data_access/notifications.go +++ b/backend/pkg/api/data_access/notifications.go @@ -27,6 +27,7 @@ import ( "github.com/gobitfly/beaconchain/pkg/commons/log" "github.com/gobitfly/beaconchain/pkg/commons/types" "github.com/gobitfly/beaconchain/pkg/commons/utils" + "github.com/gobitfly/beaconchain/pkg/notification" n "github.com/gobitfly/beaconchain/pkg/notification" "github.com/lib/pq" "github.com/shopspring/decimal" @@ -2269,14 +2270,11 @@ func (d *DataAccessService) AddOrRemoveEvent(eventsToInsert *[]goqu.Record, even } func (d *DataAccessService) QueueTestEmailNotification(ctx context.Context, userId uint64) error { - // TODO: @Data Access - return nil + return notification.SendTestEmail(ctx, types.UserId(userId), d.userReader) } func (d *DataAccessService) QueueTestPushNotification(ctx context.Context, userId uint64) error { - // TODO: @Data Access - return nil + return notification.QueueTestPushNotification(ctx, types.UserId(userId), d.userReader, d.readerDb) } func (d *DataAccessService) QueueTestWebhookNotification(ctx context.Context, userId uint64, webhookUrl string, isDiscordWebhook bool) error { - // TODO: @Data Access - return nil + return notification.SendTestWebhookNotification(ctx, types.UserId(userId), webhookUrl, isDiscordWebhook) } diff --git a/backend/pkg/commons/mail/mail.go b/backend/pkg/commons/mail/mail.go index 60f27a14c..7e2491514 100644 --- a/backend/pkg/commons/mail/mail.go +++ b/backend/pkg/commons/mail/mail.go @@ -72,15 +72,9 @@ func createTextMessage(msg types.Email) string { // SendMailRateLimited sends an email to a given address with the given message. // It will return a ratelimit-error if the configured ratelimit is exceeded. -func SendMailRateLimited(content types.TransitEmailContent) error { +func SendMailRateLimited(content types.TransitEmailContent, maxEmailsPerDay int64, bucket string) error { sendThresholdReachedMail := false - maxEmailsPerDay := int64(0) - userInfo, err := db.GetUserInfo(context.Background(), uint64(content.UserId), db.FrontendReaderDB) - if err != nil { - return err - } - maxEmailsPerDay = int64(userInfo.PremiumPerks.EmailNotificationsPerDay) - count, err := db.CountSentMessage("n_mails", content.UserId) + count, err := db.CountSentMessage(bucket, content.UserId) if err != nil { return err } diff --git a/backend/pkg/notification/db.go b/backend/pkg/notification/db.go index 56c216fb5..d7ab55394 100644 --- a/backend/pkg/notification/db.go +++ b/backend/pkg/notification/db.go @@ -12,6 +12,7 @@ import ( "github.com/gobitfly/beaconchain/pkg/commons/log" "github.com/gobitfly/beaconchain/pkg/commons/types" "github.com/gobitfly/beaconchain/pkg/commons/utils" + "github.com/jmoiron/sqlx" "github.com/lib/pq" ) @@ -282,7 +283,7 @@ func GetSubsForEventFilter(eventName types.EventName, lastSentFilter string, las return subMap, nil } -func GetUserPushTokenByIds(ids []types.UserId) (map[types.UserId][]string, error) { +func GetUserPushTokenByIds(ids []types.UserId, userDbConn *sqlx.DB) (map[types.UserId][]string, error) { pushByID := map[types.UserId][]string{} if len(ids) == 0 { return pushByID, nil @@ -292,7 +293,7 @@ func GetUserPushTokenByIds(ids []types.UserId) (map[types.UserId][]string, error Token string `db:"notification_token"` } - err := db.FrontendWriterDB.Select(&rows, "SELECT DISTINCT ON (user_id, notification_token) user_id, notification_token FROM users_devices WHERE (user_id = ANY($1) AND user_id NOT IN (SELECT user_id from users_notification_channels WHERE active = false and channel = $2)) AND notify_enabled = true AND active = true AND notification_token IS NOT NULL AND LENGTH(notification_token) > 20 ORDER BY user_id, notification_token, id DESC", pq.Array(ids), types.PushNotificationChannel) + err := userDbConn.Select(&rows, "SELECT DISTINCT ON (user_id, notification_token) user_id, notification_token FROM users_devices WHERE (user_id = ANY($1) AND user_id NOT IN (SELECT user_id from users_notification_channels WHERE active = false and channel = $2)) AND notify_enabled = true AND active = true AND notification_token IS NOT NULL AND LENGTH(notification_token) > 20 ORDER BY user_id, notification_token, id DESC", pq.Array(ids), types.PushNotificationChannel) if err != nil { return nil, err } diff --git a/backend/pkg/notification/queuing.go b/backend/pkg/notification/queuing.go index 803a618c8..0a93b7846 100644 --- a/backend/pkg/notification/queuing.go +++ b/backend/pkg/notification/queuing.go @@ -3,6 +3,7 @@ package notification import ( "bytes" "compress/gzip" + "context" "database/sql" "encoding/gob" "fmt" @@ -450,7 +451,7 @@ func RenderPushMessagesForUserEvents(epoch uint64, notificationsByUserID types.N userIDs := slices.Collect(maps.Keys(notificationsByUserID)) - tokensByUserID, err := GetUserPushTokenByIds(userIDs) + tokensByUserID, err := GetUserPushTokenByIds(userIDs, db.FrontendReaderDB) if err != nil { metrics.Errors.WithLabelValues("notifications_send_push_notifications").Inc() return nil, fmt.Errorf("error when sending push-notifications: could not get tokens: %w", err) @@ -587,6 +588,47 @@ func QueuePushNotification(epoch uint64, notificationsByUserID types.Notificatio return nil } +func QueueTestPushNotification(ctx context.Context, userId types.UserId, userDbConn *sqlx.DB, networkDbConn *sqlx.DB) error { + count, err := db.CountSentMessage("n_test_push", userId) + if err != nil { + return err + } + if count > 10 { + return fmt.Errorf("rate limit has been exceeded") + } + tokens, err := GetUserPushTokenByIds([]types.UserId{userId}, userDbConn) + if err != nil { + return err + } + + messages := []*messaging.Message{} + for _, tokensOfUser := range tokens { + for _, token := range tokensOfUser { + log.Infof("sending test push to user %d with token %v", userId, token) + messages = append(messages, &messaging.Message{ + Notification: &messaging.Notification{ + Title: "Test Push", + Body: "This is a test push from beaconcha.in", + }, + Token: token, + }) + } + } + + if len(messages) == 0 { + return fmt.Errorf("no push tokens found for user %v", userId) + } + + transit := types.TransitPushContent{ + Messages: messages, + UserId: userId, + } + + _, err = networkDbConn.ExecContext(ctx, `INSERT INTO notification_queue (created, channel, content) VALUES (NOW(), 'push', $1)`, transit) + + return err +} + func QueueWebhookNotifications(notificationsByUserID types.NotificationsPerUserId, tx *sqlx.Tx) error { for userID, userNotifications := range notificationsByUserID { var webhooks []types.UserWebhook diff --git a/backend/pkg/notification/sending.go b/backend/pkg/notification/sending.go index 699f4c3f2..38e788c0c 100644 --- a/backend/pkg/notification/sending.go +++ b/backend/pkg/notification/sending.go @@ -18,6 +18,7 @@ import ( "github.com/gobitfly/beaconchain/pkg/commons/services" "github.com/gobitfly/beaconchain/pkg/commons/types" "github.com/gobitfly/beaconchain/pkg/commons/utils" + "github.com/jmoiron/sqlx" "github.com/lib/pq" ) @@ -154,7 +155,11 @@ func sendEmailNotifications() error { log.Infof("processing %v email notifications", len(notificationQueueItem)) for _, n := range notificationQueueItem { - err = mail.SendMailRateLimited(n.Content) + userInfo, err := db.GetUserInfo(context.Background(), uint64(n.Content.UserId), db.FrontendReaderDB) + if err != nil { + return err + } + err = mail.SendMailRateLimited(n.Content, int64(userInfo.PremiumPerks.EmailNotificationsPerDay), "n_emails") if err != nil { if !strings.Contains(err.Error(), "rate limit has been exceeded") { metrics.Errors.WithLabelValues("notifications_send_email").Inc() @@ -433,3 +438,75 @@ func sendDiscordNotifications() error { return nil } + +func SendTestEmail(ctx context.Context, userId types.UserId, dbConn *sqlx.DB) error { + var email string + err := dbConn.GetContext(ctx, &email, `SELECT email FROM users WHERE id = $1`, userId) + if err != nil { + return err + } + content := types.TransitEmailContent{ + UserId: userId, + Address: email, + Subject: "Test Email", + Email: types.Email{ + Title: "beaconcha.in - Test Email", + Body: "This is a test email from beaconcha.in", + }, + Attachments: []types.EmailAttachment{}, + CreatedTs: time.Now(), + } + err = mail.SendMailRateLimited(content, 10, "n_test_emails") + if err != nil { + return fmt.Errorf("error sending test email, err: %w", err) + } + + return nil +} + +func SendTestWebhookNotification(ctx context.Context, userId types.UserId, webhookUrl string, isDiscordWebhook bool) error { + count, err := db.CountSentMessage("n_test_push", userId) + if err != nil { + return err + } + if count > 10 { + return fmt.Errorf("rate limit has been exceeded") + } + + client := http.Client{Timeout: time.Second * 5} + + if isDiscordWebhook { + req := types.DiscordReq{ + Content: "This is a test notification from beaconcha.in", + } + reqBody := new(bytes.Buffer) + err := json.NewEncoder(reqBody).Encode(req) + if err != nil { + return fmt.Errorf("error marshalling discord webhook event: %w", err) + } + resp, err := client.Post(webhookUrl, "application/json", reqBody) + if err != nil { + return fmt.Errorf("error sending discord webhook request: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("error sending discord webhook request: %v", resp.Status) + } + } else { + // send a test webhook notification with the text "TEST" in the post body + reqBody := new(bytes.Buffer) + err := json.NewEncoder(reqBody).Encode(`{data: "TEST"}`) + if err != nil { + return fmt.Errorf("error marshalling webhook event: %w", err) + } + resp, err := client.Post(webhookUrl, "application/json", reqBody) + if err != nil { + return fmt.Errorf("error sending webhook request: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("error sending webhook request: %v", resp.Status) + } + } + return nil +} From 5395522261787f94a6043411368813a1b35b82e1 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Thu, 24 Oct 2024 06:24:04 +0000 Subject: [PATCH 12/82] feat(notifications): do not judge returned http code from webhook calls --- backend/pkg/notification/sending.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/backend/pkg/notification/sending.go b/backend/pkg/notification/sending.go index 38e788c0c..5d280bfae 100644 --- a/backend/pkg/notification/sending.go +++ b/backend/pkg/notification/sending.go @@ -489,9 +489,6 @@ func SendTestWebhookNotification(ctx context.Context, userId types.UserId, webho return fmt.Errorf("error sending discord webhook request: %w", err) } defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("error sending discord webhook request: %v", resp.Status) - } } else { // send a test webhook notification with the text "TEST" in the post body reqBody := new(bytes.Buffer) @@ -504,9 +501,6 @@ func SendTestWebhookNotification(ctx context.Context, userId types.UserId, webho return fmt.Errorf("error sending webhook request: %w", err) } defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("error sending webhook request: %v", resp.Status) - } } return nil } From f2807850433567dfc58967a379521c8c113d3326 Mon Sep 17 00:00:00 2001 From: Patrick Date: Thu, 24 Oct 2024 09:07:53 +0200 Subject: [PATCH 13/82] fix(ci): remove unneeded line in type-check (#1029) --- .github/workflows/backend-converted-types-check.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/backend-converted-types-check.yml b/.github/workflows/backend-converted-types-check.yml index a59593a02..b502e296e 100644 --- a/.github/workflows/backend-converted-types-check.yml +++ b/.github/workflows/backend-converted-types-check.yml @@ -41,7 +41,6 @@ jobs: newHash=$(find ../frontend/types/api -type f -print0 | sort -z | xargs -0 sha1sum | sha256sum | head -c 64) if [ "$currHash" != "$newHash" ]; then echo "frontend-types have changed, please commit the changes" - git diff --stat exit 1 fi From 7fd8571a9b6e8af1f0edf25eb5ce2ddaacf33ec9 Mon Sep 17 00:00:00 2001 From: marcel-bitfly <174338434+marcel-bitfly@users.noreply.github.com> Date: Thu, 24 Oct 2024 09:22:20 +0200 Subject: [PATCH 14/82] feat(BcInputUnit): set `decimal keyboard layout` on mobile --- frontend/.vscode/settings.json | 1 + frontend/components/bc/input/BcInputUnit.vue | 1 + 2 files changed, 2 insertions(+) diff --git a/frontend/.vscode/settings.json b/frontend/.vscode/settings.json index 8b64c0da4..b162c363d 100644 --- a/frontend/.vscode/settings.json +++ b/frontend/.vscode/settings.json @@ -1,6 +1,7 @@ { "conventionalCommits.scopes": [ "BcButton", + "BcInputUnit", "BcLink", "BcTablePager", "BcToggle", diff --git a/frontend/components/bc/input/BcInputUnit.vue b/frontend/components/bc/input/BcInputUnit.vue index a92b46b22..4f1f21f81 100644 --- a/frontend/components/bc/input/BcInputUnit.vue +++ b/frontend/components/bc/input/BcInputUnit.vue @@ -12,6 +12,7 @@ const input = defineModel() class="bc-input-unit__input" label-position="right" input-width="44px" + inputmode="decimal" :label="unit" type="number" v-bind="$attrs" From 4902c9b80f12263840ff9c266a00136fe85b69ac Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Thu, 24 Oct 2024 08:25:54 +0000 Subject: [PATCH 15/82] feat(notifications): wip dashboard discord webhooks --- backend/pkg/notification/queuing.go | 315 ++++++++++++++++++---------- backend/pkg/notification/sending.go | 11 +- 2 files changed, 214 insertions(+), 112 deletions(-) diff --git a/backend/pkg/notification/queuing.go b/backend/pkg/notification/queuing.go index 518687b44..fb03b41b3 100644 --- a/backend/pkg/notification/queuing.go +++ b/backend/pkg/notification/queuing.go @@ -644,7 +644,7 @@ func QueueWebhookNotifications(notificationsByUserID types.NotificationsPerUserI FROM users_webhooks WHERE - user_id = $1 AND user_id NOT IN (SELECT user_id from users_notification_channels WHERE active = false and channel = $2) + user_id = ANY($1) AND user_id NOT IN (SELECT user_id from users_notification_channels WHERE active = false and channel = $2) `, pq.Array(userIds), types.WebhookNotificationChannel) if err != nil { @@ -664,20 +664,24 @@ func QueueWebhookNotifications(notificationsByUserID types.NotificationsPerUserI users_val_dashboards_groups.id AS dashboard_group_id, dashboard_id AS dashboard_id, webhook_target AS url, - COALESCE(webhook_format, "webhook") AS destination, + COALESCE(webhook_format, 'webhook') AS destination, webhook_retries AS retries, webhook_last_sent AS last_sent FROM users_val_dashboards_groups LEFT JOIN users_val_dashboards ON users_val_dashboards_groups.dashboard_id = users_val_dashboards.id WHERE users_val_dashboards.user_id = ANY($1) AND webhook_target IS NOT NULL - AND webhook_format IS NOT NULL; - `, pq.Array(userIds)) + AND webhook_format IS NOT NULL + AND user_id NOT IN (SELECT user_id from users_notification_channels WHERE active = false and channel = $2); + `, pq.Array(userIds), types.WebhookNotificationChannel) if err != nil { return fmt.Errorf("error quering users_val_dashboards_groups, err: %w", err) } dashboardWebhookMap := make(map[types.UserId]map[types.DashboardId]map[types.DashboardGroupId]types.UserWebhook) for _, w := range webhooks { + if w.Destination.Valid && w.Destination.String == "discord" { + w.Destination.String = "webhook_discord" + } if _, exists := dashboardWebhookMap[types.UserId(w.UserID)]; !exists { dashboardWebhookMap[types.UserId(w.UserID)] = make(map[types.DashboardId]map[types.DashboardGroupId]types.UserWebhook) } @@ -689,136 +693,227 @@ func QueueWebhookNotifications(notificationsByUserID types.NotificationsPerUserI } for userID, userNotifications := range notificationsByUserID { - webhooks, exists := webhooksMap[uint64(userID)] - if !exists { - continue - } - // webhook => [] notifications discordNotifMap := make(map[uint64][]types.TransitDiscordContent) notifs := make([]types.TransitWebhook, 0) - // send the notifications to each registered webhook - for _, w := range webhooks { - for dashboardId, notificationsPerDashboard := range userNotifications { - for _, notificationsPerGroup := range notificationsPerDashboard { - if dashboardId != 0 { // disable webhooks for dashboard notifications for now - // retrieve the associated webhook config from the map - if _, exists := dashboardWebhookMap[types.UserId(userID)]; !exists { - continue - } - if _, exists := dashboardWebhookMap[types.UserId(userID)][types.DashboardId(dashboardId)]; !exists { - continue - } - if _, exists := dashboardWebhookMap[types.UserId(userID)][types.DashboardId(dashboardId)][0]; !exists { + webhooks, exists := webhooksMap[uint64(userID)] + if exists { + // webhook => [] notifications + // send the notifications to each registered webhook + for _, w := range webhooks { + for dashboardId, notificationsPerDashboard := range userNotifications { + for _, notificationsPerGroup := range notificationsPerDashboard { + if dashboardId != 0 { continue - } - w = dashboardWebhookMap[types.UserId(userID)][types.DashboardId(dashboardId)][0] - } else { - for event, notifications := range notificationsPerGroup { - // check if the webhook is subscribed to the type of event - eventSubscribed := slices.Contains(w.EventNames, string(event)) - - if eventSubscribed { - if len(notifications) > 0 { - // reset Retries - if w.Retries > 5 && w.LastSent.Valid && w.LastSent.Time.Add(time.Hour).Before(time.Now()) { - _, err = db.FrontendWriterDB.Exec(`UPDATE users_webhooks SET retries = 0 WHERE id = $1;`, w.ID) - if err != nil { - log.Error(err, "error updating users_webhooks table; setting retries to zero", 0) + } else { + for event, notifications := range notificationsPerGroup { + // check if the webhook is subscribed to the type of event + eventSubscribed := slices.Contains(w.EventNames, string(event)) + + if eventSubscribed { + if len(notifications) > 0 { + // reset Retries + if w.Retries > 5 && w.LastSent.Valid && w.LastSent.Time.Add(time.Hour).Before(time.Now()) { + _, err = db.FrontendWriterDB.Exec(`UPDATE users_webhooks SET retries = 0 WHERE id = $1;`, w.ID) + if err != nil { + log.Error(err, "error updating users_webhooks table; setting retries to zero", 0) + continue + } + } else if w.Retries > 5 && !w.LastSent.Valid { + log.Warnf("webhook '%v' has more than 5 retries and does not have a valid last_sent timestamp", w.Url) continue } - } else if w.Retries > 5 && !w.LastSent.Valid { - log.Warnf("webhook '%v' has more than 5 retries and does not have a valid last_sent timestamp", w.Url) - continue - } - if w.Retries >= 5 { - // early return - continue + if w.Retries >= 5 { + // early return + continue + } } - } - for _, n := range notifications { - if w.Destination.Valid && w.Destination.String == "webhook_discord" { - if _, exists := discordNotifMap[w.ID]; !exists { - discordNotifMap[w.ID] = make([]types.TransitDiscordContent, 0) - } - l_notifs := len(discordNotifMap[w.ID]) - if l_notifs == 0 || len(discordNotifMap[w.ID][l_notifs-1].DiscordRequest.Embeds) >= 10 { - discordNotifMap[w.ID] = append(discordNotifMap[w.ID], types.TransitDiscordContent{ - Webhook: w, - DiscordRequest: types.DiscordReq{ - Username: utils.Config.Frontend.SiteDomain, + for _, n := range notifications { + if w.Destination.Valid && w.Destination.String == "webhook_discord" { + if _, exists := discordNotifMap[w.ID]; !exists { + discordNotifMap[w.ID] = make([]types.TransitDiscordContent, 0) + } + l_notifs := len(discordNotifMap[w.ID]) + if l_notifs == 0 || len(discordNotifMap[w.ID][l_notifs-1].DiscordRequest.Embeds) >= 10 { + discordNotifMap[w.ID] = append(discordNotifMap[w.ID], types.TransitDiscordContent{ + Webhook: w, + DiscordRequest: types.DiscordReq{ + Username: utils.Config.Frontend.SiteDomain, + }, + UserId: userID, + }) + l_notifs++ + } + + fields := []types.DiscordEmbedField{ + { + Name: "Epoch", + Value: fmt.Sprintf("[%[1]v](https://%[2]s/%[1]v)", n.GetEpoch(), utils.Config.Frontend.SiteDomain+"/epoch"), + Inline: false, + }, + } + + if strings.HasPrefix(string(n.GetEventName()), "monitoring") || n.GetEventName() == types.EthClientUpdateEventName || n.GetEventName() == types.RocketpoolCollateralMaxReachedEventName || n.GetEventName() == types.RocketpoolCollateralMinReachedEventName { + fields = append(fields, + types.DiscordEmbedField{ + Name: "Target", + Value: fmt.Sprintf("%v", n.GetEventFilter()), + Inline: false, + }) + } + discordNotifMap[w.ID][l_notifs-1].DiscordRequest.Embeds = append(discordNotifMap[w.ID][l_notifs-1].DiscordRequest.Embeds, types.DiscordEmbed{ + Type: "rich", + Color: "16745472", + Description: n.GetLegacyInfo(), + Title: n.GetLegacyTitle(), + Fields: fields, + }) + } else { + notifs = append(notifs, types.TransitWebhook{ + Channel: w.Destination.String, + Content: types.TransitWebhookContent{ + Webhook: w, + Event: types.WebhookEvent{ + Network: utils.GetNetwork(), + Name: string(n.GetEventName()), + Title: n.GetLegacyTitle(), + Description: n.GetLegacyInfo(), + Epoch: n.GetEpoch(), + Target: n.GetEventFilter(), + }, + UserId: userID, }, - UserId: userID, }) - l_notifs++ } + } + } + } + } + } + } + } + } + // process dashboard webhooks + for dashboardId, notificationsPerDashboard := range userNotifications { + if dashboardId == 0 { + continue + } + for dashboardGroupId, notificationsPerGroup := range notificationsPerDashboard { + // retrieve the associated webhook config from the map + if _, exists := dashboardWebhookMap[userID]; !exists { + continue + } + if _, exists := dashboardWebhookMap[userID][dashboardId]; !exists { + continue + } + if _, exists := dashboardWebhookMap[userID][dashboardId][dashboardGroupId]; !exists { + continue + } + w := dashboardWebhookMap[userID][dashboardId][dashboardGroupId] - fields := []types.DiscordEmbedField{ - { - Name: "Epoch", - Value: fmt.Sprintf("[%[1]v](https://%[2]s/%[1]v)", n.GetEpoch(), utils.Config.Frontend.SiteDomain+"/epoch"), - Inline: false, - }, - } + // reset Retries + if w.Retries > 5 && w.LastSent.Valid && w.LastSent.Time.Add(time.Hour).Before(time.Now()) { + _, err = db.WriterDb.Exec(`UPDATE users_val_dashboards_groups SET webhook_retries = 0 WHERE id = $1 AND dashboard_id = $2;`, dashboardGroupId, dashboardId) + if err != nil { + log.Error(err, "error updating users_webhooks table; setting retries to zero", 0) + continue + } + } else if w.Retries > 5 && !w.LastSent.Valid { + log.Warnf("webhook '%v' for dashboard %d and group %d has more than 5 retries and does not have a valid last_sent timestamp", w.Url, dashboardId, dashboardGroupId) + continue + } - if strings.HasPrefix(string(n.GetEventName()), "monitoring") || n.GetEventName() == types.EthClientUpdateEventName || n.GetEventName() == types.RocketpoolCollateralMaxReachedEventName || n.GetEventName() == types.RocketpoolCollateralMinReachedEventName { - fields = append(fields, - types.DiscordEmbedField{ - Name: "Target", - Value: fmt.Sprintf("%v", n.GetEventFilter()), - Inline: false, - }) - } - discordNotifMap[w.ID][l_notifs-1].DiscordRequest.Embeds = append(discordNotifMap[w.ID][l_notifs-1].DiscordRequest.Embeds, types.DiscordEmbed{ - Type: "rich", - Color: "16745472", - Description: n.GetLegacyInfo(), - Title: n.GetLegacyTitle(), - Fields: fields, - }) - } else { - notifs = append(notifs, types.TransitWebhook{ - Channel: w.Destination.String, - Content: types.TransitWebhookContent{ - Webhook: w, - Event: types.WebhookEvent{ - Network: utils.GetNetwork(), - Name: string(n.GetEventName()), - Title: n.GetLegacyTitle(), - Description: n.GetLegacyInfo(), - Epoch: n.GetEpoch(), - Target: n.GetEventFilter(), - }, - UserId: userID, - }, - }) - } + if w.Retries >= 5 { + // early return + continue + } + + for event, notifications := range notificationsPerGroup { + if w.Destination.Valid && w.Destination.String == "webhook_discord" { + content := types.TransitDiscordContent{ + Webhook: w, + UserId: userID, + DiscordRequest: types.DiscordReq{ + Username: utils.Config.Frontend.SiteDomain, + }, + } + + totalBlockReward := float64(0) + details := "" + if event == types.ValidatorExecutedProposalEventName { + for _, n := range notifications { + proposalNotification, ok := n.(*ValidatorProposalNotification) + if !ok { + log.Error(fmt.Errorf("error casting proposal notification"), "", 0) + continue } + totalBlockReward += proposalNotification.Reward + + details += fmt.Sprintf("%s\n", n.GetInfo(types.NotifciationFormatMarkdown)) } } + + count := len(notifications) + summary := "" + plural := "" + if count > 1 { + plural = "s" + } + switch event { + case types.RocketpoolCollateralMaxReachedEventName, types.RocketpoolCollateralMinReachedEventName: + summary += fmt.Sprintf("%s: %d node%s", types.EventLabel[event], count, plural) + case types.TaxReportEventName, types.NetworkLivenessIncreasedEventName: + summary += fmt.Sprintf("%s: %d event%s", types.EventLabel[event], count, plural) + case types.EthClientUpdateEventName: + summary += fmt.Sprintf("%s: %d client%s", types.EventLabel[event], count, plural) + case types.MonitoringMachineCpuLoadEventName, types.MonitoringMachineMemoryUsageEventName, types.MonitoringMachineDiskAlmostFullEventName, types.MonitoringMachineOfflineEventName: + summary += fmt.Sprintf("%s: %d machine%s", types.EventLabel[event], count, plural) + case types.ValidatorExecutedProposalEventName: + summary += fmt.Sprintf("%s: %d validator%s, Reward: %.3f ETH", types.EventLabel[event], count, plural, totalBlockReward) + case types.ValidatorGroupEfficiencyEventName: + summary += fmt.Sprintf("%s: %d group%s", types.EventLabel[event], count, plural) + default: + summary += fmt.Sprintf("%s: %d validator%s", types.EventLabel[event], count, plural) + } + content.DiscordRequest.Content = summary + "\n" + details + if _, exists := discordNotifMap[w.ID]; !exists { + discordNotifMap[w.ID] = make([]types.TransitDiscordContent, 0) + } + log.Infof("adding discord notification for user %d, dashboard %d, group %d and type %s", userID, dashboardId, dashboardGroupId, event) + + discordNotifMap[w.ID] = append(discordNotifMap[w.ID], content) + } else { + // TODO: implement } } } } + // process notifs - for _, n := range notifs { - _, err = tx.Exec(`INSERT INTO notification_queue (created, channel, content) VALUES (now(), $1, $2);`, n.Channel, n.Content) - if err != nil { - log.Error(err, "error inserting into webhooks_queue", 0) - } else { - metrics.NotificationsQueued.WithLabelValues(n.Channel, n.Content.Event.Name).Inc() + if len(notifs) > 0 { + log.Infof("queueing %v webhooks notifications", len(notifs)) + for _, n := range notifs { + _, err = tx.Exec(`INSERT INTO notification_queue (created, channel, content) VALUES (now(), $1, $2);`, n.Channel, n.Content) + if err != nil { + log.Error(err, "error inserting into webhooks_queue", 0) + } else { + metrics.NotificationsQueued.WithLabelValues(n.Channel, n.Content.Event.Name).Inc() + } } } // process discord notifs - for _, dNotifs := range discordNotifMap { - for _, n := range dNotifs { - _, err = tx.Exec(`INSERT INTO notification_queue (created, channel, content) VALUES (now(), 'webhook_discord', $1);`, n) - if err != nil { - log.Error(err, "error inserting into webhooks_queue (discord)", 0) - continue - } else { - metrics.NotificationsQueued.WithLabelValues("webhook_discord", "multi").Inc() + if len(discordNotifMap) > 0 { + log.Infof("queueing %v discord notifications", len(discordNotifMap)) + for _, dNotifs := range discordNotifMap { + for _, n := range dNotifs { + _, err = tx.Exec(`INSERT INTO notification_queue (created, channel, content) VALUES (now(), 'webhook_discord', $1);`, n) + if err != nil { + log.Error(err, "error inserting into webhooks_queue (discord)", 0) + continue + } else { + metrics.NotificationsQueued.WithLabelValues("webhook_discord", "multi").Inc() + } } } } diff --git a/backend/pkg/notification/sending.go b/backend/pkg/notification/sending.go index 5d280bfae..440333333 100644 --- a/backend/pkg/notification/sending.go +++ b/backend/pkg/notification/sending.go @@ -362,7 +362,11 @@ func sendDiscordNotifications() error { go func(webhook types.UserWebhook, reqs []types.TransitDiscord) { defer func() { // update retries counters in db based on end result - _, err = db.FrontendWriterDB.Exec(`UPDATE users_webhooks SET retries = $1, last_sent = now() WHERE id = $2;`, webhook.Retries, webhook.ID) + if webhook.DashboardId == 0 && webhook.DashboardGroupId == 0 { + _, err = db.FrontendWriterDB.Exec(`UPDATE users_webhooks SET retries = $1, last_sent = now() WHERE id = $2;`, webhook.Retries, webhook.ID) + } else { + _, err = db.WriterDb.Exec(`UPDATE users_val_dashboards_groups SET webhook_retries = $1, webhook_last_sent = now() WHERE id = $2 AND dashboard_id = $3;`, webhook.Retries, webhook.DashboardGroupId, webhook.DashboardId) + } if err != nil { log.Warnf("failed to update retries counter to %v for webhook %v: %v", webhook.Retries, webhook.ID, err) } @@ -398,6 +402,7 @@ func sendDiscordNotifications() error { continue // skip } + log.Infof("sending discord webhook request to %s with: %v", webhook.Url, reqs[i].Content.DiscordRequest) resp, err := client.Post(webhook.Url, "application/json", reqBody) if err != nil { log.Warnf("failed sending discord webhook request %v: %v", webhook.ID, err) @@ -424,7 +429,9 @@ func sendDiscordNotifications() error { if resp.StatusCode != http.StatusOK { log.WarnWithFields(map[string]interface{}{"errResp.Body": utils.FirstN(errResp.Body, 1000), "webhook.Url": webhook.Url}, "error pushing discord webhook") } - _, err = db.FrontendWriterDB.Exec(`UPDATE users_webhooks SET request = $2, response = $3 WHERE id = $1;`, webhook.ID, reqs[i].Content.DiscordRequest, errResp) + if webhook.DashboardId == 0 && webhook.DashboardGroupId == 0 { + _, err = db.FrontendWriterDB.Exec(`UPDATE users_webhooks SET request = $2, response = $3 WHERE id = $1;`, webhook.ID, reqs[i].Content.DiscordRequest, errResp) + } if err != nil { log.Error(err, "error storing failure data in users_webhooks table", 0) } From f6836e6d72b84f16975662cbebca006f8e909900 Mon Sep 17 00:00:00 2001 From: Lucca Dukic <109136188+LuccaBitfly@users.noreply.github.com> Date: Thu, 24 Oct 2024 10:25:59 +0200 Subject: [PATCH 16/82] refactor: pass and use context everywhere See: BEDS-868 --- backend/pkg/api/data_access/archiver.go | 2 +- backend/pkg/api/data_access/block.go | 18 +- backend/pkg/api/data_access/data_access.go | 11 +- backend/pkg/api/data_access/dummy.go | 272 +++++++++--------- backend/pkg/api/data_access/header.go | 14 +- backend/pkg/api/data_access/mobile.go | 54 ++-- backend/pkg/api/data_access/notifications.go | 2 +- backend/pkg/api/data_access/vdb_management.go | 2 +- backend/pkg/api/handlers/auth.go | 12 +- backend/pkg/api/handlers/backward_compat.go | 2 +- backend/pkg/api/handlers/handler_service.go | 2 +- backend/pkg/api/handlers/input_validation.go | 5 +- backend/pkg/api/handlers/internal.go | 7 +- backend/pkg/api/handlers/public.go | 11 +- 14 files changed, 206 insertions(+), 208 deletions(-) diff --git a/backend/pkg/api/data_access/archiver.go b/backend/pkg/api/data_access/archiver.go index 09fd605a3..5d6d80aa1 100644 --- a/backend/pkg/api/data_access/archiver.go +++ b/backend/pkg/api/data_access/archiver.go @@ -27,7 +27,7 @@ func (d *DataAccessService) GetValidatorDashboardsCountInfo(ctx context.Context) } var dbReturn []DashboardInfo - err := d.readerDb.Select(&dbReturn, ` + err := d.readerDb.SelectContext(ctx, &dbReturn, ` WITH dashboards_groups AS (SELECT dashboard_id, diff --git a/backend/pkg/api/data_access/block.go b/backend/pkg/api/data_access/block.go index 65e63b408..7410c765f 100644 --- a/backend/pkg/api/data_access/block.go +++ b/backend/pkg/api/data_access/block.go @@ -74,7 +74,7 @@ func (d *DataAccessService) GetBlockBlobs(ctx context.Context, chainId, block ui } func (d *DataAccessService) GetSlot(ctx context.Context, chainId, slot uint64) (*t.BlockSummary, error) { - block, err := d.GetBlockHeightAt(slot) + block, err := d.GetBlockHeightAt(ctx, slot) if err != nil { return nil, err } @@ -82,7 +82,7 @@ func (d *DataAccessService) GetSlot(ctx context.Context, chainId, slot uint64) ( } func (d *DataAccessService) GetSlotOverview(ctx context.Context, chainId, slot uint64) (*t.BlockOverview, error) { - block, err := d.GetBlockHeightAt(slot) + block, err := d.GetBlockHeightAt(ctx, slot) if err != nil { return nil, err } @@ -90,7 +90,7 @@ func (d *DataAccessService) GetSlotOverview(ctx context.Context, chainId, slot u } func (d *DataAccessService) GetSlotTransactions(ctx context.Context, chainId, slot uint64) ([]t.BlockTransactionTableRow, error) { - block, err := d.GetBlockHeightAt(slot) + block, err := d.GetBlockHeightAt(ctx, slot) if err != nil { return nil, err } @@ -98,7 +98,7 @@ func (d *DataAccessService) GetSlotTransactions(ctx context.Context, chainId, sl } func (d *DataAccessService) GetSlotVotes(ctx context.Context, chainId, slot uint64) ([]t.BlockVoteTableRow, error) { - block, err := d.GetBlockHeightAt(slot) + block, err := d.GetBlockHeightAt(ctx, slot) if err != nil { return nil, err } @@ -106,7 +106,7 @@ func (d *DataAccessService) GetSlotVotes(ctx context.Context, chainId, slot uint } func (d *DataAccessService) GetSlotAttestations(ctx context.Context, chainId, slot uint64) ([]t.BlockAttestationTableRow, error) { - block, err := d.GetBlockHeightAt(slot) + block, err := d.GetBlockHeightAt(ctx, slot) if err != nil { return nil, err } @@ -114,7 +114,7 @@ func (d *DataAccessService) GetSlotAttestations(ctx context.Context, chainId, sl } func (d *DataAccessService) GetSlotWithdrawals(ctx context.Context, chainId, slot uint64) ([]t.BlockWithdrawalTableRow, error) { - block, err := d.GetBlockHeightAt(slot) + block, err := d.GetBlockHeightAt(ctx, slot) if err != nil { return nil, err } @@ -122,7 +122,7 @@ func (d *DataAccessService) GetSlotWithdrawals(ctx context.Context, chainId, slo } func (d *DataAccessService) GetSlotBlsChanges(ctx context.Context, chainId, slot uint64) ([]t.BlockBlsChangeTableRow, error) { - block, err := d.GetBlockHeightAt(slot) + block, err := d.GetBlockHeightAt(ctx, slot) if err != nil { return nil, err } @@ -130,7 +130,7 @@ func (d *DataAccessService) GetSlotBlsChanges(ctx context.Context, chainId, slot } func (d *DataAccessService) GetSlotVoluntaryExits(ctx context.Context, chainId, slot uint64) ([]t.BlockVoluntaryExitTableRow, error) { - block, err := d.GetBlockHeightAt(slot) + block, err := d.GetBlockHeightAt(ctx, slot) if err != nil { return nil, err } @@ -138,7 +138,7 @@ func (d *DataAccessService) GetSlotVoluntaryExits(ctx context.Context, chainId, } func (d *DataAccessService) GetSlotBlobs(ctx context.Context, chainId, slot uint64) ([]t.BlockBlobTableRow, error) { - block, err := d.GetBlockHeightAt(slot) + block, err := d.GetBlockHeightAt(ctx, slot) if err != nil { return nil, err } diff --git a/backend/pkg/api/data_access/data_access.go b/backend/pkg/api/data_access/data_access.go index 8bac59142..4f45f3fab 100644 --- a/backend/pkg/api/data_access/data_access.go +++ b/backend/pkg/api/data_access/data_access.go @@ -35,16 +35,15 @@ type DataAccessor interface { Close() - GetLatestFinalizedEpoch() (uint64, error) - GetLatestSlot() (uint64, error) - GetLatestBlock() (uint64, error) - GetBlockHeightAt(slot uint64) (uint64, error) - GetLatestExchangeRates() ([]t.EthConversionRate, error) + GetLatestFinalizedEpoch(ctx context.Context) (uint64, error) + GetLatestSlot(ctx context.Context) (uint64, error) + GetLatestBlock(ctx context.Context) (uint64, error) + GetLatestExchangeRates(ctx context.Context) ([]t.EthConversionRate, error) GetProductSummary(ctx context.Context) (*t.ProductSummary, error) GetFreeTierPerks(ctx context.Context) (*t.PremiumPerks, error) - GetValidatorsFromSlices(indices []uint64, publicKeys []string) ([]t.VDBValidator, error) + GetValidatorsFromSlices(ctx context.Context, indices []uint64, publicKeys []string) ([]t.VDBValidator, error) } type DataAccessService struct { diff --git a/backend/pkg/api/data_access/dummy.go b/backend/pkg/api/data_access/dummy.go index 0ec429593..387ea5c0d 100644 --- a/backend/pkg/api/data_access/dummy.go +++ b/backend/pkg/api/data_access/dummy.go @@ -53,8 +53,8 @@ func randomEthDecimal() decimal.Decimal { } // must pass a pointer to the data -func commonFakeData(a interface{}) error { - return faker.FakeData(a, options.WithRandomMapAndSliceMaxSize(5), options.WithRandomFloatBoundaries(interfaces.RandomFloatBoundary{Start: 0, End: 1})) +func populateWithFakeData(ctx context.Context, a interface{}) error { + return faker.FakeData(a, options.WithRandomMapAndSliceMaxSize(10), options.WithRandomFloatBoundaries(interfaces.RandomFloatBoundary{Start: 0, End: 1})) } func (d *DummyService) StartDataAccessServices() { @@ -62,25 +62,25 @@ func (d *DummyService) StartDataAccessServices() { } // used for any non-pointer data, e.g. all primitive types or slices -func getDummyData[T any]() (T, error) { +func getDummyData[T any](ctx context.Context) (T, error) { var r T - err := commonFakeData(&r) + err := populateWithFakeData(ctx, &r) return r, err } // used for any struct data that should be returned as a pointer -func getDummyStruct[T any]() (*T, error) { +func getDummyStruct[T any](ctx context.Context) (*T, error) { var r T - err := commonFakeData(&r) + err := populateWithFakeData(ctx, &r) return &r, err } // used for any table data that should be returned with paging -func getDummyWithPaging[T any]() ([]T, *t.Paging, error) { +func getDummyWithPaging[T any](ctx context.Context) ([]T, *t.Paging, error) { r := []T{} p := t.Paging{} - _ = commonFakeData(&r) - err := commonFakeData(&p) + _ = populateWithFakeData(ctx, &r) + err := populateWithFakeData(ctx, &p) return r, &p, err } @@ -88,32 +88,28 @@ func (d *DummyService) Close() { // nothing to close } -func (d *DummyService) GetLatestSlot() (uint64, error) { - return getDummyData[uint64]() +func (d *DummyService) GetLatestSlot(ctx context.Context) (uint64, error) { + return getDummyData[uint64](ctx) } -func (d *DummyService) GetLatestFinalizedEpoch() (uint64, error) { - return getDummyData[uint64]() +func (d *DummyService) GetLatestFinalizedEpoch(ctx context.Context) (uint64, error) { + return getDummyData[uint64](ctx) } -func (d *DummyService) GetLatestBlock() (uint64, error) { - return getDummyData[uint64]() +func (d *DummyService) GetLatestBlock(ctx context.Context) (uint64, error) { + return getDummyData[uint64](ctx) } -func (d *DummyService) GetBlockHeightAt(slot uint64) (uint64, error) { - return getDummyData[uint64]() -} - -func (d *DummyService) GetLatestExchangeRates() ([]t.EthConversionRate, error) { - return getDummyData[[]t.EthConversionRate]() +func (d *DummyService) GetLatestExchangeRates(ctx context.Context) ([]t.EthConversionRate, error) { + return getDummyData[[]t.EthConversionRate](ctx) } func (d *DummyService) GetUserByEmail(ctx context.Context, email string) (uint64, error) { - return getDummyData[uint64]() + return getDummyData[uint64](ctx) } func (d *DummyService) CreateUser(ctx context.Context, email, password string) (uint64, error) { - return getDummyData[uint64]() + return getDummyData[uint64](ctx) } func (d *DummyService) RemoveUser(ctx context.Context, userId uint64) error { @@ -129,11 +125,11 @@ func (d *DummyService) UpdateUserPassword(ctx context.Context, userId uint64, pa } func (d *DummyService) GetEmailConfirmationTime(ctx context.Context, userId uint64) (time.Time, error) { - return getDummyData[time.Time]() + return getDummyData[time.Time](ctx) } func (d *DummyService) GetPasswordResetTime(ctx context.Context, userId uint64) (time.Time, error) { - return getDummyData[time.Time]() + return getDummyData[time.Time](ctx) } func (d *DummyService) UpdateEmailConfirmationTime(ctx context.Context, userId uint64) error { @@ -157,66 +153,66 @@ func (d *DummyService) UpdatePasswordResetHash(ctx context.Context, userId uint6 } func (d *DummyService) GetUserInfo(ctx context.Context, userId uint64) (*t.UserInfo, error) { - return getDummyStruct[t.UserInfo]() + return getDummyStruct[t.UserInfo](ctx) } func (d *DummyService) GetUserCredentialInfo(ctx context.Context, userId uint64) (*t.UserCredentialInfo, error) { - return getDummyStruct[t.UserCredentialInfo]() + return getDummyStruct[t.UserCredentialInfo](ctx) } func (d *DummyService) GetUserIdByApiKey(ctx context.Context, apiKey string) (uint64, error) { - return getDummyData[uint64]() + return getDummyData[uint64](ctx) } func (d *DummyService) GetUserIdByConfirmationHash(ctx context.Context, hash string) (uint64, error) { - return getDummyData[uint64]() + return getDummyData[uint64](ctx) } func (d *DummyService) GetUserIdByResetHash(ctx context.Context, hash string) (uint64, error) { - return getDummyData[uint64]() + return getDummyData[uint64](ctx) } func (d *DummyService) GetProductSummary(ctx context.Context) (*t.ProductSummary, error) { - return getDummyStruct[t.ProductSummary]() + return getDummyStruct[t.ProductSummary](ctx) } func (d *DummyService) GetFreeTierPerks(ctx context.Context) (*t.PremiumPerks, error) { - return getDummyStruct[t.PremiumPerks]() + return getDummyStruct[t.PremiumPerks](ctx) } func (d *DummyService) GetValidatorDashboardUser(ctx context.Context, dashboardId t.VDBIdPrimary) (*t.DashboardUser, error) { - return getDummyStruct[t.DashboardUser]() + return getDummyStruct[t.DashboardUser](ctx) } func (d *DummyService) GetValidatorDashboardIdByPublicId(ctx context.Context, publicDashboardId t.VDBIdPublic) (*t.VDBIdPrimary, error) { - return getDummyStruct[t.VDBIdPrimary]() + return getDummyStruct[t.VDBIdPrimary](ctx) } func (d *DummyService) GetValidatorDashboardInfo(ctx context.Context, dashboardId t.VDBIdPrimary) (*t.ValidatorDashboard, error) { - r, err := getDummyStruct[t.ValidatorDashboard]() + r, err := getDummyStruct[t.ValidatorDashboard](ctx) // return semi-valid data to not break staging r.IsArchived = false return r, err } func (d *DummyService) GetValidatorDashboardName(ctx context.Context, dashboardId t.VDBIdPrimary) (string, error) { - return getDummyData[string]() + return getDummyData[string](ctx) } -func (d *DummyService) GetValidatorsFromSlices(indices []uint64, publicKeys []string) ([]t.VDBValidator, error) { - return getDummyData[[]t.VDBValidator]() +func (d *DummyService) GetValidatorsFromSlices(ctx context.Context, indices []uint64, publicKeys []string) ([]t.VDBValidator, error) { + return getDummyData[[]t.VDBValidator](ctx) } func (d *DummyService) GetUserDashboards(ctx context.Context, userId uint64) (*t.UserDashboardsData, error) { - return getDummyStruct[t.UserDashboardsData]() + return getDummyStruct[t.UserDashboardsData](ctx) } func (d *DummyService) CreateValidatorDashboard(ctx context.Context, userId uint64, name string, network uint64) (*t.VDBPostReturnData, error) { - return getDummyStruct[t.VDBPostReturnData]() + return getDummyStruct[t.VDBPostReturnData](ctx) } func (d *DummyService) GetValidatorDashboardOverview(ctx context.Context, dashboardId t.VDBId, protocolModes t.VDBProtocolModes) (*t.VDBOverviewData, error) { - return getDummyStruct[t.VDBOverviewData]() + return getDummyStruct[t.VDBOverviewData](ctx) } func (d *DummyService) RemoveValidatorDashboard(ctx context.Context, dashboardId t.VDBIdPrimary) error { @@ -228,7 +224,7 @@ func (d *DummyService) RemoveValidatorDashboards(ctx context.Context, dashboardI } func (d *DummyService) UpdateValidatorDashboardArchiving(ctx context.Context, dashboardId t.VDBIdPrimary, archivedReason *enums.VDBArchivedReason) (*t.VDBPostArchivingReturnData, error) { - return getDummyStruct[t.VDBPostArchivingReturnData]() + return getDummyStruct[t.VDBPostArchivingReturnData](ctx) } func (d *DummyService) UpdateValidatorDashboardsArchiving(ctx context.Context, dashboards []t.ArchiverDashboardArchiveReason) error { @@ -236,15 +232,15 @@ func (d *DummyService) UpdateValidatorDashboardsArchiving(ctx context.Context, d } func (d *DummyService) UpdateValidatorDashboardName(ctx context.Context, dashboardId t.VDBIdPrimary, name string) (*t.VDBPostReturnData, error) { - return getDummyStruct[t.VDBPostReturnData]() + return getDummyStruct[t.VDBPostReturnData](ctx) } func (d *DummyService) CreateValidatorDashboardGroup(ctx context.Context, dashboardId t.VDBIdPrimary, name string) (*t.VDBPostCreateGroupData, error) { - return getDummyStruct[t.VDBPostCreateGroupData]() + return getDummyStruct[t.VDBPostCreateGroupData](ctx) } func (d *DummyService) UpdateValidatorDashboardGroup(ctx context.Context, dashboardId t.VDBIdPrimary, groupId uint64, name string) (*t.VDBPostCreateGroupData, error) { - return getDummyStruct[t.VDBPostCreateGroupData]() + return getDummyStruct[t.VDBPostCreateGroupData](ctx) } func (d *DummyService) RemoveValidatorDashboardGroup(ctx context.Context, dashboardId t.VDBIdPrimary, groupId uint64) error { @@ -256,23 +252,23 @@ func (d *DummyService) GetValidatorDashboardGroupExists(ctx context.Context, das } func (d *DummyService) AddValidatorDashboardValidators(ctx context.Context, dashboardId t.VDBIdPrimary, groupId uint64, validators []t.VDBValidator) ([]t.VDBPostValidatorsData, error) { - return getDummyData[[]t.VDBPostValidatorsData]() + return getDummyData[[]t.VDBPostValidatorsData](ctx) } func (d *DummyService) AddValidatorDashboardValidatorsByDepositAddress(ctx context.Context, dashboardId t.VDBIdPrimary, groupId uint64, address string, limit uint64) ([]t.VDBPostValidatorsData, error) { - return getDummyData[[]t.VDBPostValidatorsData]() + return getDummyData[[]t.VDBPostValidatorsData](ctx) } func (d *DummyService) AddValidatorDashboardValidatorsByWithdrawalAddress(ctx context.Context, dashboardId t.VDBIdPrimary, groupId uint64, address string, limit uint64) ([]t.VDBPostValidatorsData, error) { - return getDummyData[[]t.VDBPostValidatorsData]() + return getDummyData[[]t.VDBPostValidatorsData](ctx) } func (d *DummyService) AddValidatorDashboardValidatorsByGraffiti(ctx context.Context, dashboardId t.VDBIdPrimary, groupId uint64, graffiti string, limit uint64) ([]t.VDBPostValidatorsData, error) { - return getDummyData[[]t.VDBPostValidatorsData]() + return getDummyData[[]t.VDBPostValidatorsData](ctx) } func (d *DummyService) GetValidatorDashboardValidators(ctx context.Context, dashboardId t.VDBId, groupId int64, cursor string, colSort t.Sort[enums.VDBManageValidatorsColumn], search string, limit uint64) ([]t.VDBManageValidatorsTableRow, *t.Paging, error) { - return getDummyWithPaging[t.VDBManageValidatorsTableRow]() + return getDummyWithPaging[t.VDBManageValidatorsTableRow](ctx) } func (d *DummyService) RemoveValidatorDashboardValidators(ctx context.Context, dashboardId t.VDBIdPrimary, validators []t.VDBValidator) error { @@ -280,15 +276,15 @@ func (d *DummyService) RemoveValidatorDashboardValidators(ctx context.Context, d } func (d *DummyService) CreateValidatorDashboardPublicId(ctx context.Context, dashboardId t.VDBIdPrimary, name string, shareGroups bool) (*t.VDBPublicId, error) { - return getDummyStruct[t.VDBPublicId]() + return getDummyStruct[t.VDBPublicId](ctx) } func (d *DummyService) GetValidatorDashboardPublicId(ctx context.Context, publicDashboardId t.VDBIdPublic) (*t.VDBPublicId, error) { - return getDummyStruct[t.VDBPublicId]() + return getDummyStruct[t.VDBPublicId](ctx) } func (d *DummyService) UpdateValidatorDashboardPublicId(ctx context.Context, publicDashboardId t.VDBIdPublic, name string, shareGroups bool) (*t.VDBPublicId, error) { - return getDummyStruct[t.VDBPublicId]() + return getDummyStruct[t.VDBPublicId](ctx) } func (d *DummyService) RemoveValidatorDashboardPublicId(ctx context.Context, publicDashboardId t.VDBIdPublic) error { @@ -299,76 +295,76 @@ func (d *DummyService) GetValidatorDashboardSlotViz(ctx context.Context, dashboa r := struct { Epochs []t.SlotVizEpoch `faker:"slice_len=4"` }{} - err := commonFakeData(&r) + err := populateWithFakeData(ctx, &r) return r.Epochs, err } func (d *DummyService) GetValidatorDashboardSummary(ctx context.Context, dashboardId t.VDBId, period enums.TimePeriod, cursor string, colSort t.Sort[enums.VDBSummaryColumn], search string, limit uint64, protocolModes t.VDBProtocolModes) ([]t.VDBSummaryTableRow, *t.Paging, error) { - return getDummyWithPaging[t.VDBSummaryTableRow]() + return getDummyWithPaging[t.VDBSummaryTableRow](ctx) } func (d *DummyService) GetValidatorDashboardGroupSummary(ctx context.Context, dashboardId t.VDBId, groupId int64, period enums.TimePeriod, protocolModes t.VDBProtocolModes) (*t.VDBGroupSummaryData, error) { - return getDummyStruct[t.VDBGroupSummaryData]() + return getDummyStruct[t.VDBGroupSummaryData](ctx) } func (d *DummyService) GetValidatorDashboardSummaryChart(ctx context.Context, dashboardId t.VDBId, groupIds []int64, efficiency enums.VDBSummaryChartEfficiencyType, aggregation enums.ChartAggregation, afterTs uint64, beforeTs uint64) (*t.ChartData[int, float64], error) { - return getDummyStruct[t.ChartData[int, float64]]() + return getDummyStruct[t.ChartData[int, float64]](ctx) } func (d *DummyService) GetValidatorDashboardSummaryValidators(ctx context.Context, dashboardId t.VDBId, groupId int64) (*t.VDBGeneralSummaryValidators, error) { - return getDummyStruct[t.VDBGeneralSummaryValidators]() + return getDummyStruct[t.VDBGeneralSummaryValidators](ctx) } func (d *DummyService) GetValidatorDashboardSyncSummaryValidators(ctx context.Context, dashboardId t.VDBId, groupId int64, period enums.TimePeriod) (*t.VDBSyncSummaryValidators, error) { - return getDummyStruct[t.VDBSyncSummaryValidators]() + return getDummyStruct[t.VDBSyncSummaryValidators](ctx) } func (d *DummyService) GetValidatorDashboardSlashingsSummaryValidators(ctx context.Context, dashboardId t.VDBId, groupId int64, period enums.TimePeriod) (*t.VDBSlashingsSummaryValidators, error) { - return getDummyStruct[t.VDBSlashingsSummaryValidators]() + return getDummyStruct[t.VDBSlashingsSummaryValidators](ctx) } func (d *DummyService) GetValidatorDashboardProposalSummaryValidators(ctx context.Context, dashboardId t.VDBId, groupId int64, period enums.TimePeriod) (*t.VDBProposalSummaryValidators, error) { - return getDummyStruct[t.VDBProposalSummaryValidators]() + return getDummyStruct[t.VDBProposalSummaryValidators](ctx) } func (d *DummyService) GetValidatorDashboardRewards(ctx context.Context, dashboardId t.VDBId, cursor string, colSort t.Sort[enums.VDBRewardsColumn], search string, limit uint64, protocolModes t.VDBProtocolModes) ([]t.VDBRewardsTableRow, *t.Paging, error) { - return getDummyWithPaging[t.VDBRewardsTableRow]() + return getDummyWithPaging[t.VDBRewardsTableRow](ctx) } func (d *DummyService) GetValidatorDashboardGroupRewards(ctx context.Context, dashboardId t.VDBId, groupId int64, epoch uint64, protocolModes t.VDBProtocolModes) (*t.VDBGroupRewardsData, error) { - return getDummyStruct[t.VDBGroupRewardsData]() + return getDummyStruct[t.VDBGroupRewardsData](ctx) } func (d *DummyService) GetValidatorDashboardRewardsChart(ctx context.Context, dashboardId t.VDBId, protocolModes t.VDBProtocolModes) (*t.ChartData[int, decimal.Decimal], error) { - return getDummyStruct[t.ChartData[int, decimal.Decimal]]() + return getDummyStruct[t.ChartData[int, decimal.Decimal]](ctx) } func (d *DummyService) GetValidatorDashboardDuties(ctx context.Context, dashboardId t.VDBId, epoch uint64, groupId int64, cursor string, colSort t.Sort[enums.VDBDutiesColumn], search string, limit uint64, protocolModes t.VDBProtocolModes) ([]t.VDBEpochDutiesTableRow, *t.Paging, error) { - return getDummyWithPaging[t.VDBEpochDutiesTableRow]() + return getDummyWithPaging[t.VDBEpochDutiesTableRow](ctx) } func (d *DummyService) GetValidatorDashboardBlocks(ctx context.Context, dashboardId t.VDBId, cursor string, colSort t.Sort[enums.VDBBlocksColumn], search string, limit uint64, protocolModes t.VDBProtocolModes) ([]t.VDBBlocksTableRow, *t.Paging, error) { - return getDummyWithPaging[t.VDBBlocksTableRow]() + return getDummyWithPaging[t.VDBBlocksTableRow](ctx) } func (d *DummyService) GetValidatorDashboardHeatmap(ctx context.Context, dashboardId t.VDBId, protocolModes t.VDBProtocolModes, aggregation enums.ChartAggregation, afterTs uint64, beforeTs uint64) (*t.VDBHeatmap, error) { - return getDummyStruct[t.VDBHeatmap]() + return getDummyStruct[t.VDBHeatmap](ctx) } func (d *DummyService) GetValidatorDashboardGroupHeatmap(ctx context.Context, dashboardId t.VDBId, groupId uint64, protocolModes t.VDBProtocolModes, aggregation enums.ChartAggregation, timestamp uint64) (*t.VDBHeatmapTooltipData, error) { - return getDummyStruct[t.VDBHeatmapTooltipData]() + return getDummyStruct[t.VDBHeatmapTooltipData](ctx) } func (d *DummyService) GetValidatorDashboardElDeposits(ctx context.Context, dashboardId t.VDBId, cursor string, limit uint64) ([]t.VDBExecutionDepositsTableRow, *t.Paging, error) { - return getDummyWithPaging[t.VDBExecutionDepositsTableRow]() + return getDummyWithPaging[t.VDBExecutionDepositsTableRow](ctx) } func (d *DummyService) GetValidatorDashboardClDeposits(ctx context.Context, dashboardId t.VDBId, cursor string, limit uint64) ([]t.VDBConsensusDepositsTableRow, *t.Paging, error) { - return getDummyWithPaging[t.VDBConsensusDepositsTableRow]() + return getDummyWithPaging[t.VDBConsensusDepositsTableRow](ctx) } func (d *DummyService) GetValidatorDashboardTotalElDeposits(ctx context.Context, dashboardId t.VDBId) (*t.VDBTotalExecutionDepositsData, error) { - return getDummyStruct[t.VDBTotalExecutionDepositsData]() + return getDummyStruct[t.VDBTotalExecutionDepositsData](ctx) } func (d *DummyService) GetValidatorDashboardTotalClDeposits(ctx context.Context, dashboardId t.VDBId) (*t.VDBTotalConsensusDepositsData, error) { - return getDummyStruct[t.VDBTotalConsensusDepositsData]() + return getDummyStruct[t.VDBTotalConsensusDepositsData](ctx) } func (d *DummyService) GetValidatorDashboardWithdrawals(ctx context.Context, dashboardId t.VDBId, cursor string, colSort t.Sort[enums.VDBWithdrawalsColumn], search string, limit uint64, protocolModes t.VDBProtocolModes) ([]t.VDBWithdrawalsTableRow, *t.Paging, error) { @@ -376,19 +372,19 @@ func (d *DummyService) GetValidatorDashboardWithdrawals(ctx context.Context, das } func (d *DummyService) GetValidatorDashboardTotalWithdrawals(ctx context.Context, dashboardId t.VDBId, search string, protocolModes t.VDBProtocolModes) (*t.VDBTotalWithdrawalsData, error) { - return getDummyStruct[t.VDBTotalWithdrawalsData]() + return getDummyStruct[t.VDBTotalWithdrawalsData](ctx) } func (d *DummyService) GetValidatorDashboardRocketPool(ctx context.Context, dashboardId t.VDBId, cursor string, colSort t.Sort[enums.VDBRocketPoolColumn], search string, limit uint64) ([]t.VDBRocketPoolTableRow, *t.Paging, error) { - return getDummyWithPaging[t.VDBRocketPoolTableRow]() + return getDummyWithPaging[t.VDBRocketPoolTableRow](ctx) } func (d *DummyService) GetValidatorDashboardTotalRocketPool(ctx context.Context, dashboardId t.VDBId, search string) (*t.VDBRocketPoolTableRow, error) { - return getDummyStruct[t.VDBRocketPoolTableRow]() + return getDummyStruct[t.VDBRocketPoolTableRow](ctx) } func (d *DummyService) GetValidatorDashboardRocketPoolMinipools(ctx context.Context, dashboardId t.VDBId, node string, cursor string, colSort t.Sort[enums.VDBRocketPoolMinipoolsColumn], search string, limit uint64) ([]t.VDBRocketPoolMinipoolsTableRow, *t.Paging, error) { - return getDummyWithPaging[t.VDBRocketPoolMinipoolsTableRow]() + return getDummyWithPaging[t.VDBRocketPoolMinipoolsTableRow](ctx) } func (d *DummyService) GetAllNetworks() ([]t.NetworkInfo, error) { @@ -492,82 +488,82 @@ func (d *DummyService) GetAllClients() ([]t.ClientInfo, error) { } func (d *DummyService) GetSearchValidatorByIndex(ctx context.Context, chainId, index uint64) (*t.SearchValidator, error) { - return getDummyStruct[t.SearchValidator]() + return getDummyStruct[t.SearchValidator](ctx) } func (d *DummyService) GetSearchValidatorByPublicKey(ctx context.Context, chainId uint64, publicKey []byte) (*t.SearchValidator, error) { - return getDummyStruct[t.SearchValidator]() + return getDummyStruct[t.SearchValidator](ctx) } func (d *DummyService) GetSearchValidatorsByDepositAddress(ctx context.Context, chainId uint64, address []byte) (*t.SearchValidatorsByDepositAddress, error) { - return getDummyStruct[t.SearchValidatorsByDepositAddress]() + return getDummyStruct[t.SearchValidatorsByDepositAddress](ctx) } func (d *DummyService) GetSearchValidatorsByDepositEnsName(ctx context.Context, chainId uint64, ensName string) (*t.SearchValidatorsByDepositEnsName, error) { - return getDummyStruct[t.SearchValidatorsByDepositEnsName]() + return getDummyStruct[t.SearchValidatorsByDepositEnsName](ctx) } func (d *DummyService) GetSearchValidatorsByWithdrawalCredential(ctx context.Context, chainId uint64, credential []byte) (*t.SearchValidatorsByWithdrwalCredential, error) { - return getDummyStruct[t.SearchValidatorsByWithdrwalCredential]() + return getDummyStruct[t.SearchValidatorsByWithdrwalCredential](ctx) } func (d *DummyService) GetSearchValidatorsByWithdrawalEnsName(ctx context.Context, chainId uint64, ensName string) (*t.SearchValidatorsByWithrawalEnsName, error) { - return getDummyStruct[t.SearchValidatorsByWithrawalEnsName]() + return getDummyStruct[t.SearchValidatorsByWithrawalEnsName](ctx) } func (d *DummyService) GetSearchValidatorsByGraffiti(ctx context.Context, chainId uint64, graffiti string) (*t.SearchValidatorsByGraffiti, error) { - return getDummyStruct[t.SearchValidatorsByGraffiti]() + return getDummyStruct[t.SearchValidatorsByGraffiti](ctx) } func (d *DummyService) GetUserValidatorDashboardCount(ctx context.Context, userId uint64, active bool) (uint64, error) { - return getDummyData[uint64]() + return getDummyData[uint64](ctx) } func (d *DummyService) GetValidatorDashboardGroupCount(ctx context.Context, dashboardId t.VDBIdPrimary) (uint64, error) { - return getDummyData[uint64]() + return getDummyData[uint64](ctx) } func (d *DummyService) GetValidatorDashboardValidatorsCount(ctx context.Context, dashboardId t.VDBIdPrimary) (uint64, error) { - return getDummyData[uint64]() + return getDummyData[uint64](ctx) } func (d *DummyService) GetValidatorDashboardPublicIdCount(ctx context.Context, dashboardId t.VDBIdPrimary) (uint64, error) { - return getDummyData[uint64]() + return getDummyData[uint64](ctx) } func (d *DummyService) GetNotificationOverview(ctx context.Context, userId uint64) (*t.NotificationOverviewData, error) { - return getDummyStruct[t.NotificationOverviewData]() + return getDummyStruct[t.NotificationOverviewData](ctx) } func (d *DummyService) GetDashboardNotifications(ctx context.Context, userId uint64, chainIds []uint64, cursor string, colSort t.Sort[enums.NotificationDashboardsColumn], search string, limit uint64) ([]t.NotificationDashboardsTableRow, *t.Paging, error) { - return getDummyWithPaging[t.NotificationDashboardsTableRow]() + return getDummyWithPaging[t.NotificationDashboardsTableRow](ctx) } func (d *DummyService) GetValidatorDashboardNotificationDetails(ctx context.Context, dashboardId t.VDBIdPrimary, groupId uint64, epoch uint64, search string) (*t.NotificationValidatorDashboardDetail, error) { - return getDummyStruct[t.NotificationValidatorDashboardDetail]() + return getDummyStruct[t.NotificationValidatorDashboardDetail](ctx) } func (d *DummyService) GetAccountDashboardNotificationDetails(ctx context.Context, dashboardId uint64, groupId uint64, epoch uint64, search string) (*t.NotificationAccountDashboardDetail, error) { - return getDummyStruct[t.NotificationAccountDashboardDetail]() + return getDummyStruct[t.NotificationAccountDashboardDetail](ctx) } func (d *DummyService) GetMachineNotifications(ctx context.Context, userId uint64, cursor string, colSort t.Sort[enums.NotificationMachinesColumn], search string, limit uint64) ([]t.NotificationMachinesTableRow, *t.Paging, error) { - return getDummyWithPaging[t.NotificationMachinesTableRow]() + return getDummyWithPaging[t.NotificationMachinesTableRow](ctx) } func (d *DummyService) GetClientNotifications(ctx context.Context, userId uint64, cursor string, colSort t.Sort[enums.NotificationClientsColumn], search string, limit uint64) ([]t.NotificationClientsTableRow, *t.Paging, error) { - return getDummyWithPaging[t.NotificationClientsTableRow]() + return getDummyWithPaging[t.NotificationClientsTableRow](ctx) } func (d *DummyService) GetRocketPoolNotifications(ctx context.Context, userId uint64, cursor string, colSort t.Sort[enums.NotificationRocketPoolColumn], search string, limit uint64) ([]t.NotificationRocketPoolTableRow, *t.Paging, error) { - return getDummyWithPaging[t.NotificationRocketPoolTableRow]() + return getDummyWithPaging[t.NotificationRocketPoolTableRow](ctx) } func (d *DummyService) GetNetworkNotifications(ctx context.Context, userId uint64, cursor string, colSort t.Sort[enums.NotificationNetworksColumn], limit uint64) ([]t.NotificationNetworksTableRow, *t.Paging, error) { - return getDummyWithPaging[t.NotificationNetworksTableRow]() + return getDummyWithPaging[t.NotificationNetworksTableRow](ctx) } func (d *DummyService) GetNotificationSettings(ctx context.Context, userId uint64) (*t.NotificationSettings, error) { - return getDummyStruct[t.NotificationSettings]() + return getDummyStruct[t.NotificationSettings](ctx) } func (d *DummyService) GetNotificationSettingsDefaultValues(ctx context.Context) (*t.NotificationSettingsDefaultValues, error) { - return getDummyStruct[t.NotificationSettingsDefaultValues]() + return getDummyStruct[t.NotificationSettingsDefaultValues](ctx) } func (d *DummyService) UpdateNotificationSettingsGeneral(ctx context.Context, userId uint64, settings t.NotificationSettingsGeneral) error { return nil @@ -583,11 +579,11 @@ func (d *DummyService) DeleteNotificationSettingsPairedDevice(ctx context.Contex } func (d *DummyService) UpdateNotificationSettingsClients(ctx context.Context, userId uint64, clientId uint64, IsSubscribed bool) (*t.NotificationSettingsClient, error) { - return getDummyStruct[t.NotificationSettingsClient]() + return getDummyStruct[t.NotificationSettingsClient](ctx) } func (d *DummyService) GetNotificationSettingsDashboards(ctx context.Context, userId uint64, cursor string, colSort t.Sort[enums.NotificationSettingsDashboardColumn], search string, limit uint64) ([]t.NotificationSettingsDashboardsTableRow, *t.Paging, error) { - r, p, err := getDummyWithPaging[t.NotificationSettingsDashboardsTableRow]() + r, p, err := getDummyWithPaging[t.NotificationSettingsDashboardsTableRow](ctx) for i, n := range r { var settings interface{} if n.IsAccountDashboard { @@ -595,7 +591,7 @@ func (d *DummyService) GetNotificationSettingsDashboards(ctx context.Context, us } else { settings = t.NotificationSettingsValidatorDashboard{} } - _ = commonFakeData(&settings) + _ = populateWithFakeData(ctx, &settings) r[i].Settings = settings } return r, p, err @@ -611,7 +607,7 @@ func (d *DummyService) CreateAdConfiguration(ctx context.Context, key, jquerySel } func (d *DummyService) GetAdConfigurations(ctx context.Context, keys []string) ([]t.AdConfigurationData, error) { - return getDummyData[[]t.AdConfigurationData]() + return getDummyData[[]t.AdConfigurationData](ctx) } func (d *DummyService) UpdateAdConfiguration(ctx context.Context, key, jquerySelector string, insertMode enums.AdInsertMode, refreshInterval uint64, forAllUsers bool, bannerId uint64, htmlContent string, enabled bool) error { @@ -623,128 +619,128 @@ func (d *DummyService) RemoveAdConfiguration(ctx context.Context, key string) er } func (d *DummyService) GetLatestExportedChartTs(ctx context.Context, aggregation enums.ChartAggregation) (uint64, error) { - return getDummyData[uint64]() + return getDummyData[uint64](ctx) } -func (d *DummyService) GetUserIdByRefreshToken(claimUserID, claimAppID, claimDeviceID uint64, hashedRefreshToken string) (uint64, error) { - return getDummyData[uint64]() +func (d *DummyService) GetUserIdByRefreshToken(ctx context.Context, claimUserID, claimAppID, claimDeviceID uint64, hashedRefreshToken string) (uint64, error) { + return getDummyData[uint64](ctx) } -func (d *DummyService) MigrateMobileSession(oldHashedRefreshToken, newHashedRefreshToken, deviceID, deviceName string) error { +func (d *DummyService) MigrateMobileSession(ctx context.Context, oldHashedRefreshToken, newHashedRefreshToken, deviceID, deviceName string) error { return nil } -func (d *DummyService) GetAppDataFromRedirectUri(callback string) (*t.OAuthAppData, error) { - return getDummyStruct[t.OAuthAppData]() +func (d *DummyService) GetAppDataFromRedirectUri(ctx context.Context, callback string) (*t.OAuthAppData, error) { + return getDummyStruct[t.OAuthAppData](ctx) } -func (d *DummyService) AddUserDevice(userID uint64, hashedRefreshToken string, deviceID, deviceName string, appID uint64) error { +func (d *DummyService) AddUserDevice(ctx context.Context, userID uint64, hashedRefreshToken string, deviceID, deviceName string, appID uint64) error { return nil } -func (d *DummyService) AddMobileNotificationToken(userID uint64, deviceID, notifyToken string) error { +func (d *DummyService) AddMobileNotificationToken(ctx context.Context, userID uint64, deviceID, notifyToken string) error { return nil } -func (d *DummyService) GetAppSubscriptionCount(userID uint64) (uint64, error) { - return getDummyData[uint64]() +func (d *DummyService) GetAppSubscriptionCount(ctx context.Context, userID uint64) (uint64, error) { + return getDummyData[uint64](ctx) } -func (d *DummyService) AddMobilePurchase(tx *sql.Tx, userID uint64, paymentDetails t.MobileSubscription, verifyResponse *userservice.VerifyResponse, extSubscriptionId string) error { +func (d *DummyService) AddMobilePurchase(ctx context.Context, tx *sql.Tx, userID uint64, paymentDetails t.MobileSubscription, verifyResponse *userservice.VerifyResponse, extSubscriptionId string) error { return nil } func (d *DummyService) GetBlockOverview(ctx context.Context, chainId, block uint64) (*t.BlockOverview, error) { - return getDummyStruct[t.BlockOverview]() + return getDummyStruct[t.BlockOverview](ctx) } func (d *DummyService) GetBlockTransactions(ctx context.Context, chainId, block uint64) ([]t.BlockTransactionTableRow, error) { - return getDummyData[[]t.BlockTransactionTableRow]() + return getDummyData[[]t.BlockTransactionTableRow](ctx) } func (d *DummyService) GetBlock(ctx context.Context, chainId, block uint64) (*t.BlockSummary, error) { - return getDummyStruct[t.BlockSummary]() + return getDummyStruct[t.BlockSummary](ctx) } func (d *DummyService) GetBlockVotes(ctx context.Context, chainId, block uint64) ([]t.BlockVoteTableRow, error) { - return getDummyData[[]t.BlockVoteTableRow]() + return getDummyData[[]t.BlockVoteTableRow](ctx) } func (d *DummyService) GetBlockAttestations(ctx context.Context, chainId, block uint64) ([]t.BlockAttestationTableRow, error) { - return getDummyData[[]t.BlockAttestationTableRow]() + return getDummyData[[]t.BlockAttestationTableRow](ctx) } func (d *DummyService) GetBlockWithdrawals(ctx context.Context, chainId, block uint64) ([]t.BlockWithdrawalTableRow, error) { - return getDummyData[[]t.BlockWithdrawalTableRow]() + return getDummyData[[]t.BlockWithdrawalTableRow](ctx) } func (d *DummyService) GetBlockBlsChanges(ctx context.Context, chainId, block uint64) ([]t.BlockBlsChangeTableRow, error) { - return getDummyData[[]t.BlockBlsChangeTableRow]() + return getDummyData[[]t.BlockBlsChangeTableRow](ctx) } func (d *DummyService) GetBlockVoluntaryExits(ctx context.Context, chainId, block uint64) ([]t.BlockVoluntaryExitTableRow, error) { - return getDummyData[[]t.BlockVoluntaryExitTableRow]() + return getDummyData[[]t.BlockVoluntaryExitTableRow](ctx) } func (d *DummyService) GetBlockBlobs(ctx context.Context, chainId, block uint64) ([]t.BlockBlobTableRow, error) { - return getDummyData[[]t.BlockBlobTableRow]() + return getDummyData[[]t.BlockBlobTableRow](ctx) } func (d *DummyService) GetSlot(ctx context.Context, chainId, block uint64) (*t.BlockSummary, error) { - return getDummyStruct[t.BlockSummary]() + return getDummyStruct[t.BlockSummary](ctx) } func (d *DummyService) GetSlotOverview(ctx context.Context, chainId, block uint64) (*t.BlockOverview, error) { - return getDummyStruct[t.BlockOverview]() + return getDummyStruct[t.BlockOverview](ctx) } func (d *DummyService) GetSlotTransactions(ctx context.Context, chainId, block uint64) ([]t.BlockTransactionTableRow, error) { - return getDummyData[[]t.BlockTransactionTableRow]() + return getDummyData[[]t.BlockTransactionTableRow](ctx) } func (d *DummyService) GetSlotVotes(ctx context.Context, chainId, block uint64) ([]t.BlockVoteTableRow, error) { - return getDummyData[[]t.BlockVoteTableRow]() + return getDummyData[[]t.BlockVoteTableRow](ctx) } func (d *DummyService) GetSlotAttestations(ctx context.Context, chainId, block uint64) ([]t.BlockAttestationTableRow, error) { - return getDummyData[[]t.BlockAttestationTableRow]() + return getDummyData[[]t.BlockAttestationTableRow](ctx) } func (d *DummyService) GetSlotWithdrawals(ctx context.Context, chainId, block uint64) ([]t.BlockWithdrawalTableRow, error) { - return getDummyData[[]t.BlockWithdrawalTableRow]() + return getDummyData[[]t.BlockWithdrawalTableRow](ctx) } func (d *DummyService) GetSlotBlsChanges(ctx context.Context, chainId, block uint64) ([]t.BlockBlsChangeTableRow, error) { - return getDummyData[[]t.BlockBlsChangeTableRow]() + return getDummyData[[]t.BlockBlsChangeTableRow](ctx) } func (d *DummyService) GetSlotVoluntaryExits(ctx context.Context, chainId, block uint64) ([]t.BlockVoluntaryExitTableRow, error) { - return getDummyData[[]t.BlockVoluntaryExitTableRow]() + return getDummyData[[]t.BlockVoluntaryExitTableRow](ctx) } func (d *DummyService) GetSlotBlobs(ctx context.Context, chainId, block uint64) ([]t.BlockBlobTableRow, error) { - return getDummyData[[]t.BlockBlobTableRow]() + return getDummyData[[]t.BlockBlobTableRow](ctx) } func (d *DummyService) GetValidatorDashboardsCountInfo(ctx context.Context) (map[uint64][]t.ArchiverDashboard, error) { - return getDummyData[map[uint64][]t.ArchiverDashboard]() + return getDummyData[map[uint64][]t.ArchiverDashboard](ctx) } func (d *DummyService) GetRocketPoolOverview(ctx context.Context) (*t.RocketPoolData, error) { - return getDummyStruct[t.RocketPoolData]() + return getDummyStruct[t.RocketPoolData](ctx) } func (d *DummyService) GetApiWeights(ctx context.Context) ([]t.ApiWeightItem, error) { - return getDummyData[[]t.ApiWeightItem]() + return getDummyData[[]t.ApiWeightItem](ctx) } func (d *DummyService) GetHealthz(ctx context.Context, showAll bool) t.HealthzData { - r, _ := getDummyData[t.HealthzData]() + r, _ := getDummyData[t.HealthzData](ctx) return r } func (d *DummyService) GetLatestBundleForNativeVersion(ctx context.Context, nativeVersion uint64) (*t.MobileAppBundleStats, error) { - return getDummyStruct[t.MobileAppBundleStats]() + return getDummyStruct[t.MobileAppBundleStats](ctx) } func (d *DummyService) IncrementBundleDeliveryCount(ctx context.Context, bundleVerison uint64) error { @@ -752,11 +748,11 @@ func (d *DummyService) IncrementBundleDeliveryCount(ctx context.Context, bundleV } func (d *DummyService) GetValidatorDashboardMobileWidget(ctx context.Context, dashboardId t.VDBIdPrimary) (*t.MobileWidgetData, error) { - return getDummyStruct[t.MobileWidgetData]() + return getDummyStruct[t.MobileWidgetData](ctx) } func (d *DummyService) GetUserMachineMetrics(ctx context.Context, userID uint64, limit int, offset int) (*t.MachineMetricsData, error) { - data, err := getDummyStruct[t.MachineMetricsData]() + data, err := getDummyStruct[t.MachineMetricsData](ctx) if err != nil { return nil, err } @@ -777,7 +773,7 @@ func (d *DummyService) PostUserMachineMetrics(ctx context.Context, userID uint64 } func (d *DummyService) GetValidatorDashboardMobileValidators(ctx context.Context, dashboardId t.VDBId, period enums.TimePeriod, cursor string, colSort t.Sort[enums.VDBMobileValidatorsColumn], search string, limit uint64) ([]t.MobileValidatorDashboardValidatorsTableRow, *t.Paging, error) { - return getDummyWithPaging[t.MobileValidatorDashboardValidatorsTableRow]() + return getDummyWithPaging[t.MobileValidatorDashboardValidatorsTableRow](ctx) } func (d *DummyService) QueueTestEmailNotification(ctx context.Context, userId uint64) error { diff --git a/backend/pkg/api/data_access/header.go b/backend/pkg/api/data_access/header.go index 1a91ad832..d06b41439 100644 --- a/backend/pkg/api/data_access/header.go +++ b/backend/pkg/api/data_access/header.go @@ -12,24 +12,24 @@ import ( "github.com/gobitfly/beaconchain/pkg/commons/utils" ) -func (d *DataAccessService) GetLatestSlot() (uint64, error) { +func (d *DataAccessService) GetLatestSlot(ctx context.Context) (uint64, error) { latestSlot := cache.LatestSlot.Get() return latestSlot, nil } -func (d *DataAccessService) GetLatestFinalizedEpoch() (uint64, error) { +func (d *DataAccessService) GetLatestFinalizedEpoch(ctx context.Context) (uint64, error) { finalizedEpoch := cache.LatestFinalizedEpoch.Get() return finalizedEpoch, nil } -func (d *DataAccessService) GetLatestBlock() (uint64, error) { +func (d *DataAccessService) GetLatestBlock(ctx context.Context) (uint64, error) { // @DATA-ACCESS implement - return d.dummy.GetLatestBlock() + return d.dummy.GetLatestBlock(ctx) } -func (d *DataAccessService) GetBlockHeightAt(slot uint64) (uint64, error) { +func (d *DataAccessService) GetBlockHeightAt(ctx context.Context, slot uint64) (uint64, error) { // @DATA-ACCESS implement; return error if no block at slot - return d.dummy.GetBlockHeightAt(slot) + return getDummyData[uint64](ctx) } // returns the block number of the latest existing block at or before the given slot @@ -69,7 +69,7 @@ func (d *DataAccessService) GetLatestBlockHeightsForEpoch(ctx context.Context, e return res, nil } -func (d *DataAccessService) GetLatestExchangeRates() ([]t.EthConversionRate, error) { +func (d *DataAccessService) GetLatestExchangeRates(ctx context.Context) ([]t.EthConversionRate, error) { result := []t.EthConversionRate{} availableCurrencies := price.GetAvailableCurrencies() diff --git a/backend/pkg/api/data_access/mobile.go b/backend/pkg/api/data_access/mobile.go index dc526746b..373688439 100644 --- a/backend/pkg/api/data_access/mobile.go +++ b/backend/pkg/api/data_access/mobile.go @@ -18,25 +18,25 @@ import ( ) type AppRepository interface { - GetUserIdByRefreshToken(claimUserID, claimAppID, claimDeviceID uint64, hashedRefreshToken string) (uint64, error) - MigrateMobileSession(oldHashedRefreshToken, newHashedRefreshToken, deviceID, deviceName string) error - AddUserDevice(userID uint64, hashedRefreshToken string, deviceID, deviceName string, appID uint64) error - GetAppDataFromRedirectUri(callback string) (*t.OAuthAppData, error) - AddMobileNotificationToken(userID uint64, deviceID, notifyToken string) error - GetAppSubscriptionCount(userID uint64) (uint64, error) - AddMobilePurchase(tx *sql.Tx, userID uint64, paymentDetails t.MobileSubscription, verifyResponse *userservice.VerifyResponse, extSubscriptionId string) error + GetUserIdByRefreshToken(ctx context.Context, claimUserID, claimAppID, claimDeviceID uint64, hashedRefreshToken string) (uint64, error) + MigrateMobileSession(ctx context.Context, oldHashedRefreshToken, newHashedRefreshToken, deviceID, deviceName string) error + AddUserDevice(ctx context.Context, userID uint64, hashedRefreshToken string, deviceID, deviceName string, appID uint64) error + GetAppDataFromRedirectUri(ctx context.Context, callback string) (*t.OAuthAppData, error) + AddMobileNotificationToken(ctx context.Context, userID uint64, deviceID, notifyToken string) error + GetAppSubscriptionCount(ctx context.Context, userID uint64) (uint64, error) + AddMobilePurchase(ctx context.Context, tx *sql.Tx, userID uint64, paymentDetails t.MobileSubscription, verifyResponse *userservice.VerifyResponse, extSubscriptionId string) error GetLatestBundleForNativeVersion(ctx context.Context, nativeVersion uint64) (*t.MobileAppBundleStats, error) IncrementBundleDeliveryCount(ctx context.Context, bundleVerison uint64) error GetValidatorDashboardMobileValidators(ctx context.Context, dashboardId t.VDBId, period enums.TimePeriod, cursor string, colSort t.Sort[enums.VDBMobileValidatorsColumn], search string, limit uint64) ([]t.MobileValidatorDashboardValidatorsTableRow, *t.Paging, error) } // GetUserIdByRefreshToken basically used to confirm the claimed user id with the refresh token. Returns the userId if successful -func (d *DataAccessService) GetUserIdByRefreshToken(claimUserID, claimAppID, claimDeviceID uint64, hashedRefreshToken string) (uint64, error) { +func (d *DataAccessService) GetUserIdByRefreshToken(ctx context.Context, claimUserID, claimAppID, claimDeviceID uint64, hashedRefreshToken string) (uint64, error) { if hashedRefreshToken == "" { // sanity return 0, errors.New("empty refresh token") } var userID uint64 - err := d.userWriter.Get(&userID, + err := d.userWriter.GetContext(ctx, &userID, `SELECT user_id FROM users_devices WHERE user_id = $1 AND refresh_token = $2 AND app_id = $3 AND id = $4 AND active = true`, claimUserID, hashedRefreshToken, claimAppID, claimDeviceID) if errors.Is(err, sql.ErrNoRows) { @@ -45,8 +45,8 @@ func (d *DataAccessService) GetUserIdByRefreshToken(claimUserID, claimAppID, cla return userID, err } -func (d *DataAccessService) MigrateMobileSession(oldHashedRefreshToken, newHashedRefreshToken, deviceID, deviceName string) error { - result, err := d.userWriter.Exec("UPDATE users_devices SET refresh_token = $2, device_identifier = $3, device_name = $4 WHERE refresh_token = $1", oldHashedRefreshToken, newHashedRefreshToken, deviceID, deviceName) +func (d *DataAccessService) MigrateMobileSession(ctx context.Context, oldHashedRefreshToken, newHashedRefreshToken, deviceID, deviceName string) error { + result, err := d.userWriter.ExecContext(ctx, "UPDATE users_devices SET refresh_token = $2, device_identifier = $3, device_name = $4 WHERE refresh_token = $1", oldHashedRefreshToken, newHashedRefreshToken, deviceID, deviceName) if err != nil { return errors.Wrap(err, "Error updating refresh token") } @@ -63,21 +63,21 @@ func (d *DataAccessService) MigrateMobileSession(oldHashedRefreshToken, newHashe return err } -func (d *DataAccessService) GetAppDataFromRedirectUri(callback string) (*t.OAuthAppData, error) { +func (d *DataAccessService) GetAppDataFromRedirectUri(ctx context.Context, callback string) (*t.OAuthAppData, error) { data := t.OAuthAppData{} - err := d.userWriter.Get(&data, "SELECT id, app_name, redirect_uri, active, owner_id FROM oauth_apps WHERE active = true AND redirect_uri = $1", callback) + err := d.userWriter.GetContext(ctx, &data, "SELECT id, app_name, redirect_uri, active, owner_id FROM oauth_apps WHERE active = true AND redirect_uri = $1", callback) return &data, err } -func (d *DataAccessService) AddUserDevice(userID uint64, hashedRefreshToken string, deviceID, deviceName string, appID uint64) error { - _, err := d.userWriter.Exec("INSERT INTO users_devices (user_id, refresh_token, device_identifier, device_name, app_id, created_ts) VALUES($1, $2, $3, $4, $5, 'NOW()') ON CONFLICT DO NOTHING", +func (d *DataAccessService) AddUserDevice(ctx context.Context, userID uint64, hashedRefreshToken string, deviceID, deviceName string, appID uint64) error { + _, err := d.userWriter.ExecContext(ctx, "INSERT INTO users_devices (user_id, refresh_token, device_identifier, device_name, app_id, created_ts) VALUES($1, $2, $3, $4, $5, 'NOW()') ON CONFLICT DO NOTHING", userID, hashedRefreshToken, deviceID, deviceName, appID, ) return err } -func (d *DataAccessService) AddMobileNotificationToken(userID uint64, deviceID, notifyToken string) error { - _, err := d.userWriter.Exec("UPDATE users_devices SET notification_token = $1 WHERE user_id = $2 AND device_identifier = $3;", +func (d *DataAccessService) AddMobileNotificationToken(ctx context.Context, userID uint64, deviceID, notifyToken string) error { + _, err := d.userWriter.ExecContext(ctx, "UPDATE users_devices SET notification_token = $1 WHERE user_id = $2 AND device_identifier = $3;", notifyToken, userID, deviceID, ) if errors.Is(err, sql.ErrNoRows) { @@ -86,13 +86,13 @@ func (d *DataAccessService) AddMobileNotificationToken(userID uint64, deviceID, return err } -func (d *DataAccessService) GetAppSubscriptionCount(userID uint64) (uint64, error) { +func (d *DataAccessService) GetAppSubscriptionCount(ctx context.Context, userID uint64) (uint64, error) { var count uint64 - err := d.userReader.Get(&count, "SELECT COUNT(receipt) FROM users_app_subscriptions WHERE user_id = $1", userID) + err := d.userReader.GetContext(ctx, &count, "SELECT COUNT(receipt) FROM users_app_subscriptions WHERE user_id = $1", userID) return count, err } -func (d *DataAccessService) AddMobilePurchase(tx *sql.Tx, userID uint64, paymentDetails t.MobileSubscription, verifyResponse *userservice.VerifyResponse, extSubscriptionId string) error { +func (d *DataAccessService) AddMobilePurchase(ctx context.Context, tx *sql.Tx, userID uint64, paymentDetails t.MobileSubscription, verifyResponse *userservice.VerifyResponse, extSubscriptionId string) error { now := time.Now() nowTs := now.Unix() receiptHash := utils.HashAndEncode(verifyResponse.Receipt) @@ -103,11 +103,11 @@ func (d *DataAccessService) AddMobilePurchase(tx *sql.Tx, userID uint64, payment ON CONFLICT(receipt_hash) DO UPDATE SET product_id = $2, active = $7, updated_at = TO_TIMESTAMP($5);` var err error if tx == nil { - _, err = d.userWriter.Exec(query, + _, err = d.userWriter.ExecContext(ctx, query, userID, verifyResponse.ProductID, paymentDetails.PriceMicros, paymentDetails.Currency, nowTs, nowTs, verifyResponse.Valid, verifyResponse.Valid, paymentDetails.Transaction.Type, verifyResponse.Receipt, verifyResponse.ExpirationDate, verifyResponse.RejectReason, receiptHash, extSubscriptionId, ) } else { - _, err = tx.Exec(query, + _, err = tx.ExecContext(ctx, query, userID, verifyResponse.ProductID, paymentDetails.PriceMicros, paymentDetails.Currency, nowTs, nowTs, verifyResponse.Valid, verifyResponse.Valid, paymentDetails.Transaction.Type, verifyResponse.Receipt, verifyResponse.ExpirationDate, verifyResponse.RejectReason, receiptHash, extSubscriptionId, ) } @@ -117,7 +117,7 @@ func (d *DataAccessService) AddMobilePurchase(tx *sql.Tx, userID uint64, payment func (d *DataAccessService) GetLatestBundleForNativeVersion(ctx context.Context, nativeVersion uint64) (*t.MobileAppBundleStats, error) { var bundle t.MobileAppBundleStats - err := d.userReader.Get(&bundle, ` + err := d.userReader.GetContext(ctx, &bundle, ` WITH latest_native AS ( SELECT max(min_native_version) as max_native_version @@ -152,7 +152,7 @@ func (d *DataAccessService) GetLatestBundleForNativeVersion(ctx context.Context, } func (d *DataAccessService) IncrementBundleDeliveryCount(ctx context.Context, bundleVersion uint64) error { - _, err := d.userWriter.Exec("UPDATE mobile_app_bundles SET delivered_count = COALESCE(delivered_count, 0) + 1 WHERE bundle_version = $1", bundleVersion) + _, err := d.userWriter.ExecContext(ctx, "UPDATE mobile_app_bundles SET delivered_count = COALESCE(delivered_count, 0) + 1 WHERE bundle_version = $1", bundleVersion) return err } @@ -209,7 +209,7 @@ func (d *DataAccessService) GetValidatorDashboardMobileWidget(ctx context.Contex // RPL eg.Go(func() error { - rpNetworkStats, err := d.internal_rp_network_stats() + rpNetworkStats, err := d.getInternalRpNetworkStats(ctx) if err != nil { return fmt.Errorf("error retrieving rocketpool network stats: %w", err) } @@ -347,9 +347,9 @@ func (d *DataAccessService) GetValidatorDashboardMobileWidget(ctx context.Contex return &data, nil } -func (d *DataAccessService) internal_rp_network_stats() (*t.RPNetworkStats, error) { +func (d *DataAccessService) getInternalRpNetworkStats(ctx context.Context) (*t.RPNetworkStats, error) { var networkStats t.RPNetworkStats - err := d.alloyReader.Get(&networkStats, ` + err := d.alloyReader.GetContext(ctx, &networkStats, ` SELECT EXTRACT(EPOCH FROM claim_interval_time) / 3600 AS claim_interval_hours, node_operator_rewards, diff --git a/backend/pkg/api/data_access/notifications.go b/backend/pkg/api/data_access/notifications.go index 028d325da..5c9adc406 100644 --- a/backend/pkg/api/data_access/notifications.go +++ b/backend/pkg/api/data_access/notifications.go @@ -129,7 +129,7 @@ func (d *DataAccessService) GetNotificationOverview(ctx context.Context, userId }) // most notified groups - latestSlot, err := d.GetLatestSlot() + latestSlot, err := d.GetLatestSlot(ctx) if err != nil { return nil, err } diff --git a/backend/pkg/api/data_access/vdb_management.go b/backend/pkg/api/data_access/vdb_management.go index d152d6cac..91d15a24b 100644 --- a/backend/pkg/api/data_access/vdb_management.go +++ b/backend/pkg/api/data_access/vdb_management.go @@ -168,7 +168,7 @@ func (d *DataAccessService) GetValidatorDashboardName(ctx context.Context, dashb } // param validators: slice of validator public keys or indices -func (d *DataAccessService) GetValidatorsFromSlices(indices []t.VDBValidator, publicKeys []string) ([]t.VDBValidator, error) { +func (d *DataAccessService) GetValidatorsFromSlices(ctx context.Context, indices []t.VDBValidator, publicKeys []string) ([]t.VDBValidator, error) { if len(indices) == 0 && len(publicKeys) == 0 { return []t.VDBValidator{}, nil } diff --git a/backend/pkg/api/handlers/auth.go b/backend/pkg/api/handlers/auth.go index 3f3c2f3b3..d4e0ba44b 100644 --- a/backend/pkg/api/handlers/auth.go +++ b/backend/pkg/api/handlers/auth.go @@ -528,7 +528,7 @@ func (h *HandlerService) InternalPostMobileAuthorize(w http.ResponseWriter, r *h } // check if oauth app exists to validate whether redirect uri is valid - appInfo, err := h.daService.GetAppDataFromRedirectUri(req.RedirectURI) + appInfo, err := h.daService.GetAppDataFromRedirectUri(r.Context(), req.RedirectURI) if err != nil { callback := req.RedirectURI + "?error=invalid_request&error_description=missing_redirect_uri" + state http.Redirect(w, r, callback, http.StatusSeeOther) @@ -545,7 +545,7 @@ func (h *HandlerService) InternalPostMobileAuthorize(w http.ResponseWriter, r *h session := h.scs.Token(r.Context()) sanitizedDeviceName := html.EscapeString(clientName) - err = h.daService.AddUserDevice(userInfo.Id, utils.HashAndEncode(session+session), clientID, sanitizedDeviceName, appInfo.ID) + err = h.daService.AddUserDevice(r.Context(), userInfo.Id, utils.HashAndEncode(session+session), clientID, sanitizedDeviceName, appInfo.ID) if err != nil { log.Warnf("Error adding user device: %v", err) callback := req.RedirectURI + "?error=invalid_request&error_description=server_error" + state @@ -608,7 +608,7 @@ func (h *HandlerService) InternalPostMobileEquivalentExchange(w http.ResponseWri // invalidate old refresh token and replace with hashed session id sanitizedDeviceName := html.EscapeString(req.DeviceName) - err = h.daService.MigrateMobileSession(refreshTokenHashed, utils.HashAndEncode(session+session), req.DeviceID, sanitizedDeviceName) // salted with session + err = h.daService.MigrateMobileSession(r.Context(), refreshTokenHashed, utils.HashAndEncode(session+session), req.DeviceID, sanitizedDeviceName) // salted with session if err != nil { handleErr(w, r, err) return @@ -649,7 +649,7 @@ func (h *HandlerService) InternalPostUsersMeNotificationSettingsPairedDevicesTok return } - err = h.daService.AddMobileNotificationToken(user.Id, deviceID, req.Token) + err = h.daService.AddMobileNotificationToken(r.Context(), user.Id, deviceID, req.Token) if err != nil { handleErr(w, r, err) return @@ -689,7 +689,7 @@ func (h *HandlerService) InternalHandleMobilePurchase(w http.ResponseWriter, r * return } - subscriptionCount, err := h.daService.GetAppSubscriptionCount(user.Id) + subscriptionCount, err := h.daService.GetAppSubscriptionCount(r.Context(), user.Id) if err != nil { handleErr(w, r, err) return @@ -720,7 +720,7 @@ func (h *HandlerService) InternalHandleMobilePurchase(w http.ResponseWriter, r * } } - err = h.daService.AddMobilePurchase(nil, user.Id, req, validationResult, "") + err = h.daService.AddMobilePurchase(r.Context(), nil, user.Id, req, validationResult, "") if err != nil { handleErr(w, r, err) return diff --git a/backend/pkg/api/handlers/backward_compat.go b/backend/pkg/api/handlers/backward_compat.go index e9d823c69..8b8e69442 100644 --- a/backend/pkg/api/handlers/backward_compat.go +++ b/backend/pkg/api/handlers/backward_compat.go @@ -43,7 +43,7 @@ func (h *HandlerService) getTokenByRefresh(r *http.Request, refreshToken string) log.Infof("refresh token: %v, claims: %v, hashed refresh: %v", refreshToken, unsafeClaims, refreshTokenHashed) // confirm all claims via db lookup and refreshtoken check - userID, err := h.daService.GetUserIdByRefreshToken(unsafeClaims.UserID, unsafeClaims.AppID, unsafeClaims.DeviceID, refreshTokenHashed) + userID, err := h.daService.GetUserIdByRefreshToken(r.Context(), unsafeClaims.UserID, unsafeClaims.AppID, unsafeClaims.DeviceID, refreshTokenHashed) if err != nil { if err == sql.ErrNoRows { return 0, "", dataaccess.ErrNotFound diff --git a/backend/pkg/api/handlers/handler_service.go b/backend/pkg/api/handlers/handler_service.go index 71a43e9ad..fd8c5734e 100644 --- a/backend/pkg/api/handlers/handler_service.go +++ b/backend/pkg/api/handlers/handler_service.go @@ -119,7 +119,7 @@ func (h *HandlerService) getDashboardId(ctx context.Context, dashboardIdParam in } return &types.VDBId{Id: types.VDBIdPrimary(dashboardInfo.DashboardId), Validators: nil, AggregateGroups: !dashboardInfo.ShareSettings.ShareGroups}, nil case validatorSet: - validators, err := h.daService.GetValidatorsFromSlices(dashboardId.Indexes, dashboardId.PublicKeys) + validators, err := h.daService.GetValidatorsFromSlices(ctx, dashboardId.Indexes, dashboardId.PublicKeys) if err != nil { return nil, err } diff --git a/backend/pkg/api/handlers/input_validation.go b/backend/pkg/api/handlers/input_validation.go index eca791aa8..3545b0c63 100644 --- a/backend/pkg/api/handlers/input_validation.go +++ b/backend/pkg/api/handlers/input_validation.go @@ -269,10 +269,11 @@ func (h *HandlerService) validateBlockRequest(r *http.Request, paramName string) switch paramValue := mux.Vars(r)[paramName]; paramValue { // possibly add other values like "genesis", "finalized", hardforks etc. later case "latest": + ctx := r.Context() if paramName == "block" { - value, err = h.daService.GetLatestBlock() + value, err = h.daService.GetLatestBlock(ctx) } else if paramName == "slot" { - value, err = h.daService.GetLatestSlot() + value, err = h.daService.GetLatestSlot(ctx) } if err != nil { return 0, 0, err diff --git a/backend/pkg/api/handlers/internal.go b/backend/pkg/api/handlers/internal.go index 3093352f9..83bb4faf2 100644 --- a/backend/pkg/api/handlers/internal.go +++ b/backend/pkg/api/handlers/internal.go @@ -44,19 +44,20 @@ func (h *HandlerService) InternalGetRatelimitWeights(w http.ResponseWriter, r *h // Latest State func (h *HandlerService) InternalGetLatestState(w http.ResponseWriter, r *http.Request) { - latestSlot, err := h.daService.GetLatestSlot() + ctx := r.Context() + latestSlot, err := h.daService.GetLatestSlot(ctx) if err != nil { handleErr(w, r, err) return } - finalizedEpoch, err := h.daService.GetLatestFinalizedEpoch() + finalizedEpoch, err := h.daService.GetLatestFinalizedEpoch(ctx) if err != nil { handleErr(w, r, err) return } - exchangeRates, err := h.daService.GetLatestExchangeRates() + exchangeRates, err := h.daService.GetLatestExchangeRates(ctx) if err != nil { handleErr(w, r, err) return diff --git a/backend/pkg/api/handlers/public.go b/backend/pkg/api/handlers/public.go index b35b5ec5f..5a6473f0e 100644 --- a/backend/pkg/api/handlers/public.go +++ b/backend/pkg/api/handlers/public.go @@ -589,7 +589,7 @@ func (h *HandlerService) PublicPostValidatorDashboardValidators(w http.ResponseW handleErr(w, r, v) return } - validators, err := h.getDataAccessor(r).GetValidatorsFromSlices(indices, pubkeys) + validators, err := h.getDataAccessor(r).GetValidatorsFromSlices(ctx, indices, pubkeys) if err != nil { handleErr(w, r, err) return @@ -637,7 +637,7 @@ func (h *HandlerService) PublicPostValidatorDashboardValidators(w http.ResponseW // PublicGetValidatorDashboardValidators godoc // -// @Description Get a list of groups in a specified validator dashboard. +// @Description Get a list of validators in a specified validator dashboard. // @Tags Validator Dashboard // @Produce json // @Param dashboard_id path string true "The ID of the dashboard." @@ -647,7 +647,7 @@ func (h *HandlerService) PublicPostValidatorDashboardValidators(w http.ResponseW // @Param search query string false "Search for Address, ENS." // @Success 200 {object} types.GetValidatorDashboardValidatorsResponse // @Failure 400 {object} types.ApiErrorResponse -// @Router /validator-dashboards/{dashboard_id}/groups [get] +// @Router /validator-dashboards/{dashboard_id}/validators [get] func (h *HandlerService) PublicGetValidatorDashboardValidators(w http.ResponseWriter, r *http.Request) { var v validationError dashboardId, err := h.handleDashboardId(r.Context(), mux.Vars(r)["dashboard_id"]) @@ -703,12 +703,13 @@ func (h *HandlerService) PublicDeleteValidatorDashboardValidators(w http.Respons handleErr(w, r, v) return } - validators, err := h.getDataAccessor(r).GetValidatorsFromSlices(indices, publicKeys) + ctx := r.Context() + validators, err := h.getDataAccessor(r).GetValidatorsFromSlices(ctx, indices, publicKeys) if err != nil { handleErr(w, r, err) return } - err = h.getDataAccessor(r).RemoveValidatorDashboardValidators(r.Context(), dashboardId, validators) + err = h.getDataAccessor(r).RemoveValidatorDashboardValidators(ctx, dashboardId, validators) if err != nil { handleErr(w, r, err) return From ff0a4c8f3c25b22068637e52a7494e7ab242fa96 Mon Sep 17 00:00:00 2001 From: Lucca Dukic <109136188+LuccaBitfly@users.noreply.github.com> Date: Thu, 24 Oct 2024 10:29:58 +0200 Subject: [PATCH 17/82] refactor: move context keys to types pkg See: BEDS-868 --- backend/pkg/api/handlers/auth.go | 7 +------ backend/pkg/api/handlers/handler_service.go | 4 ++-- backend/pkg/api/handlers/middlewares.go | 14 ++++---------- backend/pkg/api/types/data_access.go | 8 ++++++++ 4 files changed, 15 insertions(+), 18 deletions(-) diff --git a/backend/pkg/api/handlers/auth.go b/backend/pkg/api/handlers/auth.go index d4e0ba44b..357a47cb5 100644 --- a/backend/pkg/api/handlers/auth.go +++ b/backend/pkg/api/handlers/auth.go @@ -34,11 +34,6 @@ const authConfirmEmailRateLimit = time.Minute * 2 const authResetEmailRateLimit = time.Minute * 2 const authEmailExpireTime = time.Minute * 30 -type ctxKey string - -const ctxUserIdKey ctxKey = "user_id" -const ctxIsMockedKey ctxKey = "is_mocked" - var errBadCredentials = newUnauthorizedErr("invalid email or password") func (h *HandlerService) getUserBySession(r *http.Request) (types.UserCredentialInfo, error) { @@ -203,7 +198,7 @@ func (h *HandlerService) GetUserIdByApiKey(r *http.Request) (uint64, error) { // if this is used, user ID should've been stored in context (by GetUserIdStoreMiddleware) func GetUserIdByContext(r *http.Request) (uint64, error) { - userId, ok := r.Context().Value(ctxUserIdKey).(uint64) + userId, ok := r.Context().Value(types.CtxUserIdKey).(uint64) if !ok { return 0, newUnauthorizedErr("user not authenticated") } diff --git a/backend/pkg/api/handlers/handler_service.go b/backend/pkg/api/handlers/handler_service.go index fd8c5734e..6e76c91db 100644 --- a/backend/pkg/api/handlers/handler_service.go +++ b/backend/pkg/api/handlers/handler_service.go @@ -225,7 +225,7 @@ func getMaxChartAge(aggregation enums.ChartAggregation, perkSeconds types.ChartH } func isUserAdmin(user *types.UserInfo) bool { - if user == nil { + if user == nil { // can happen for guest or shared dashboards return false } return user.UserGroup == types.UserGroupAdmin @@ -542,6 +542,6 @@ func (intOrString) JSONSchema() *jsonschema.Schema { } func isMocked(r *http.Request) bool { - isMocked, ok := r.Context().Value(ctxIsMockedKey).(bool) + isMocked, ok := r.Context().Value(types.CtxIsMockedKey).(bool) return ok && isMocked } diff --git a/backend/pkg/api/handlers/middlewares.go b/backend/pkg/api/handlers/middlewares.go index 54238cc59..d59cb33f4 100644 --- a/backend/pkg/api/handlers/middlewares.go +++ b/backend/pkg/api/handlers/middlewares.go @@ -29,7 +29,7 @@ func StoreUserIdMiddleware(next http.Handler, userIdFunc func(r *http.Request) ( // store user id in context ctx := r.Context() - ctx = context.WithValue(ctx, ctxUserIdKey, userId) + ctx = context.WithValue(ctx, types.CtxUserIdKey, userId) r = r.WithContext(ctx) next.ServeHTTP(w, r) }) @@ -53,7 +53,7 @@ func (h *HandlerService) StoreUserIdByApiKeyMiddleware(next http.Handler) http.H func (h *HandlerService) VDBAuthMiddleware(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { // if mock data is used, no need to check access - if isMockEnabled, ok := r.Context().Value(ctxIsMockedKey).(bool); ok && isMockEnabled { + if isMocked, ok := r.Context().Value(types.CtxIsMockedKey).(bool); ok && isMocked { next.ServeHTTP(w, r) return } @@ -71,12 +71,6 @@ func (h *HandlerService) VDBAuthMiddleware(next http.Handler) http.Handler { handleErr(w, r, err) return } - - // store user id in context - ctx := r.Context() - ctx = context.WithValue(ctx, ctxUserIdKey, userId) - r = r.WithContext(ctx) - dashboardUser, err := h.daService.GetValidatorDashboardUser(r.Context(), types.VDBIdPrimary(dashboardId)) if err != nil { handleErr(w, r, err) @@ -138,7 +132,7 @@ func (h *HandlerService) ManageNotificationsViaApiCheckMiddleware(next http.Hand // middleware check to return if specified dashboard is not archived (and accessible) func (h *HandlerService) VDBArchivedCheckMiddleware(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if isMockEnabled, ok := r.Context().Value(ctxIsMockedKey).(bool); ok && isMockEnabled { + if isMocked, ok := r.Context().Value(types.CtxIsMockedKey).(bool); ok && isMocked { next.ServeHTTP(w, r) return } @@ -193,7 +187,7 @@ func (h *HandlerService) StoreIsMockedFlagMiddleware(next http.Handler) http.Han } // store isMocked flag in context ctx := r.Context() - ctx = context.WithValue(ctx, ctxIsMockedKey, true) + ctx = context.WithValue(ctx, types.CtxIsMockedKey, true) r = r.WithContext(ctx) next.ServeHTTP(w, r) }) diff --git a/backend/pkg/api/types/data_access.go b/backend/pkg/api/types/data_access.go index 6a2859f6f..00c909d12 100644 --- a/backend/pkg/api/types/data_access.go +++ b/backend/pkg/api/types/data_access.go @@ -322,3 +322,11 @@ type NotificationSettingsDefaultValues struct { GasBelowThreshold decimal.Decimal NetworkParticipationRateThreshold float64 } + +// ------------------------------ + +type CtxKey string + +const CtxUserIdKey CtxKey = "user_id" +const CtxIsMockedKey CtxKey = "is_mocked" +const CtxDashboardIdKey CtxKey = "dashboard_id" From e658294303a8ee595f14846cfb47c750cc69b0d7 Mon Sep 17 00:00:00 2001 From: Lucca Dukic <109136188+LuccaBitfly@users.noreply.github.com> Date: Thu, 24 Oct 2024 10:32:07 +0200 Subject: [PATCH 18/82] feat: store dashboard id in context in archiving middleware - avoids fetching from db twice when calling `handleDashboardId` both in archiving middleware and the actual handler See: BEDS-868 --- backend/pkg/api/handlers/handler_service.go | 4 ++++ backend/pkg/api/handlers/middlewares.go | 7 ++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/backend/pkg/api/handlers/handler_service.go b/backend/pkg/api/handlers/handler_service.go index 6e76c91db..167662579 100644 --- a/backend/pkg/api/handlers/handler_service.go +++ b/backend/pkg/api/handlers/handler_service.go @@ -138,6 +138,10 @@ func (h *HandlerService) getDashboardId(ctx context.Context, dashboardIdParam in // it should be used as the last validation step for all internal dashboard GET-handlers. // Modifying handlers (POST, PUT, DELETE) should only accept primary dashboard ids and just use checkPrimaryDashboardId. func (h *HandlerService) handleDashboardId(ctx context.Context, param string) (*types.VDBId, error) { + //check if dashboard id is stored in context + if dashboardId, ok := ctx.Value(types.CtxDashboardIdKey).(*types.VDBId); ok { + return dashboardId, nil + } // validate dashboard id param dashboardIdParam, err := parseDashboardId(param) if err != nil { diff --git a/backend/pkg/api/handlers/middlewares.go b/backend/pkg/api/handlers/middlewares.go index d59cb33f4..bac5f6822 100644 --- a/backend/pkg/api/handlers/middlewares.go +++ b/backend/pkg/api/handlers/middlewares.go @@ -141,7 +141,12 @@ func (h *HandlerService) VDBArchivedCheckMiddleware(next http.Handler) http.Hand handleErr(w, r, err) return } - if len(dashboardId.Validators) > 0 { + // store dashboard id in context + ctx := r.Context() + ctx = context.WithValue(ctx, types.CtxDashboardIdKey, dashboardId) + r = r.WithContext(ctx) + + if len(dashboardId.Validators) > 0 { // don't check guest dashboards next.ServeHTTP(w, r) return } From 9a1813340d1f61717fba865082ef5d1cf68b0952 Mon Sep 17 00:00:00 2001 From: Lucca Dukic <109136188+LuccaBitfly@users.noreply.github.com> Date: Thu, 24 Oct 2024 10:34:40 +0200 Subject: [PATCH 19/82] feat: implement mock_seed query param - allows for passind a seed when the is_mocked flag is being used, which will cause deterministic mocked data. - useful for some auth sensitive endpoints which rely on fetched data staying the same See: BEDS-868 --- backend/pkg/api/data_access/dummy.go | 11 +++++++++++ backend/pkg/api/handlers/middlewares.go | 15 ++++++++++++++- backend/pkg/api/types/data_access.go | 1 + 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/backend/pkg/api/data_access/dummy.go b/backend/pkg/api/data_access/dummy.go index 387ea5c0d..89f915e46 100644 --- a/backend/pkg/api/data_access/dummy.go +++ b/backend/pkg/api/data_access/dummy.go @@ -7,8 +7,11 @@ import ( "math/rand/v2" "reflect" "slices" + "sync" "time" + mathrand "math/rand" + "github.com/go-faker/faker/v4" "github.com/go-faker/faker/v4/pkg/interfaces" "github.com/go-faker/faker/v4/pkg/options" @@ -52,8 +55,16 @@ func randomEthDecimal() decimal.Decimal { return decimal } +var mockLock sync.Mutex = sync.Mutex{} + // must pass a pointer to the data func populateWithFakeData(ctx context.Context, a interface{}) error { + if seed, ok := ctx.Value(t.CtxMockSeedKey).(int64); ok { + mockLock.Lock() + defer mockLock.Unlock() + faker.SetRandomSource(mathrand.NewSource(seed)) + } + return faker.FakeData(a, options.WithRandomMapAndSliceMaxSize(10), options.WithRandomFloatBoundaries(interfaces.RandomFloatBoundary{Start: 0, End: 1})) } diff --git a/backend/pkg/api/handlers/middlewares.go b/backend/pkg/api/handlers/middlewares.go index bac5f6822..f24ae39a7 100644 --- a/backend/pkg/api/handlers/middlewares.go +++ b/backend/pkg/api/handlers/middlewares.go @@ -169,7 +169,17 @@ func (h *HandlerService) VDBArchivedCheckMiddleware(next http.Handler) http.Hand // note that mocked data is only returned by handlers that check for it. func (h *HandlerService) StoreIsMockedFlagMiddleware(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - isMocked, _ := strconv.ParseBool(r.URL.Query().Get("is_mocked")) + var v validationError + q := r.URL.Query() + isMocked := v.checkBool(q.Get("is_mocked"), "is_mocked") + var mockSeed int64 + if mockSeedStr := q.Get("mock_seed"); mockSeedStr != "" { + mockSeed = v.checkInt(mockSeedStr, "mock_seed") + } + if v.hasErrors() { + handleErr(w, r, v) + return + } if !isMocked { next.ServeHTTP(w, r) return @@ -193,6 +203,9 @@ func (h *HandlerService) StoreIsMockedFlagMiddleware(next http.Handler) http.Han // store isMocked flag in context ctx := r.Context() ctx = context.WithValue(ctx, types.CtxIsMockedKey, true) + if mockSeed != 0 { + ctx = context.WithValue(ctx, types.CtxMockSeedKey, mockSeed) + } r = r.WithContext(ctx) next.ServeHTTP(w, r) }) diff --git a/backend/pkg/api/types/data_access.go b/backend/pkg/api/types/data_access.go index 00c909d12..8cf95256e 100644 --- a/backend/pkg/api/types/data_access.go +++ b/backend/pkg/api/types/data_access.go @@ -329,4 +329,5 @@ type CtxKey string const CtxUserIdKey CtxKey = "user_id" const CtxIsMockedKey CtxKey = "is_mocked" +const CtxMockSeedKey CtxKey = "mock_seed" const CtxDashboardIdKey CtxKey = "dashboard_id" From 351dc83e7d9bd096963025478944161c414f5254 Mon Sep 17 00:00:00 2001 From: Lucca Dukic <109136188+LuccaBitfly@users.noreply.github.com> Date: Thu, 24 Oct 2024 13:01:10 +0200 Subject: [PATCH 20/82] fix: cap do not disturb timestamp if greater than maxInt32 See: BEDS-856 --- backend/pkg/api/handlers/public.go | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/pkg/api/handlers/public.go b/backend/pkg/api/handlers/public.go index 5a6473f0e..974060972 100644 --- a/backend/pkg/api/handlers/public.go +++ b/backend/pkg/api/handlers/public.go @@ -2215,6 +2215,7 @@ func (h *HandlerService) PublicPutUserNotificationSettingsGeneral(w http.Respons handleErr(w, r, v) return } + req.DoNotDisturbTimestamp = min(req.DoNotDisturbTimestamp, math.MaxInt32) // check premium perks userInfo, err := h.getDataAccessor(r).GetUserInfo(r.Context(), userId) From 103de90d84a29236f2431db89a210181e30f2138 Mon Sep 17 00:00:00 2001 From: Patrick Date: Thu, 24 Oct 2024 13:12:45 +0200 Subject: [PATCH 21/82] fix(evm_indexer): fix parsing flags, add version-flag, serve metrics if set in config (#1033) --- backend/cmd/evm_node_indexer/main.go | 51 ++++++++++++++++++++-------- 1 file changed, 36 insertions(+), 15 deletions(-) diff --git a/backend/cmd/evm_node_indexer/main.go b/backend/cmd/evm_node_indexer/main.go index b431720bc..4975bb3d0 100644 --- a/backend/cmd/evm_node_indexer/main.go +++ b/backend/cmd/evm_node_indexer/main.go @@ -13,6 +13,7 @@ import ( "fmt" "io" "net/http" + "os" "regexp" "strings" "sync/atomic" @@ -24,6 +25,7 @@ import ( "github.com/gobitfly/beaconchain/pkg/commons/db" "github.com/gobitfly/beaconchain/pkg/commons/hexutil" "github.com/gobitfly/beaconchain/pkg/commons/log" + "github.com/gobitfly/beaconchain/pkg/commons/metrics" "github.com/gobitfly/beaconchain/pkg/commons/types" "github.com/gobitfly/beaconchain/pkg/commons/utils" "github.com/gobitfly/beaconchain/pkg/commons/version" @@ -103,22 +105,32 @@ func init() { // main func Run() { + fs := flag.NewFlagSet("fs", flag.ExitOnError) + // read / set parameter - configPath := flag.String("config", "config/default.config.yml", "Path to the config file") - startBlockNumber := flag.Int64("start-block-number", -1, "trigger a REEXPORT, only working in combination with end-block-number, defined block is included, will be the first action done and will quite afterwards, ignore every other action") - endBlockNumber := flag.Int64("end-block-number", -1, "trigger a REEXPORT, only working in combination with start-block-number, defined block is included, will be the first action done and will quite afterwards, ignore every other action") - reorgDepth = flag.Int64("reorg.depth", 32, fmt.Sprintf("lookback to check and handle chain reorgs (MAX %s), you should NEVER reduce this after the first start, otherwise there will be unchecked areas", _formatInt64(MAX_REORG_DEPTH))) - concurrency := flag.Int64("concurrency", 8, "maximum threads used (running on maximum whenever possible)") - nodeRequestsAtOnce := flag.Int64("node-requests-at-once", 16, fmt.Sprintf("bulk size per node = bt = db request (MAX %s)", _formatInt64(MAX_NODE_REQUESTS_AT_ONCE))) - skipHoleCheck := flag.Bool("skip-hole-check", false, "skips the initial check for holes, doesn't go very well with only-hole-check") - onlyHoleCheck := flag.Bool("only-hole-check", false, "just check for holes and quit, can be used for a reexport running simulation to a normal setup, just remove entries in postgres and start with this flag, doesn't go very well with skip-hole-check") - noNewBlocks := flag.Bool("ignore-new-blocks", false, "there are no new blocks, at all") - noNewBlocksThresholdSeconds := flag.Int("fatal-if-no-new-block-for-x-seconds", 600, "will fatal if there is no new block for x seconds (MIN 30), will start throwing errors at 2/3 of the time, will start throwing warnings at 1/3 of the time, doesn't go very well with ignore-new-blocks") - discordWebhookBlockThreshold := flag.Int64("discord-block-threshold", 100000, "every x blocks an update is send to Discord") - discordWebhookReportUrl := flag.String("discord-url", "", "report progress to discord url") - discordWebhookUser := flag.String("discord-user", "", "report progress to discord user") - discordWebhookAddTextFatal := flag.String("discord-fatal-text", "", "this text will be added to the discord message in the case of an fatal") - flag.Parse() + configPath := fs.String("config", "config/default.config.yml", "Path to the config file") + versionFlag := fs.Bool("version", false, "print version and exit") + startBlockNumber := fs.Int64("start-block-number", -1, "trigger a REEXPORT, only working in combination with end-block-number, defined block is included, will be the first action done and will quite afterwards, ignore every other action") + endBlockNumber := fs.Int64("end-block-number", -1, "trigger a REEXPORT, only working in combination with start-block-number, defined block is included, will be the first action done and will quite afterwards, ignore every other action") + reorgDepth = fs.Int64("reorg.depth", 32, fmt.Sprintf("lookback to check and handle chain reorgs (MAX %s), you should NEVER reduce this after the first start, otherwise there will be unchecked areas", _formatInt64(MAX_REORG_DEPTH))) + concurrency := fs.Int64("concurrency", 8, "maximum threads used (running on maximum whenever possible)") + nodeRequestsAtOnce := fs.Int64("node-requests-at-once", 16, fmt.Sprintf("bulk size per node = bt = db request (MAX %s)", _formatInt64(MAX_NODE_REQUESTS_AT_ONCE))) + skipHoleCheck := fs.Bool("skip-hole-check", false, "skips the initial check for holes, doesn't go very well with only-hole-check") + onlyHoleCheck := fs.Bool("only-hole-check", false, "just check for holes and quit, can be used for a reexport running simulation to a normal setup, just remove entries in postgres and start with this flag, doesn't go very well with skip-hole-check") + noNewBlocks := fs.Bool("ignore-new-blocks", false, "there are no new blocks, at all") + noNewBlocksThresholdSeconds := fs.Int("fatal-if-no-new-block-for-x-seconds", 600, "will fatal if there is no new block for x seconds (MIN 30), will start throwing errors at 2/3 of the time, will start throwing warnings at 1/3 of the time, doesn't go very well with ignore-new-blocks") + discordWebhookBlockThreshold := fs.Int64("discord-block-threshold", 100000, "every x blocks an update is send to Discord") + discordWebhookReportUrl := fs.String("discord-url", "", "report progress to discord url") + discordWebhookUser := fs.String("discord-user", "", "report progress to discord user") + discordWebhookAddTextFatal := fs.String("discord-fatal-text", "", "this text will be added to the discord message in the case of an fatal") + err := fs.Parse(os.Args[2:]) + if err != nil { + log.Fatal(err, "error parsing flags", 0) + } + if *versionFlag { + log.Info(version.Version) + return + } // tell the user about all parameter { @@ -161,6 +173,15 @@ func Run() { } else { eth1RpcEndpoint = utils.Config.Eth1GethEndpoint } + + if utils.Config.Metrics.Enabled { + go func() { + log.Infof("serving metrics on %v", utils.Config.Metrics.Address) + if err := metrics.Serve(utils.Config.Metrics.Address, utils.Config.Metrics.Pprof, utils.Config.Metrics.PprofExtra); err != nil { + log.Fatal(err, "error serving metrics", 0) + } + }() + } } // check parameters From 4340a77bf2645063e7ef22063ff3f4d21e284d74 Mon Sep 17 00:00:00 2001 From: remoterami <142154971+remoterami@users.noreply.github.com> Date: Thu, 24 Oct 2024 13:56:31 +0200 Subject: [PATCH 22/82] applied CR feedback --- backend/pkg/api/data_access/general.go | 12 ++-- backend/pkg/api/data_access/vdb_blocks.go | 80 ++++++++++++++--------- 2 files changed, 54 insertions(+), 38 deletions(-) diff --git a/backend/pkg/api/data_access/general.go b/backend/pkg/api/data_access/general.go index d629d52df..9d7ee0d97 100644 --- a/backend/pkg/api/data_access/general.go +++ b/backend/pkg/api/data_access/general.go @@ -74,9 +74,9 @@ func applySortAndPagination(defaultColumns []types.SortColumn, primary types.Sor if cursor.IsReverse() { column.Desc = !column.Desc } - colOrder := column.Column.Asc() + colOrder := column.Column.Asc().NullsFirst() if column.Desc { - colOrder = column.Column.Desc() + colOrder = column.Column.Desc().NullsLast() } queryOrder = append(queryOrder, colOrder) } @@ -89,10 +89,10 @@ func applySortAndPagination(defaultColumns []types.SortColumn, primary types.Sor column := queryOrderColumns[i] var colWhere exp.Expression - // current convention is the psql default (ASC: nulls last, DESC: nulls first) - colWhere = goqu.Or(column.Column.Gt(column.Offset), column.Column.IsNull()) - if column.Desc { - colWhere = column.Column.Lt(column.Offset) + // current convention is opposite of the psql default (ASC: nulls first, DESC: nulls last) + colWhere = goqu.Or(column.Column.Lt(column.Offset), column.Column.IsNull()) + if !column.Desc { + colWhere = column.Column.Gt(column.Offset) if column.Offset == nil { colWhere = goqu.Or(colWhere, column.Column.IsNull()) } diff --git a/backend/pkg/api/data_access/vdb_blocks.go b/backend/pkg/api/data_access/vdb_blocks.go index 00495e265..b8c1ba352 100644 --- a/backend/pkg/api/data_access/vdb_blocks.go +++ b/backend/pkg/api/data_access/vdb_blocks.go @@ -10,6 +10,7 @@ import ( "time" "github.com/doug-martin/goqu/v9" + "github.com/doug-martin/goqu/v9/exp" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/gobitfly/beaconchain/pkg/api/enums" t "github.com/gobitfly/beaconchain/pkg/api/types" @@ -45,7 +46,7 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das searchGroup := regexp.MustCompile(`^[a-zA-Z0-9_\-.\ ]+$`).MatchString(search) searchIndex := regexp.MustCompile(`^[0-9]+$`).MatchString(search) - validators := goqu.T("users_val_dashboards_validators").As("validators") + validators := goqu.T("validators") blocks := goqu.T("blocks") groups := goqu.T("groups") @@ -65,21 +66,22 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das ) if dashboardId.Validators == nil { filteredValidatorsDs = filteredValidatorsDs. - From(validators). + From(goqu.T("users_val_dashboards_validators").As(validators.GetTable())). Where(validators.Col("dashboard_id").Eq(dashboardId.Id)) // apply search filters + searches := []exp.Expression{} if searchIndex { - filteredValidatorsDs = filteredValidatorsDs.Where(validators.Col("validator_index").Eq(search)) + searches = append(searches, validators.Col("validator_index").Eq(search)) } if searchGroup { filteredValidatorsDs = filteredValidatorsDs. InnerJoin(goqu.T("users_val_dashboards_groups").As(groups), goqu.On( validators.Col("group_id").Eq(groups.Col("id")), validators.Col("dashboard_id").Eq(groups.Col("dashboard_id")), - )). - Where( - goqu.L("LOWER(?)", groups.Col("name")).Like(strings.Replace(search, "_", "\\_", -1) + "%"), - ) + )) + searches = append(searches, + goqu.L("LOWER(?)", groups.Col("name")).Like(strings.Replace(strings.ToLower(search), "_", "\\_", -1)+"%"), + ) } if searchPubkey { index, ok := validatorMapping.ValidatorIndices[search] @@ -87,11 +89,15 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das // searched pubkey doesn't exist, don't even need to query anything return make([]t.VDBBlocksTableRow, 0), &t.Paging{}, nil } - - filteredValidatorsDs = filteredValidatorsDs. - Where(validators.Col("validator_index").Eq(index)) + searches = append(searches, + validators.Col("validator_index").Eq(index), + ) + } + if len(searches) > 0 { + filteredValidatorsDs = filteredValidatorsDs.Where(goqu.Or(searches...)) } } else { + validatorList := make([]t.VDBValidator, 0, len(dashboardId.Validators)) for _, validator := range dashboardId.Validators { if searchIndex && fmt.Sprint(validator) != search || searchPubkey && validator != validatorMapping.ValidatorIndices[search] { @@ -101,14 +107,19 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das Validator: validator, Group: t.DefaultGroupId, }) + validatorList = append(validatorList, validator) if searchIndex || searchPubkey { break } } filteredValidatorsDs = filteredValidatorsDs. From( - goqu.L("unnest(?)", pq.Array(filteredValidators)).As("validator_index"), - ).As("validators") // TODO ? + goqu.Dialect("postgres"). + From( + goqu.L("unnest(?::int[])", pq.Array(validatorList)).As("validator_index"), + ). + As(validators.GetTable()), + ) } // ------------------------------------- @@ -144,10 +155,17 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das ) // 2. Selects + groupIdQ := goqu.C("group_id").(exp.Aliaseable) + if dashboardId.Validators != nil { + groupIdQ = exp.NewLiteralExpression("?::int", t.DefaultGroupId) + } + groupId := groupIdQ.As("group_id") + blocksDs = blocksDs. SelectAppend( blocks.Col("epoch"), blocks.Col("slot"), + groupId, blocks.Col("status"), blocks.Col("exec_block_number"), blocks.Col("graffiti_text"), @@ -156,16 +174,7 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das goqu.L("cp.cl_attestations_reward / 1e9 + cp.cl_sync_aggregate_reward / 1e9 + cp.cl_slashing_inclusion_reward / 1e9").As("cl_reward"), ) - groupId := validators.Col("group_id") - if dashboardId.Validators != nil { - groupId = goqu.V(t.DefaultGroupId).As("group_id").GetAs() - } - blocksDs = blocksDs.SelectAppend(groupId) - - // 3. Limit - blocksDs = blocksDs.Limit(uint(limit + 1)) - - // 4. Sorting and pagination + // 3. Sorting and pagination defaultColumns := []t.SortColumn{ {Column: enums.VDBBlocksColumns.Slot.ToExpr(), Desc: true, Offset: currentCursor.Slot}, } @@ -185,18 +194,23 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das } order, directions := applySortAndPagination(defaultColumns, t.SortColumn{Column: colSort.Column.ToExpr(), Desc: colSort.Desc, Offset: offset}, currentCursor.GenericCursor) - blocksDs = goqu.From(blocksDs). // encapsulate so we can use selected fields - Order(order...) + blocksDs = goqu.Dialect("postgres").From(goqu.T("past_blocks_cte")). + With("past_blocks_cte", blocksDs). // encapsulate so we can use selected fields + Order(order...) if directions != nil { blocksDs = blocksDs.Where(directions) } + // 4. Limit + blocksDs = blocksDs.Limit(uint(limit + 1)) + // 5. Gather and supply scheduled blocks to let db do the sorting etc latestSlot := cache.LatestSlot.Get() onlyPrimarySort := colSort.Column == enums.VDBBlockSlot - if !(onlyPrimarySort || colSort.Column == enums.VDBBlockBlock) || !currentCursor.IsValid() || - currentCursor.Slot > latestSlot+1 && currentCursor.Reverse != colSort.Desc || - currentCursor.Slot < latestSlot+1 && currentCursor.Reverse == colSort.Desc { + if !(onlyPrimarySort || colSort.Column == enums.VDBBlockBlock) || + !currentCursor.IsValid() || + currentCursor.Slot > latestSlot+1 || + colSort.Desc == currentCursor.Reverse { dutiesInfo, err := d.services.GetCurrentDutiesInfo() if err == nil { if dashboardId.Validators == nil { @@ -213,11 +227,12 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das return make([]t.VDBBlocksTableRow, 0), &t.Paging{}, nil } - validatorSet := make(map[t.VDBValidator]bool) + validatorSet := make(map[t.VDBValidator]uint64) for _, v := range filteredValidators { - validatorSet[v.Validator] = true + validatorSet[v.Validator] = v.Group } var scheduledProposers []t.VDBValidator + var scheduledGroups []uint64 var scheduledEpochs []uint64 var scheduledSlots []uint64 // don't need if requested slots are in the past @@ -231,25 +246,26 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das continue } scheduledProposers = append(scheduledProposers, dutiesInfo.PropAssignmentsForSlot[slot]) + scheduledGroups = append(scheduledGroups, validatorSet[vali]) scheduledEpochs = append(scheduledEpochs, slot/utils.Config.Chain.ClConfig.SlotsPerEpoch) scheduledSlots = append(scheduledSlots, slot) } scheduledDs := goqu.Dialect("postgres"). From( - goqu.L("unnest(?::int[], ?::int[], ?::int[]) AS prov(validator_index, epoch, slot)", pq.Array(scheduledProposers), pq.Array(scheduledEpochs), pq.Array(scheduledSlots)), + goqu.L("unnest(?::int[], ?::int[], ?::int[], ?::int[]) AS prov(validator_index, group_id, epoch, slot)", pq.Array(scheduledProposers), pq.Array(scheduledGroups), pq.Array(scheduledEpochs), pq.Array(scheduledSlots)), ). Select( goqu.C("validator_index"), goqu.C("epoch"), goqu.C("slot"), + groupId, goqu.V("0").As("status"), goqu.V(nil).As("exec_block_number"), + goqu.V(nil).As("graffiti_text"), goqu.V(nil).As("fee_recipient"), goqu.V(nil).As("el_reward"), goqu.V(nil).As("cl_reward"), - goqu.V(nil).As("graffiti_text"), - goqu.V(t.DefaultGroupId).As("group_id"), ). As("scheduled_blocks") From a5bc4281350a1e46372e76cd972cc57bef4ebc1c Mon Sep 17 00:00:00 2001 From: Lucca Dukic <109136188+LuccaBitfly@users.noreply.github.com> Date: Wed, 23 Oct 2024 13:18:21 +0200 Subject: [PATCH 23/82] fix: implement proper auth check for editing paired devices See: BEDS-863 --- backend/pkg/api/data_access/dummy.go | 8 +++- backend/pkg/api/data_access/notifications.go | 45 +++++++++++++------- backend/pkg/api/handlers/public.go | 23 ++++++++-- 3 files changed, 56 insertions(+), 20 deletions(-) diff --git a/backend/pkg/api/data_access/dummy.go b/backend/pkg/api/data_access/dummy.go index 89f915e46..f1063084a 100644 --- a/backend/pkg/api/data_access/dummy.go +++ b/backend/pkg/api/data_access/dummy.go @@ -582,10 +582,10 @@ func (d *DummyService) UpdateNotificationSettingsGeneral(ctx context.Context, us func (d *DummyService) UpdateNotificationSettingsNetworks(ctx context.Context, userId uint64, chainId uint64, settings t.NotificationSettingsNetwork) error { return nil } -func (d *DummyService) UpdateNotificationSettingsPairedDevice(ctx context.Context, userId uint64, pairedDeviceId uint64, name string, IsNotificationsEnabled bool) error { +func (d *DummyService) UpdateNotificationSettingsPairedDevice(ctx context.Context, pairedDeviceId uint64, name string, IsNotificationsEnabled bool) error { return nil } -func (d *DummyService) DeleteNotificationSettingsPairedDevice(ctx context.Context, userId uint64, pairedDeviceId uint64) error { +func (d *DummyService) DeleteNotificationSettingsPairedDevice(ctx context.Context, pairedDeviceId uint64) error { return nil } @@ -796,3 +796,7 @@ func (d *DummyService) QueueTestPushNotification(ctx context.Context, userId uin func (d *DummyService) QueueTestWebhookNotification(ctx context.Context, userId uint64, webhookUrl string, isDiscordWebhook bool) error { return nil } + +func (d *DummyService) GetPairedDeviceUserId(ctx context.Context, pairedDeviceId uint64) (uint64, error) { + return getDummyData[uint64](ctx) +} diff --git a/backend/pkg/api/data_access/notifications.go b/backend/pkg/api/data_access/notifications.go index 5c9adc406..39b10eec9 100644 --- a/backend/pkg/api/data_access/notifications.go +++ b/backend/pkg/api/data_access/notifications.go @@ -51,8 +51,9 @@ type NotificationsRepository interface { GetNotificationSettingsDefaultValues(ctx context.Context) (*t.NotificationSettingsDefaultValues, error) UpdateNotificationSettingsGeneral(ctx context.Context, userId uint64, settings t.NotificationSettingsGeneral) error UpdateNotificationSettingsNetworks(ctx context.Context, userId uint64, chainId uint64, settings t.NotificationSettingsNetwork) error - UpdateNotificationSettingsPairedDevice(ctx context.Context, userId uint64, pairedDeviceId uint64, name string, IsNotificationsEnabled bool) error - DeleteNotificationSettingsPairedDevice(ctx context.Context, userId uint64, pairedDeviceId uint64) error + GetPairedDeviceUserId(ctx context.Context, pairedDeviceId uint64) (uint64, error) + UpdateNotificationSettingsPairedDevice(ctx context.Context, pairedDeviceId uint64, name string, IsNotificationsEnabled bool) error + DeleteNotificationSettingsPairedDevice(ctx context.Context, pairedDeviceId uint64) error UpdateNotificationSettingsClients(ctx context.Context, userId uint64, clientId uint64, IsSubscribed bool) (*t.NotificationSettingsClient, error) GetNotificationSettingsDashboards(ctx context.Context, userId uint64, cursor string, colSort t.Sort[enums.NotificationSettingsDashboardColumn], search string, limit uint64) ([]t.NotificationSettingsDashboardsTableRow, *t.Paging, error) UpdateNotificationSettingsValidatorDashboard(ctx context.Context, userId uint64, dashboardId t.VDBIdPrimary, groupId uint64, settings t.NotificationSettingsValidatorDashboard) error @@ -1637,47 +1638,61 @@ func (d *DataAccessService) UpdateNotificationSettingsNetworks(ctx context.Conte } return nil } -func (d *DataAccessService) UpdateNotificationSettingsPairedDevice(ctx context.Context, userId uint64, pairedDeviceId uint64, name string, IsNotificationsEnabled bool) error { + +func (d *DataAccessService) GetPairedDeviceUserId(ctx context.Context, pairedDeviceId uint64) (uint64, error) { + var userId uint64 + err := d.userReader.GetContext(context.Background(), &userId, ` + SELECT user_id + FROM users_devices + WHERE id = $1`, pairedDeviceId) + if err != nil { + if err == sql.ErrNoRows { + return 0, fmt.Errorf("%w, paired device with id %v not found", ErrNotFound, pairedDeviceId) + } + return 0, err + } + return userId, nil +} + +func (d *DataAccessService) UpdateNotificationSettingsPairedDevice(ctx context.Context, pairedDeviceId uint64, name string, IsNotificationsEnabled bool) error { result, err := d.userWriter.ExecContext(ctx, ` UPDATE users_devices SET device_name = $1, notify_enabled = $2 - WHERE user_id = $3 AND id = $4`, - name, IsNotificationsEnabled, userId, pairedDeviceId) + WHERE id = $3`, + name, IsNotificationsEnabled, pairedDeviceId) if err != nil { return err } - - // TODO: This can be deleted when the API layer has an improved check for the device id rowsAffected, err := result.RowsAffected() if err != nil { return err } if rowsAffected == 0 { - return fmt.Errorf("device with id %v to update notification settings not found", pairedDeviceId) + return fmt.Errorf("%w, paired device with id %v not found", ErrNotFound, pairedDeviceId) } return nil } -func (d *DataAccessService) DeleteNotificationSettingsPairedDevice(ctx context.Context, userId uint64, pairedDeviceId uint64) error { + +func (d *DataAccessService) DeleteNotificationSettingsPairedDevice(ctx context.Context, pairedDeviceId uint64) error { result, err := d.userWriter.ExecContext(ctx, ` - DELETE FROM users_devices - WHERE user_id = $1 AND id = $2`, - userId, pairedDeviceId) + DELETE FROM users_devices + WHERE id = $1`, + pairedDeviceId) if err != nil { return err } - - // TODO: This can be deleted when the API layer has an improved check for the device id rowsAffected, err := result.RowsAffected() if err != nil { return err } if rowsAffected == 0 { - return fmt.Errorf("device with id %v to delete not found", pairedDeviceId) + return fmt.Errorf("%w, paired device with id %v not found", ErrNotFound, pairedDeviceId) } return nil } + func (d *DataAccessService) UpdateNotificationSettingsClients(ctx context.Context, userId uint64, clientId uint64, IsSubscribed bool) (*t.NotificationSettingsClient, error) { result := &t.NotificationSettingsClient{Id: clientId, IsSubscribed: IsSubscribed} diff --git a/backend/pkg/api/handlers/public.go b/backend/pkg/api/handlers/public.go index 974060972..72813b142 100644 --- a/backend/pkg/api/handlers/public.go +++ b/backend/pkg/api/handlers/public.go @@ -2353,7 +2353,16 @@ func (h *HandlerService) PublicPutUserNotificationSettingsPairedDevices(w http.R handleErr(w, r, v) return } - err = h.getDataAccessor(r).UpdateNotificationSettingsPairedDevice(r.Context(), userId, pairedDeviceId, name, req.IsNotificationsEnabled) + pairedDeviceUserId, err := h.getDataAccessor(r).GetPairedDeviceUserId(r.Context(), pairedDeviceId) + if err != nil { + handleErr(w, r, err) + return + } + if userId != pairedDeviceUserId { + returnNotFound(w, r, fmt.Errorf("not found: paired device with id %d not found", pairedDeviceId)) // return 404 to not leak information + return + } + err = h.getDataAccessor(r).UpdateNotificationSettingsPairedDevice(r.Context(), pairedDeviceId, name, req.IsNotificationsEnabled) if err != nil { handleErr(w, r, err) return @@ -2387,13 +2396,21 @@ func (h *HandlerService) PublicDeleteUserNotificationSettingsPairedDevices(w htt handleErr(w, r, err) return } - // TODO use a better way to validate the paired device id pairedDeviceId := v.checkUint(mux.Vars(r)["paired_device_id"], "paired_device_id") if v.hasErrors() { handleErr(w, r, v) return } - err = h.getDataAccessor(r).DeleteNotificationSettingsPairedDevice(r.Context(), userId, pairedDeviceId) + pairedDeviceUserId, err := h.getDataAccessor(r).GetPairedDeviceUserId(r.Context(), pairedDeviceId) + if err != nil { + handleErr(w, r, err) + return + } + if userId != pairedDeviceUserId { + returnNotFound(w, r, fmt.Errorf("not found: paired device with id %d not found", pairedDeviceId)) // return 404 to not leak information + return + } + err = h.getDataAccessor(r).DeleteNotificationSettingsPairedDevice(r.Context(), pairedDeviceId) if err != nil { handleErr(w, r, err) return From 21d9cde3bf64f0ba5dd426a3cffb3aba5e321e96 Mon Sep 17 00:00:00 2001 From: benji-bitfly Date: Thu, 24 Oct 2024 14:49:20 +0200 Subject: [PATCH 24/82] feat(NotificationsOverview): add `empty fields` state See: BEDS-860 --- .../notifications/NotificationsOverview.vue | 54 ++++++++++++------- 1 file changed, 34 insertions(+), 20 deletions(-) diff --git a/frontend/components/notifications/NotificationsOverview.vue b/frontend/components/notifications/NotificationsOverview.vue index 1db754293..f6a261cea 100644 --- a/frontend/components/notifications/NotificationsOverview.vue +++ b/frontend/components/notifications/NotificationsOverview.vue @@ -100,31 +100,41 @@ const emit = defineEmits<{ {{ $t('notifications.overview.headers.most_notifications_30d') }}
- - {{ $t('notifications.overview.headers.validator_groups') }} - -
    -
  1. - - - {{ group }} - -
    2. -
- +
- {{ $t('notifications.overview.headers.account_groups') }} + {{ $t('notifications.overview.headers.validator_groups') }} -
    -
  1. - +
      +
    1. + - {{ group }} + {{ index + 1 }}. {{ group || '-' }}
    +
+ +
+ + {{ $t('notifications.overview.headers.account_groups') }} + +
    +
  1. + + + {{ index + 1 }}. {{ group || '-' }} + +
    2. +
+
@@ -203,7 +213,11 @@ a:hover { } .lists-container { display: flex; - gap: 20px; + gap: 1.25rem; +} +.lists-container-column { + flex: 1; + min-width: 0; } .icon-list { min-width: 0; From 5af248b504536e123ed38c864ece62260136ec2c Mon Sep 17 00:00:00 2001 From: benji-bitfly Date: Thu, 24 Oct 2024 14:57:30 +0200 Subject: [PATCH 25/82] refactor(NotificationsOverview): convert to `rem` for `a11y` --- .../notifications/NotificationsOverview.vue | 27 +++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/frontend/components/notifications/NotificationsOverview.vue b/frontend/components/notifications/NotificationsOverview.vue index f6a261cea..6de7c17b7 100644 --- a/frontend/components/notifications/NotificationsOverview.vue +++ b/frontend/components/notifications/NotificationsOverview.vue @@ -159,25 +159,25 @@ const emit = defineEmits<{ .container { @include main.container; - padding: 17px 20px; + padding: 1.0625rem 1.25rem; position: relative; } .info-section, .action-section { display: flex; align-items: center; justify-content: center; - gap: 10px; + gap: .625rem; } .icon { - font-size: 24px; + font-size: 1.5rem; } .text { - font-size: 18px; + font-size: 1.125rem; font-weight: 500; } .list-item { display: flex; - gap: 10px; + gap: .625rem; .list-text { @include utils.truncate-text; } @@ -201,12 +201,12 @@ const emit = defineEmits<{ .box-item { display: flex; flex-direction: column; - gap: 10px; + gap: .625rem; } .inline-items { display: flex; align-items: center; - gap: 10px; + gap: .625rem; } a:hover { color: var(--light-blue); @@ -221,15 +221,14 @@ a:hover { } .icon-list { min-width: 0; - list-style-type: none; padding: 0; margin: 0; display: flex; flex-direction: column; - gap: 10px; + gap: .625rem; } .icon { - font-size: 16px; + font-size: 1rem; } .inline-link, .gem { @@ -238,21 +237,21 @@ a:hover { .premium-invitation { display: flex; align-items: center; - gap: 5px; /* Adjust the gap as needed */ + gap: .3125rem; } .push-invitation { display: flex; align-items: center; - gap: 5px; /* Adjust the gap as needed */ + gap: .3125rem; flex-wrap: wrap; } @media (max-width: 600px) { .box { flex-direction: row; - gap: 20px; + gap: 1.25rem; } .box-item { - min-width: 250px; /* Adjust based on content width */ + min-width: 15.625rem; } } From 9e87e31ac5417a3772b975a2ec1074307b2edb63 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Fri, 25 Oct 2024 06:36:48 +0000 Subject: [PATCH 26/82] feat(notifications): working discord notifications --- backend/cmd/misc/main.go | 33 ++++++++++++++++------- backend/pkg/notification/collection.go | 4 +-- backend/pkg/notification/notifications.go | 15 ++++++++++- backend/pkg/notification/queuing.go | 33 ++++++++++++++++++++--- 4 files changed, 69 insertions(+), 16 deletions(-) diff --git a/backend/cmd/misc/main.go b/backend/cmd/misc/main.go index c567a0371..5134cc5e9 100644 --- a/backend/cmd/misc/main.go +++ b/backend/cmd/misc/main.go @@ -555,7 +555,7 @@ func Run() { func collectNotifications(startEpoch uint64) error { epoch := startEpoch - notifications, err := notification.GetNotificationsForEpoch(utils.Config.Notifications.PubkeyCachePath, epoch) + notifications, err := notification.GetHeadNotificationsForEpoch(utils.Config.Notifications.PubkeyCachePath, epoch) if err != nil { return err } @@ -565,19 +565,34 @@ func collectNotifications(startEpoch uint64) error { spew.Dump(notifications[0]) } - emails, err := notification.RenderEmailsForUserEvents(0, notifications) + tx, err := db.WriterDb.Beginx() if err != nil { return err } + defer tx.Rollback() - for _, email := range emails { - // if email.Address == "" { - log.Infof("to: %v", email.Address) - log.Infof("subject: %v", email.Subject) - log.Infof("body: %v", email.Email.Body) - log.Info("-----") - // } + err = notification.QueueWebhookNotifications(notifications, tx) + if err != nil { + return err } + err = tx.Commit() + if err != nil { + return err + } + + // emails, err := notification.RenderEmailsForUserEvents(0, notifications) + // if err != nil { + // return err + // } + + // for _, email := range emails { + // // if email.Address == "" { + // log.Infof("to: %v", email.Address) + // log.Infof("subject: %v", email.Subject) + // log.Infof("body: %v", email.Email.Body) + // log.Info("-----") + // // } + // } // pushMessages, err := notification.RenderPushMessagesForUserEvents(0, notifications) // if err != nil { diff --git a/backend/pkg/notification/collection.go b/backend/pkg/notification/collection.go index ba6b0568f..2c703e583 100644 --- a/backend/pkg/notification/collection.go +++ b/backend/pkg/notification/collection.go @@ -161,7 +161,7 @@ func notificationCollector() { log.Infof("collecting notifications for epoch %v", epoch) // Network DB Notifications (network related) - notifications, err := collectNotifications(epoch, mc) + notifications, err := collectNotifications(mc, epoch) if err != nil { log.Error(err, "error collection notifications", 0) @@ -280,7 +280,7 @@ func collectUpcomingBlockProposalNotifications(notificationsByUserID types.Notif return nil } -func collectNotifications(epoch uint64, mc modules.ModuleContext) (types.NotificationsPerUserId, error) { +func collectNotifications(mc modules.ModuleContext, epoch uint64) (types.NotificationsPerUserId, error) { notificationsByUserID := types.NotificationsPerUserId{} start := time.Now() var err error diff --git a/backend/pkg/notification/notifications.go b/backend/pkg/notification/notifications.go index 7bb71162a..ebddb2aad 100644 --- a/backend/pkg/notification/notifications.go +++ b/backend/pkg/notification/notifications.go @@ -17,7 +17,20 @@ func GetNotificationsForEpoch(pubkeyCachePath string, epoch uint64) (types.Notif if err != nil { log.Fatal(err, "error initializing pubkey cache path for notifications", 0) } - return collectNotifications(epoch, mc) + return collectNotifications(mc, epoch) +} + +func GetHeadNotificationsForEpoch(pubkeyCachePath string, epoch uint64) (types.NotificationsPerUserId, error) { + mc, err := modules.GetModuleContext() + if err != nil { + log.Fatal(err, "error getting module context", 0) + } + + err = initPubkeyCache(pubkeyCachePath) + if err != nil { + log.Fatal(err, "error initializing pubkey cache path for notifications", 0) + } + return collectHeadNotifications(mc, epoch) } // Used for isolated testing diff --git a/backend/pkg/notification/queuing.go b/backend/pkg/notification/queuing.go index fb03b41b3..0822a53f4 100644 --- a/backend/pkg/notification/queuing.go +++ b/backend/pkg/notification/queuing.go @@ -661,6 +661,7 @@ func QueueWebhookNotifications(notificationsByUserID types.NotificationsPerUserI // now fetch the webhooks for each dashboard config err = db.ReaderDb.Select(&webhooks, ` SELECT + users_val_dashboards.user_id AS user_id, users_val_dashboards_groups.id AS dashboard_group_id, dashboard_id AS dashboard_id, webhook_target AS url, @@ -840,18 +841,29 @@ func QueueWebhookNotifications(notificationsByUserID types.NotificationsPerUserI } totalBlockReward := float64(0) + epoch := uint64(0) details := "" - if event == types.ValidatorExecutedProposalEventName { - for _, n := range notifications { + i := 0 + for _, n := range notifications { + if event == types.ValidatorExecutedProposalEventName { proposalNotification, ok := n.(*ValidatorProposalNotification) if !ok { log.Error(fmt.Errorf("error casting proposal notification"), "", 0) continue } totalBlockReward += proposalNotification.Reward - + } + if i <= 10 { details += fmt.Sprintf("%s\n", n.GetInfo(types.NotifciationFormatMarkdown)) } + i++ + if i == 11 { + details += fmt.Sprintf("... and %d more notifications\n", len(notifications)-i) + continue + } + if epoch == 0 { + epoch = n.GetEpoch() + } } count := len(notifications) @@ -876,7 +888,20 @@ func QueueWebhookNotifications(notificationsByUserID types.NotificationsPerUserI default: summary += fmt.Sprintf("%s: %d validator%s", types.EventLabel[event], count, plural) } - content.DiscordRequest.Content = summary + "\n" + details + content.DiscordRequest.Embeds = append(content.DiscordRequest.Embeds, types.DiscordEmbed{ + Type: "rich", + Color: "16745472", + Description: details, + Title: summary, + Fields: []types.DiscordEmbedField{ + { + Name: "Epoch", + Value: fmt.Sprintf("[%[1]v](https://%[2]s/epoch/%[1]v)", epoch, utils.Config.Frontend.SiteDomain), + Inline: false, + }, + }, + }) + if _, exists := discordNotifMap[w.ID]; !exists { discordNotifMap[w.ID] = make([]types.TransitDiscordContent, 0) } From ff9221f99b464cc92212a03b0b75cc37cc606630 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Fri, 25 Oct 2024 07:11:03 +0000 Subject: [PATCH 27/82] feat(notifications): implement normal webhook handling --- backend/pkg/commons/types/frontend.go | 5 ++-- backend/pkg/notification/collection.go | 8 +++---- backend/pkg/notification/queuing.go | 32 ++++++++++++++++++++++---- 3 files changed, 35 insertions(+), 10 deletions(-) diff --git a/backend/pkg/commons/types/frontend.go b/backend/pkg/commons/types/frontend.go index 0089ed96b..b66a02f1a 100644 --- a/backend/pkg/commons/types/frontend.go +++ b/backend/pkg/commons/types/frontend.go @@ -549,8 +549,9 @@ type TransitWebhook struct { type TransitWebhookContent struct { Webhook UserWebhook - Event WebhookEvent `json:"event"` - UserId UserId `json:"userId"` + Event *WebhookEvent `json:"event,omitempty"` + Events []*WebhookEvent `json:"events,omitempty"` + UserId UserId `json:"userId"` } type WebhookEvent struct { diff --git a/backend/pkg/notification/collection.go b/backend/pkg/notification/collection.go index 2c703e583..56929acf7 100644 --- a/backend/pkg/notification/collection.go +++ b/backend/pkg/notification/collection.go @@ -230,10 +230,10 @@ func collectUpcomingBlockProposalNotifications(notificationsByUserID types.Notif nextEpoch := headEpoch + 1 log.Infof("collecting upcoming block proposal notifications for epoch %v (head epoch is %d)", nextEpoch, headEpoch) - if utils.EpochToTime(nextEpoch).Before(time.Now()) { - log.Error(fmt.Errorf("error upcoming block proposal notifications for epoch %v are already in the past", nextEpoch), "", 0) - return nil - } + // if utils.EpochToTime(nextEpoch).Before(time.Now()) { + // log.Error(fmt.Errorf("error upcoming block proposal notifications for epoch %v are already in the past", nextEpoch), "", 0) + // return nil + // } assignments, err := mc.CL.GetPropoalAssignments(nextEpoch) if err != nil { diff --git a/backend/pkg/notification/queuing.go b/backend/pkg/notification/queuing.go index 0822a53f4..7ca956298 100644 --- a/backend/pkg/notification/queuing.go +++ b/backend/pkg/notification/queuing.go @@ -775,7 +775,7 @@ func QueueWebhookNotifications(notificationsByUserID types.NotificationsPerUserI Channel: w.Destination.String, Content: types.TransitWebhookContent{ Webhook: w, - Event: types.WebhookEvent{ + Event: &types.WebhookEvent{ Network: utils.GetNetwork(), Name: string(n.GetEventName()), Title: n.GetLegacyTitle(), @@ -908,8 +908,26 @@ func QueueWebhookNotifications(notificationsByUserID types.NotificationsPerUserI log.Infof("adding discord notification for user %d, dashboard %d, group %d and type %s", userID, dashboardId, dashboardGroupId, event) discordNotifMap[w.ID] = append(discordNotifMap[w.ID], content) - } else { - // TODO: implement + } else if w.Destination.Valid && w.Destination.String == "webhook" { + events := []*types.WebhookEvent{} + for _, n := range notifications { + events = append(events, &types.WebhookEvent{ + Network: utils.GetNetwork(), + Name: string(n.GetEventName()), + Title: n.GetTitle(), + Description: n.GetInfo(types.NotifciationFormatText), + Epoch: n.GetEpoch(), + Target: n.GetEventFilter(), + }) + } + notifs = append(notifs, types.TransitWebhook{ + Channel: w.Destination.String, + Content: types.TransitWebhookContent{ + Webhook: w, + Events: events, + UserId: userID, + }, + }) } } } @@ -923,7 +941,13 @@ func QueueWebhookNotifications(notificationsByUserID types.NotificationsPerUserI if err != nil { log.Error(err, "error inserting into webhooks_queue", 0) } else { - metrics.NotificationsQueued.WithLabelValues(n.Channel, n.Content.Event.Name).Inc() + if n.Content.Event != nil { + metrics.NotificationsQueued.WithLabelValues(n.Channel, n.Content.Event.Name).Inc() + } else { + for _, e := range n.Content.Events { + metrics.NotificationsQueued.WithLabelValues(n.Channel, e.Name).Inc() + } + } } } } From 6832664d57ac0dda5aec000bb6c2f42cdd551a82 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Fri, 25 Oct 2024 07:28:33 +0000 Subject: [PATCH 28/82] feat(notifications): improve webhook data processing --- backend/pkg/notification/collection.go | 8 +-- backend/pkg/notification/queuing.go | 77 ++++++++++++++++---------- 2 files changed, 51 insertions(+), 34 deletions(-) diff --git a/backend/pkg/notification/collection.go b/backend/pkg/notification/collection.go index 56929acf7..2c703e583 100644 --- a/backend/pkg/notification/collection.go +++ b/backend/pkg/notification/collection.go @@ -230,10 +230,10 @@ func collectUpcomingBlockProposalNotifications(notificationsByUserID types.Notif nextEpoch := headEpoch + 1 log.Infof("collecting upcoming block proposal notifications for epoch %v (head epoch is %d)", nextEpoch, headEpoch) - // if utils.EpochToTime(nextEpoch).Before(time.Now()) { - // log.Error(fmt.Errorf("error upcoming block proposal notifications for epoch %v are already in the past", nextEpoch), "", 0) - // return nil - // } + if utils.EpochToTime(nextEpoch).Before(time.Now()) { + log.Error(fmt.Errorf("error upcoming block proposal notifications for epoch %v are already in the past", nextEpoch), "", 0) + return nil + } assignments, err := mc.CL.GetPropoalAssignments(nextEpoch) if err != nil { diff --git a/backend/pkg/notification/queuing.go b/backend/pkg/notification/queuing.go index 7ca956298..6787a03b3 100644 --- a/backend/pkg/notification/queuing.go +++ b/backend/pkg/notification/queuing.go @@ -693,9 +693,10 @@ func QueueWebhookNotifications(notificationsByUserID types.NotificationsPerUserI dashboardWebhookMap[types.UserId(w.UserID)][types.DashboardId(w.DashboardId)][types.DashboardGroupId(w.DashboardGroupId)] = w } + discordNotifMap := make(map[uint64][]types.TransitDiscordContent) + notifs := make([]types.TransitWebhook, 0) + for userID, userNotifications := range notificationsByUserID { - discordNotifMap := make(map[uint64][]types.TransitDiscordContent) - notifs := make([]types.TransitWebhook, 0) webhooks, exists := webhooksMap[uint64(userID)] if exists { // webhook => [] notifications @@ -932,41 +933,57 @@ func QueueWebhookNotifications(notificationsByUserID types.NotificationsPerUserI } } } + } - // process notifs - if len(notifs) > 0 { - log.Infof("queueing %v webhooks notifications", len(notifs)) - for _, n := range notifs { - _, err = tx.Exec(`INSERT INTO notification_queue (created, channel, content) VALUES (now(), $1, $2);`, n.Channel, n.Content) - if err != nil { - log.Error(err, "error inserting into webhooks_queue", 0) - } else { - if n.Content.Event != nil { - metrics.NotificationsQueued.WithLabelValues(n.Channel, n.Content.Event.Name).Inc() - } else { - for _, e := range n.Content.Events { - metrics.NotificationsQueued.WithLabelValues(n.Channel, e.Name).Inc() - } - } + // process notifs + log.Infof("queueing %v webhooks notifications", len(notifs)) + if len(notifs) > 0 { + type insertData struct { + Content types.TransitWebhookContent `db:"content"` + } + insertRows := make([]insertData, 0, len(notifs)) + for _, n := range notifs { + if n.Content.Event != nil { + metrics.NotificationsQueued.WithLabelValues(n.Channel, n.Content.Event.Name).Inc() + } else { + for _, e := range n.Content.Events { + metrics.NotificationsQueued.WithLabelValues(n.Channel, e.Name).Inc() } } + + insertRows = append(insertRows, insertData{ + Content: n.Content, + }) } - // process discord notifs - if len(discordNotifMap) > 0 { - log.Infof("queueing %v discord notifications", len(discordNotifMap)) - for _, dNotifs := range discordNotifMap { - for _, n := range dNotifs { - _, err = tx.Exec(`INSERT INTO notification_queue (created, channel, content) VALUES (now(), 'webhook_discord', $1);`, n) - if err != nil { - log.Error(err, "error inserting into webhooks_queue (discord)", 0) - continue - } else { - metrics.NotificationsQueued.WithLabelValues("webhook_discord", "multi").Inc() - } - } + _, err = tx.NamedExec(`INSERT INTO notification_queue (created, channel, content) VALUES (NOW(), 'webhook', :content)`, insertRows) + if err != nil { + return fmt.Errorf("error writing transit push to db: %w", err) + } + } + + // process discord notifs + log.Infof("queueing %v discord notifications", len(discordNotifMap)) + if len(discordNotifMap) > 0 { + type insertData struct { + Content types.TransitDiscordContent `db:"content"` + } + insertRows := make([]insertData, 0, len(discordNotifMap)) + + for _, dNotifs := range discordNotifMap { + for _, n := range dNotifs { + insertRows = append(insertRows, insertData{ + Content: n, + }) + metrics.NotificationsQueued.WithLabelValues("webhook_discord", "multi").Inc() } } + + _, err = tx.NamedExec(`INSERT INTO notification_queue (created, channel, content) VALUES (NOW(), 'webhook_discord', :content)`, insertRows) + if err != nil { + return fmt.Errorf("error writing transit push to db: %w", err) + } } + return nil } From 1553236aa20c8543a66a5a90c3848a92f676ff15 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Fri, 25 Oct 2024 08:08:56 +0000 Subject: [PATCH 29/82] chore(notifications): remove debug logging --- backend/pkg/notification/sending.go | 42 +++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/backend/pkg/notification/sending.go b/backend/pkg/notification/sending.go index 440333333..95c7101fb 100644 --- a/backend/pkg/notification/sending.go +++ b/backend/pkg/notification/sending.go @@ -20,6 +20,7 @@ import ( "github.com/gobitfly/beaconchain/pkg/commons/utils" "github.com/jmoiron/sqlx" "github.com/lib/pq" + "golang.org/x/sync/errgroup" ) func InitNotificationSender() { @@ -233,11 +234,17 @@ func sendWebhookNotifications() error { if err != nil { return fmt.Errorf("error querying notification queue, err: %w", err) } - client := &http.Client{Timeout: time.Second * 30} + + // webhooks have 5 seconds to respond + client := &http.Client{Timeout: time.Second * 5} log.Infof("processing %v webhook notifications", len(notificationQueueItem)) + // use an error group to throttle webhook requests + g := &errgroup.Group{} + g.SetLimit(50) // issue at most 50 requests at a time for _, n := range notificationQueueItem { + n := n _, err := db.CountSentMessage("n_webhooks", n.Content.UserId) if err != nil { log.Error(err, "error counting sent webhook", 0) @@ -268,7 +275,7 @@ func sendWebhookNotifications() error { continue } - go func(n types.TransitWebhook) { + g.Go(func() error { if n.Content.Webhook.Retries > 0 { time.Sleep(time.Duration(n.Content.Webhook.Retries) * time.Second) } @@ -276,7 +283,7 @@ func sendWebhookNotifications() error { if err != nil { log.Warnf("error sending webhook request: %v", err) metrics.NotificationsSent.WithLabelValues("webhook", "error").Inc() - return + return nil } else { metrics.NotificationsSent.WithLabelValues("webhook", resp.Status).Inc() } @@ -285,14 +292,18 @@ func sendWebhookNotifications() error { _, err = db.WriterDb.Exec(`UPDATE notification_queue SET sent = now() WHERE id = $1`, n.Id) if err != nil { log.Error(err, "error updating notification_queue table", 0) - return + return nil } if resp != nil && resp.StatusCode < 400 { - _, err = db.FrontendWriterDB.Exec(`UPDATE users_webhooks SET retries = 0, last_sent = now() WHERE id = $1;`, n.Content.Webhook.ID) + // update retries counters in db based on end result + if n.Content.Webhook.DashboardId == 0 && n.Content.Webhook.DashboardGroupId == 0 { + _, err = db.FrontendWriterDB.Exec(`UPDATE users_webhooks SET retries = $1, last_sent = now() WHERE id = $2;`, n.Content.Webhook.Retries, n.Content.Webhook.ID) + } else { + _, err = db.WriterDb.Exec(`UPDATE users_val_dashboards_groups SET webhook_retries = $1, webhook_last_sent = now() WHERE id = $2 AND dashboard_id = $3;`, n.Content.Webhook.Retries, n.Content.Webhook.DashboardGroupId, n.Content.Webhook.DashboardId) + } if err != nil { - log.Error(err, "error updating users_webhooks table", 0) - return + log.Warnf("failed to update retries counter to %v for webhook %v: %v", n.Content.Webhook.Retries, n.Content.Webhook.ID, err) } } else { var errResp types.ErrorResponse @@ -307,13 +318,23 @@ func sendWebhookNotifications() error { errResp.Body = string(b) } - _, err = db.FrontendWriterDB.Exec(`UPDATE users_webhooks SET retries = retries + 1, last_sent = now(), request = $2, response = $3 WHERE id = $1;`, n.Content.Webhook.ID, n.Content, errResp) + if n.Content.Webhook.DashboardId == 0 && n.Content.Webhook.DashboardGroupId == 0 { + _, err = db.FrontendWriterDB.Exec(`UPDATE users_val_dashboards_groups SET webhook_retries = retries + 1, webhook_last_sent = now() WHERE id = $1 AND dashboard_id = $2;`, n.Content.Webhook.DashboardGroupId, n.Content.Webhook.DashboardId) + } else { + _, err = db.FrontendWriterDB.Exec(`UPDATE users_webhooks SET retries = retries + 1, last_sent = now(), request = $2, response = $3 WHERE id = $1;`, n.Content.Webhook.ID, n.Content, errResp) + } if err != nil { log.Error(err, "error updating users_webhooks table", 0) - return + return nil } } - }(n) + return nil + }) + } + + err = g.Wait() + if err != nil { + log.Error(err, "error waiting for errgroup", 0) } return nil } @@ -402,7 +423,6 @@ func sendDiscordNotifications() error { continue // skip } - log.Infof("sending discord webhook request to %s with: %v", webhook.Url, reqs[i].Content.DiscordRequest) resp, err := client.Post(webhook.Url, "application/json", reqBody) if err != nil { log.Warnf("failed sending discord webhook request %v: %v", webhook.ID, err) From 3adaf9defda3cae9e64460e4410c17c13edbaada Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Fri, 25 Oct 2024 11:20:57 +0000 Subject: [PATCH 30/82] chore(notifications): throttle discord webhook calls --- backend/pkg/notification/sending.go | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/backend/pkg/notification/sending.go b/backend/pkg/notification/sending.go index 95c7101fb..12cdd867b 100644 --- a/backend/pkg/notification/sending.go +++ b/backend/pkg/notification/sending.go @@ -377,10 +377,13 @@ func sendDiscordNotifications() error { } notifMap[n.Content.Webhook.ID] = append(notifMap[n.Content.Webhook.ID], n) } + // use an error group to throttle webhook requests + g := &errgroup.Group{} + g.SetLimit(50) // issue at most 50 requests at a time + for _, webhook := range webhookMap { - // todo: this has the potential to spin up thousands of go routines - // should use an errgroup instead if we decide to keep the aproach - go func(webhook types.UserWebhook, reqs []types.TransitDiscord) { + webhook := webhook + g.Go(func() error { defer func() { // update retries counters in db based on end result if webhook.DashboardId == 0 && webhook.DashboardGroupId == 0 { @@ -394,7 +397,7 @@ func sendDiscordNotifications() error { // mark notifcations as sent in db ids := make([]uint64, 0) - for _, req := range reqs { + for _, req := range notifMap[webhook.ID] { ids = append(ids, req.Id) } _, err = db.WriterDb.Exec(`UPDATE notification_queue SET sent = now() where id = ANY($1)`, pq.Array(ids)) @@ -406,10 +409,10 @@ func sendDiscordNotifications() error { _, err = url.Parse(webhook.Url) if err != nil { log.Error(err, "error parsing url", 0, log.Fields{"webhook_id": webhook.ID}) - return + return nil } - for i := 0; i < len(reqs); i++ { + for i := 0; i < len(notifMap[webhook.ID]); i++ { if webhook.Retries > 5 { break // stop } @@ -417,7 +420,7 @@ func sendDiscordNotifications() error { time.Sleep(time.Duration(webhook.Retries) * time.Second) reqBody := new(bytes.Buffer) - err := json.NewEncoder(reqBody).Encode(reqs[i].Content.DiscordRequest) + err := json.NewEncoder(reqBody).Encode(notifMap[webhook.ID][i].Content.DiscordRequest) if err != nil { log.Error(err, "error marshalling discord webhook event", 0) continue // skip @@ -450,7 +453,7 @@ func sendDiscordNotifications() error { log.WarnWithFields(map[string]interface{}{"errResp.Body": utils.FirstN(errResp.Body, 1000), "webhook.Url": webhook.Url}, "error pushing discord webhook") } if webhook.DashboardId == 0 && webhook.DashboardGroupId == 0 { - _, err = db.FrontendWriterDB.Exec(`UPDATE users_webhooks SET request = $2, response = $3 WHERE id = $1;`, webhook.ID, reqs[i].Content.DiscordRequest, errResp) + _, err = db.FrontendWriterDB.Exec(`UPDATE users_webhooks SET request = $2, response = $3 WHERE id = $1;`, webhook.ID, notifMap[webhook.ID][i].Content.DiscordRequest, errResp) } if err != nil { log.Error(err, "error storing failure data in users_webhooks table", 0) @@ -460,7 +463,13 @@ func sendDiscordNotifications() error { i-- // retry, IMPORTANT to be at the END of the ELSE, otherwise the wrong index will be used in the commands above! } } - }(webhook, notifMap[webhook.ID]) + return nil + }) + } + + err = g.Wait() + if err != nil { + log.Error(err, "error waiting for errgroup", 0) } return nil From b39b06c91f5de6ca79871618ebe85cf807893da5 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Thu, 17 Oct 2024 13:33:08 +0000 Subject: [PATCH 31/82] feat(BcFormatPercent): change threshold for efficiency See: BEDS-622 Co-authored-by: marcel-bitfly <174338434+marcel-bitfly@users.noreply.github.com> --- frontend/components/bc/format/FormatPercent.vue | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frontend/components/bc/format/FormatPercent.vue b/frontend/components/bc/format/FormatPercent.vue index 4f9b41036..c6dfbc025 100644 --- a/frontend/components/bc/format/FormatPercent.vue +++ b/frontend/components/bc/format/FormatPercent.vue @@ -52,7 +52,8 @@ const data = computed(() => { } label = formatPercent(percent, config) if (props.comparePercent !== undefined) { - if (Math.abs(props.comparePercent - percent) <= 0.5) { + const thresholdToDifferenciateUnderperformerAndOverperformer = 0.25 + if (Math.abs(props.comparePercent - percent) <= thresholdToDifferenciateUnderperformerAndOverperformer) { className = 'text-equal' leadingIcon = faArrowsLeftRight compareResult = 'equal' From fcdea3682269fbbd98b4e0a2849695438c1478e0 Mon Sep 17 00:00:00 2001 From: remoterami <142154971+remoterami@users.noreply.github.com> Date: Mon, 28 Oct 2024 10:51:47 +0100 Subject: [PATCH 32/82] CR feedback --- backend/pkg/api/data_access/vdb_blocks.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/pkg/api/data_access/vdb_blocks.go b/backend/pkg/api/data_access/vdb_blocks.go index b8c1ba352..47bbd651f 100644 --- a/backend/pkg/api/data_access/vdb_blocks.go +++ b/backend/pkg/api/data_access/vdb_blocks.go @@ -46,7 +46,7 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das searchGroup := regexp.MustCompile(`^[a-zA-Z0-9_\-.\ ]+$`).MatchString(search) searchIndex := regexp.MustCompile(`^[0-9]+$`).MatchString(search) - validators := goqu.T("validators") + validators := goqu.T("validators") // could adapt data type to make handling as table/alias less confusing blocks := goqu.T("blocks") groups := goqu.T("groups") @@ -215,6 +215,8 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das if err == nil { if dashboardId.Validators == nil { // fetch filtered validators if not done yet + filteredValidatorsDs = filteredValidatorsDs. + SelectAppend(groupIdQ) validatorsQuery, validatorsArgs, err := filteredValidatorsDs.Prepared(true).ToSQL() if err != nil { return nil, nil, err From 7192f4a1aa31302ccaf15aacb34a8ed1dc899202 Mon Sep 17 00:00:00 2001 From: benji-bitfly Date: Mon, 28 Oct 2024 13:13:12 +0100 Subject: [PATCH 33/82] feat(DashboardTableValidators): change representation of `validator indexes` In `DashboardTableSummary` more then `3 indexes` should not be shown. See: BEDS-649 --- frontend/.vscode/settings.json | 1 + .../dashboard/table/DashboardTableValidators.vue | 15 +++++++++------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/frontend/.vscode/settings.json b/frontend/.vscode/settings.json index b162c363d..3657d47ec 100644 --- a/frontend/.vscode/settings.json +++ b/frontend/.vscode/settings.json @@ -7,6 +7,7 @@ "BcToggle", "DashboardChartSummaryChartFilter", "DashboardGroupManagementModal", + "DashboardTableValidators", "DashboardValidatorManagmentModal", "NotificationMachinesTable", "NotificationsClientsTable", diff --git a/frontend/components/dashboard/table/DashboardTableValidators.vue b/frontend/components/dashboard/table/DashboardTableValidators.vue index 0428f5658..477c162c7 100644 --- a/frontend/components/dashboard/table/DashboardTableValidators.vue +++ b/frontend/components/dashboard/table/DashboardTableValidators.vue @@ -58,21 +58,24 @@ const cappedValidators = computed(() => From a99fecc2bcd0f7e8126f5e6c286588fd727efd09 Mon Sep 17 00:00:00 2001 From: marcel-bitfly <174338434+marcel-bitfly@users.noreply.github.com> Date: Mon, 4 Nov 2024 08:32:28 +0100 Subject: [PATCH 70/82] fix: show `3 validator indexes` or correct `validator count` On DashboardTableSummary See: BEDS-649 --- .../dashboard/table/DashboardTableValidators.vue | 15 +++++++-------- .../components/dashboard/table/SummaryValue.vue | 8 +++++++- frontend/utils/dashboard/validator.ts | 7 ------- 3 files changed, 14 insertions(+), 16 deletions(-) diff --git a/frontend/components/dashboard/table/DashboardTableValidators.vue b/frontend/components/dashboard/table/DashboardTableValidators.vue index 477c162c7..a1359940f 100644 --- a/frontend/components/dashboard/table/DashboardTableValidators.vue +++ b/frontend/components/dashboard/table/DashboardTableValidators.vue @@ -7,7 +7,6 @@ import type { } from '~/types/dashboard/summary' import { DashboardValidatorSubsetModal } from '#components' import { getGroupLabel } from '~/utils/dashboard/group' -import { sortValidatorIds } from '~/utils/dashboard/validator' import type { DashboardKey } from '~/types/dashboard' import type { VDBGroupSummaryData, @@ -21,6 +20,7 @@ interface Props { groupId?: number, row: VDBSummaryTableRow, timeFrame?: SummaryTimeFrame, + validatorCount: number, validators: number[], } const props = defineProps() @@ -50,17 +50,16 @@ const openValidatorModal = () => { const groupName = computed(() => { return getGroupLabel($t, props.groupId, groups.value, $t('common.total')) }) - -const cappedValidators = computed(() => - sortValidatorIds(props.validators).slice(0, 10), -)