From 52db45f80c3c6ff54a71c87a104c19454cf52e09 Mon Sep 17 00:00:00 2001 From: remoterami <142154971+remoterami@users.noreply.github.com> Date: Wed, 16 Oct 2024 13:33:43 +0200 Subject: [PATCH 1/7] scheduled blocks retrieval (WIP) --- backend/pkg/api/data_access/vdb_blocks.go | 213 +++++++++++++--------- 1 file changed, 126 insertions(+), 87 deletions(-) diff --git a/backend/pkg/api/data_access/vdb_blocks.go b/backend/pkg/api/data_access/vdb_blocks.go index ca881c0fb..4ab744b00 100644 --- a/backend/pkg/api/data_access/vdb_blocks.go +++ b/backend/pkg/api/data_access/vdb_blocks.go @@ -9,6 +9,7 @@ import ( "strings" "time" + "github.com/doug-martin/goqu/v9" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/gobitfly/beaconchain/pkg/api/enums" t "github.com/gobitfly/beaconchain/pkg/api/types" @@ -17,13 +18,38 @@ import ( "github.com/gobitfly/beaconchain/pkg/commons/log" "github.com/gobitfly/beaconchain/pkg/commons/types" "github.com/gobitfly/beaconchain/pkg/commons/utils" + "github.com/lib/pq" "github.com/shopspring/decimal" ) +type table string + +// Stringer interface +func (t table) String() string { + return string(t) +} + +//func (t table) C(column string) exp.IdentifierExpression { +// return goqu.I(string(t) + "." + column) +//} + +func (t table) C(column string) string { + return string(t) + "." + column +} + func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, dashboardId t.VDBId, cursor string, colSort t.Sort[enums.VDBBlocksColumn], search string, limit uint64, protocolModes t.VDBProtocolModes) ([]t.VDBBlocksTableRow, *t.Paging, error) { // @DATA-ACCESS incorporate protocolModes + + // ------------------------------------- + // Setup var err error var currentCursor t.BlocksCursor + validatorMapping, err := d.services.GetCurrentValidatorMapping() + if err != nil { + return nil, nil, err + } + validators := table("validators") + groups := table("goups") // TODO @LuccaBitfly move validation to handler? if cursor != "" { @@ -32,83 +58,127 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das } } - // regexes taken from api handler common.go searchPubkey := regexp.MustCompile(`^0x[0-9a-fA-F]{96}$`).MatchString(search) searchGroup := regexp.MustCompile(`^[a-zA-Z0-9_\-.\ ]+$`).MatchString(search) searchIndex := regexp.MustCompile(`^[0-9]+$`).MatchString(search) - validatorMap := make(map[t.VDBValidator]bool) - params := []interface{}{} - filteredValidatorsQuery := "" - validatorMapping, err := d.services.GetCurrentValidatorMapping() - if err != nil { - return nil, nil, err + // ------------------------------------- + // Goqu Query: Determine validators filtered by search + type validatorGroup struct { + Validator t.VDBValidator `db:"validator_index"` + Group uint64 `db:"group_id"` } - - // determine validators of interest first + var filteredValidators []validatorGroup + validatorsDs := goqu.Dialect("postgres"). + From( + goqu.T("users_val_dashboards_validators").As(validators), + ). + Select( + validators.C("validator_index"), + ) if dashboardId.Validators == nil { - // could also optimize this for the average and/or the whale case; will go with some middle-ground, needs testing - // (query validators twice: once without search applied (fast) to pre-filter scheduled proposals (which are sent to db, want to minimize), - // again for blocks query with search applied to not having to send potentially huge validator-list) - startTime := time.Now() - valis, err := d.getDashboardValidators(ctx, dashboardId, nil) - log.Debugf("=== getting validators took %s", time.Since(startTime)) - if err != nil { - return nil, nil, err - } - for _, v := range valis { - validatorMap[v] = true - } + validatorsDs = validatorsDs. + Select( + // TODO mustn't be here, can be done further down + validators.C("group_id"), + ). + Where(goqu.Ex{validators.C("dashboard_id"): dashboardId.Id}) - // create a subquery to get the (potentially filtered) validators and their groups for later - params = append(params, dashboardId.Id) - selectStr := `SELECT validator_index, group_id ` - from := `FROM users_val_dashboards_validators validators ` - where := `WHERE validators.dashboard_id = $1` - extraConds := make([]string, 0, 3) + // apply search filters if searchIndex { - params = append(params, search) - extraConds = append(extraConds, fmt.Sprintf(`validator_index = $%d`, len(params))) + validatorsDs = validatorsDs.Where(goqu.Ex{validators.C("validator_index"): search}) } if searchGroup { - from += `INNER JOIN users_val_dashboards_groups groups ON validators.dashboard_id = groups.dashboard_id AND validators.group_id = groups.id ` - // escape the psql single character wildcard "_"; apply prefix-search - params = append(params, strings.Replace(search, "_", "\\_", -1)+"%") - extraConds = append(extraConds, fmt.Sprintf(`LOWER(name) LIKE LOWER($%d)`, len(params))) + validatorsDs = validatorsDs. + InnerJoin(goqu.T("users_val_dashboards_groups").As(groups), goqu.On( + goqu.Ex{validators.C("dashboard_id"): groups.C("dashboard_id")}, + goqu.Ex{validators.C("group_id"): groups.C("id")}, + )). + Where( + goqu.L("LOWER(?)", groups.C("name")).Like(strings.Replace(search, "_", "\\_", -1) + "%"), + ) } if searchPubkey { index, ok := validatorMapping.ValidatorIndices[search] - if !ok && len(extraConds) == 0 { - // don't even need to query + if !ok && !searchGroup && !searchIndex { + // searched pubkey doesn't exist, don't even need to query anything return make([]t.VDBBlocksTableRow, 0), &t.Paging{}, nil } - params = append(params, index) - extraConds = append(extraConds, fmt.Sprintf(`validator_index = $%d`, len(params))) - } - if len(extraConds) > 0 { - where += ` AND (` + strings.Join(extraConds, ` OR `) + `)` - } - filteredValidatorsQuery = selectStr + from + where + validatorsDs = validatorsDs. + Where(goqu.Ex{validators.C("validator_index"): index}) + } } else { - validators := make([]t.VDBValidator, 0, len(dashboardId.Validators)) for _, validator := range dashboardId.Validators { if searchIndex && fmt.Sprint(validator) != search || searchPubkey && validator != validatorMapping.ValidatorIndices[search] { continue } - validatorMap[validator] = true - validators = append(validators, validator) + filteredValidators = append(filteredValidators, validatorGroup{ + Validator: validator, + Group: t.DefaultGroupId, + }) if searchIndex || searchPubkey { break } } - if len(validators) == 0 { - return make([]t.VDBBlocksTableRow, 0), &t.Paging{}, nil + validatorsDs = validatorsDs. + Where(goqu.L( + validators.C("validator_index")+" = ANY(?)", pq.Array(filteredValidators)), + ) + } + + if dashboardId.Validators == nil { + validatorsQuery, validatorsArgs, err := validatorsDs.Prepared(true).ToSQL() + if err != nil { + return nil, nil, err + } + if err = d.alloyReader.SelectContext(ctx, &filteredValidators, validatorsQuery, validatorsArgs...); err != nil { + return nil, nil, err + } + } + if len(filteredValidators) == 0 { + return make([]t.VDBBlocksTableRow, 0), &t.Paging{}, nil + } + + // ------------------------------------- + // Gather scheduled blocks + // found in dutiesInfo; pass results to final query later and let db do the sorting etc + validatorSet := make(map[t.VDBValidator]bool) + for _, v := range filteredValidators { + validatorSet[v.Validator] = true + } + var scheduledProposers []t.VDBValidator + var scheduledEpochs []uint64 + var scheduledSlots []uint64 + // don't need if requested slots are in the past + latestSlot := cache.LatestSlot.Get() + onlyPrimarySort := colSort.Column == enums.VDBBlockSlot || colSort.Column == enums.VDBBlockBlock + if !onlyPrimarySort || !currentCursor.IsValid() || + currentCursor.Slot > latestSlot+1 && currentCursor.Reverse != colSort.Desc || + currentCursor.Slot < latestSlot+1 && currentCursor.Reverse == colSort.Desc { + dutiesInfo, err := d.services.GetCurrentDutiesInfo() + if err == nil { + for slot, vali := range dutiesInfo.PropAssignmentsForSlot { + // only gather scheduled slots + if _, ok := dutiesInfo.SlotStatus[slot]; ok { + continue + } + // only gather slots scheduled for our validators + if _, ok := validatorSet[vali]; !ok { + continue + } + scheduledProposers = append(scheduledProposers, dutiesInfo.PropAssignmentsForSlot[slot]) + scheduledEpochs = append(scheduledEpochs, slot/utils.Config.Chain.ClConfig.SlotsPerEpoch) + scheduledSlots = append(scheduledSlots, slot) + } + } else { + log.Debugf("duties info not available, skipping scheduled slots: %s", err) } - params = append(params, validators) } + // WIP + var proposals []struct { Proposer t.VDBValidator `db:"proposer"` Group uint64 `db:"group_id"` @@ -126,26 +196,26 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das } // handle sorting + params := make([]any, 0) where := `` orderBy := `ORDER BY ` sortOrder := ` ASC` if colSort.Desc { sortOrder = ` DESC` } - var val any + var offset any sortColName := `slot` switch colSort.Column { case enums.VDBBlockProposer: sortColName = `proposer` - val = currentCursor.Proposer + offset = currentCursor.Proposer case enums.VDBBlockStatus: sortColName = `status` - val = currentCursor.Status + offset = currentCursor.Status case enums.VDBBlockProposerReward: sortColName = `el_reward + cl_reward` - val = currentCursor.Reward + offset = currentCursor.Reward } - onlyPrimarySort := sortColName == `slot` if currentCursor.IsValid() { sign := ` > ` if colSort.Desc && !currentCursor.IsReverse() || !colSort.Desc && currentCursor.IsReverse() { @@ -163,7 +233,7 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das if onlyPrimarySort { where += `slot` + sign + fmt.Sprintf(`$%d`, len(params)) } else { - params = append(params, val) + params = append(params, offset) secSign := ` < ` if currentCursor.IsReverse() { secSign = ` > ` @@ -190,36 +260,6 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das orderBy += `, slot ` + secSort } - // Get scheduled blocks. They aren't written to blocks table, get from duties - // Will just pass scheduled proposals to query and let db do the sorting etc - var scheduledProposers []t.VDBValidator - var scheduledEpochs []uint64 - var scheduledSlots []uint64 - // don't need to query if requested slots are in the past - latestSlot := cache.LatestSlot.Get() - if !onlyPrimarySort || !currentCursor.IsValid() || - currentCursor.Slot > latestSlot+1 && currentCursor.Reverse != colSort.Desc || - currentCursor.Slot < latestSlot+1 && currentCursor.Reverse == colSort.Desc { - dutiesInfo, err := d.services.GetCurrentDutiesInfo() - if err == nil { - for slot, vali := range dutiesInfo.PropAssignmentsForSlot { - // only gather scheduled slots - if _, ok := dutiesInfo.SlotStatus[slot]; ok { - continue - } - // only gather slots scheduled for our validators - if _, ok := validatorMap[vali]; !ok { - continue - } - scheduledProposers = append(scheduledProposers, dutiesInfo.PropAssignmentsForSlot[slot]) - scheduledEpochs = append(scheduledEpochs, slot/utils.Config.Chain.ClConfig.SlotsPerEpoch) - scheduledSlots = append(scheduledSlots, slot) - } - } else { - log.Debugf("duties info not available, skipping scheduled slots: %s", err) - } - } - groupIdCol := "group_id" if dashboardId.Validators != nil { groupIdCol = fmt.Sprintf("%d AS %s", t.DefaultGroupId, groupIdCol) @@ -256,9 +296,8 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das } query += `) as u `*/ if dashboardId.Validators == nil { - cte += fmt.Sprintf(` - INNER JOIN (%s) validators ON validators.validator_index = proposer - `, filteredValidatorsQuery) + //cte += fmt.Sprintf(` + //INNER JOIN (%s) validators ON validators.validator_index = proposer`, filteredValidatorsQuery) } else { if len(where) == 0 { where += `WHERE ` From ee680640335b128eb076b1c3e2e7cd7b411a9ff8 Mon Sep 17 00:00:00 2001 From: remoterami <142154971+remoterami@users.noreply.github.com> Date: Thu, 17 Oct 2024 14:07:45 +0200 Subject: [PATCH 2/7] base query (WIP) --- backend/pkg/api/data_access/vdb_blocks.go | 213 ++++++++---------- .../api/enums/validator_dashboard_enums.go | 17 ++ backend/pkg/api/types/data_access.go | 5 +- 3 files changed, 110 insertions(+), 125 deletions(-) diff --git a/backend/pkg/api/data_access/vdb_blocks.go b/backend/pkg/api/data_access/vdb_blocks.go index b5de790af..409099ebc 100644 --- a/backend/pkg/api/data_access/vdb_blocks.go +++ b/backend/pkg/api/data_access/vdb_blocks.go @@ -49,6 +49,7 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das return nil, nil, err } validators := table("validators") + blocks := table("blocks") groups := table("goups") // TODO @LuccaBitfly move validation to handler? @@ -70,18 +71,18 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das } var filteredValidators []validatorGroup validatorsDs := goqu.Dialect("postgres"). - From( - goqu.T("users_val_dashboards_validators").As(validators), - ). Select( validators.C("validator_index"), ) if dashboardId.Validators == nil { validatorsDs = validatorsDs. - Select( + From( + goqu.T("users_val_dashboards_validators").As(validators), + ). + /*Select( // TODO mustn't be here, can be done further down validators.C("group_id"), - ). + ).*/ Where(goqu.Ex{validators.C("dashboard_id"): dashboardId.Id}) // apply search filters @@ -123,9 +124,9 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das } } validatorsDs = validatorsDs. - Where(goqu.L( - validators.C("validator_index")+" = ANY(?)", pq.Array(filteredValidators)), - ) + From( + goqu.L("unnest(?)", pq.Array(filteredValidators)).As("validator_index"), + ).As(string(validators)) } if dashboardId.Validators == nil { @@ -177,124 +178,84 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das } } - // WIP - - var proposals []struct { - Proposer t.VDBValidator `db:"proposer"` - Group uint64 `db:"group_id"` - Epoch uint64 `db:"epoch"` - Slot uint64 `db:"slot"` - Status uint64 `db:"status"` - Block sql.NullInt64 `db:"exec_block_number"` - FeeRecipient []byte `db:"fee_recipient"` - ElReward decimal.NullDecimal `db:"el_reward"` - ClReward decimal.NullDecimal `db:"cl_reward"` - GraffitiText string `db:"graffiti_text"` - - // for cursor only - Reward decimal.Decimal - } - - // handle sorting - params := make([]any, 0) - where := `` - orderBy := `ORDER BY ` - sortOrder := ` ASC` - if colSort.Desc { - sortOrder = ` DESC` + // Sorting and pagination if cursor is present + defaultColumns := []t.SortColumn{ + {Column: enums.VDBBlocksColumns.Slot.ToString(), Desc: true, Offset: currentCursor.Slot}, } var offset any - sortColName := `slot` switch colSort.Column { - case enums.VDBBlockProposer: - sortColName = `proposer` + case enums.VDBBlocksColumns.Proposer: offset = currentCursor.Proposer - case enums.VDBBlockStatus: - sortColName = `status` - offset = currentCursor.Status - case enums.VDBBlockProposerReward: - sortColName = `el_reward + cl_reward` + case enums.VDBBlocksColumns.Block: + offset = currentCursor.Block + case enums.VDBBlocksColumns.Status: + offset = fmt.Sprintf("%d", currentCursor.Status) // type of 'status' column is text for some reason + case enums.VDBBlocksColumns.ProposerReward: offset = currentCursor.Reward } - if currentCursor.IsValid() { - sign := ` > ` - if colSort.Desc && !currentCursor.IsReverse() || !colSort.Desc && currentCursor.IsReverse() { - sign = ` < ` - } - if currentCursor.IsReverse() { - if sortOrder == ` ASC` { - sortOrder = ` DESC` - } else { - sortOrder = ` ASC` - } - } - params = append(params, currentCursor.Slot) - where += `WHERE (` - if onlyPrimarySort { - where += `slot` + sign + fmt.Sprintf(`$%d`, len(params)) - } else { - params = append(params, offset) - secSign := ` < ` - if currentCursor.IsReverse() { - secSign = ` > ` - } - if sortColName == "status" { - // explicit cast to int because type of 'status' column is text for some reason - sortColName += "::int" - } - where += fmt.Sprintf(`(slot`+secSign+`$%d AND `+sortColName+` = $%d) OR `+sortColName+sign+`$%d`, len(params)-1, len(params), len(params)) - } - where += `) ` + + order, directions := applySortAndPagination(defaultColumns, t.SortColumn{Column: colSort.Column.ToString(), Desc: colSort.Desc, Offset: offset}, currentCursor.GenericCursor) + validatorsDs = validatorsDs.Order(order...) + if directions != nil { + validatorsDs = validatorsDs.Where(directions) } - if sortOrder == ` ASC` { - sortOrder += ` NULLS FIRST` + + // group id + if dashboardId.Validators == nil { + validatorsDs = validatorsDs.Select( + validators.C("group_id"), + ) } else { - sortOrder += ` NULLS LAST` - } - orderBy += sortColName + sortOrder - secSort := `DESC` - if !onlyPrimarySort { - if currentCursor.IsReverse() { - secSort = `ASC` - } - orderBy += `, slot ` + secSort + validatorsDs = validatorsDs.Select( + goqu.L("?", t.DefaultGroupId).As("group_id"), + ) } - groupIdCol := "group_id" - if dashboardId.Validators != nil { - groupIdCol = fmt.Sprintf("%d AS %s", t.DefaultGroupId, groupIdCol) - } - selectFields := fmt.Sprintf(` - blocks.proposer, - blocks.epoch, - blocks.slot, - %s, - blocks.status, - exec_block_number, - COALESCE(rb.proposer_fee_recipient, blocks.exec_fee_recipient) AS fee_recipient, - COALESCE(rb.value / 1e18, ep.fee_recipient_reward) AS el_reward, - cp.cl_attestations_reward / 1e9 + cp.cl_sync_aggregate_reward / 1e9 + cp.cl_slashing_inclusion_reward / 1e9 as cl_reward, - blocks.graffiti_text`, groupIdCol) + validatorsDs = validatorsDs. + Select( + blocks.C("proposer"), + blocks.C("epoch"), + blocks.C("slot"), + blocks.C("status"), + blocks.C("exec_block_number"), + blocks.C("graffiti_text"), + ). + LeftJoin(goqu.T("consensus_payloads").As("cp"), goqu.On( + goqu.Ex{blocks.C("slot"): goqu.I("cp.slot")}, + )). + LeftJoin(goqu.T("execution_payloads").As("ep"), goqu.On( + goqu.Ex{blocks.C("exec_block_hash"): goqu.I("ep.block_hash")}, + )). + LeftJoin( + // relay bribe deduplication; select most likely (=max) relay bribe value for each block + goqu.Lateral(goqu.Dialect("postgres"). + From(goqu.T("relays_blocks")). + Select( + goqu.I("relays_blocks.exec_block_hash"), + goqu.MAX(goqu.I("relays_blocks.value")).As("value")). + // needed? TODO test + // Where(goqu.L("relays_blocks.exec_block_hash = blocks.exec_block_hash")). + GroupBy("exec_block_hash")).As("rb"), + goqu.On( + goqu.Ex{"rb.exec_block_hash": blocks.C("exec_block_hash")}, + ), + ). + Select( + goqu.COALESCE(goqu.I("rb.proposer_fee_recipient"), blocks.C("exec_fee_recipient")).As("fee_recipient"), + goqu.COALESCE(goqu.L("rb.value / 1e18"), goqu.I("ep.fee_recipient_reward")).As("el_reward"), + goqu.L("cp.cl_attestations_reward / 1e9 + cp.cl_sync_aggregate_reward / 1e9 + cp.cl_slashing_inclusion_reward / 1e9").As("cl_reward"), + ) + + // union scheduled blocks if present + // WIP + + params := make([]any, 0) + selectFields, where, orderBy, groupIdCol, sortColName := "", "", "", "", "" cte := fmt.Sprintf(`WITH past_blocks AS (SELECT %s FROM blocks `, selectFields) - /*if dashboardId.Validators == nil { - query += ` - LEFT JOIN cached_proposal_rewards ON cached_proposal_rewards.dashboard_id = $1 AND blocks.slot = cached_proposal_rewards.slot - ` - } else { - query += ` - LEFT JOIN execution_payloads ep ON ep.block_hash = blocks.exec_block_hash - LEFT JOIN relays_blocks rb ON rb.exec_block_hash = blocks.exec_block_hash - ` - } - // shrink selection to our filtered validators - if len(scheduledProposers) > 0 { - query += `)` - } - query += `) as u `*/ if dashboardId.Validators == nil { //cte += fmt.Sprintf(` //INNER JOIN (%s) validators ON validators.validator_index = proposer`, filteredValidatorsQuery) @@ -311,17 +272,6 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das limitStr := fmt.Sprintf(` LIMIT $%d `, len(params)) - // relay bribe deduplication; select most likely (=max) relay bribe value for each block - cte += ` - LEFT JOIN consensus_payloads cp on blocks.slot = cp.slot - LEFT JOIN execution_payloads ep ON ep.block_hash = blocks.exec_block_hash - LEFT JOIN LATERAL (SELECT exec_block_hash, proposer_fee_recipient, max(value) as value - FROM relays_blocks - WHERE relays_blocks.exec_block_hash = blocks.exec_block_hash - GROUP BY exec_block_hash, proposer_fee_recipient - ) rb ON rb.exec_block_hash = blocks.exec_block_hash - ) - ` from := `past_blocks ` selectStr := `SELECT * FROM ` @@ -375,7 +325,26 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das query = selectStr + from + where + orderBy + limitStr } + var proposals []struct { + Proposer t.VDBValidator `db:"proposer"` + Group uint64 `db:"group_id"` + Epoch uint64 `db:"epoch"` + Slot uint64 `db:"slot"` + Status uint64 `db:"status"` + Block sql.NullInt64 `db:"exec_block_number"` + FeeRecipient []byte `db:"fee_recipient"` + ElReward decimal.NullDecimal `db:"el_reward"` + ClReward decimal.NullDecimal `db:"cl_reward"` + GraffitiText string `db:"graffiti_text"` + + // for cursor only + Reward decimal.Decimal + } startTime := time.Now() + _, _, err = validatorsDs.Prepared(true).ToSQL() + if err != nil { + return nil, nil, err + } err = d.alloyReader.SelectContext(ctx, &proposals, cte+query, params...) log.Debugf("=== getting past blocks took %s", time.Since(startTime)) if err != nil { diff --git a/backend/pkg/api/enums/validator_dashboard_enums.go b/backend/pkg/api/enums/validator_dashboard_enums.go index 58e87f337..2646244e8 100644 --- a/backend/pkg/api/enums/validator_dashboard_enums.go +++ b/backend/pkg/api/enums/validator_dashboard_enums.go @@ -156,6 +156,23 @@ func (VDBBlocksColumn) NewFromString(s string) VDBBlocksColumn { } } +func (c VDBBlocksColumn) ToString() string { + switch c { + case VDBBlockProposer: + return "proposer" + case VDBBlockSlot: + return "slot" + case VDBBlockBlock: + return "block" + case VDBBlockStatus: + return "status" + case VDBBlockProposerReward: + return "reward" + default: + return "" + } +} + var VDBBlocksColumns = struct { Proposer VDBBlocksColumn Slot VDBBlocksColumn diff --git a/backend/pkg/api/types/data_access.go b/backend/pkg/api/types/data_access.go index db35871fb..3b1d77c16 100644 --- a/backend/pkg/api/types/data_access.go +++ b/backend/pkg/api/types/data_access.go @@ -164,11 +164,10 @@ type UserCredentialInfo struct { type BlocksCursor struct { GenericCursor - Slot uint64 // basically the same as Block, Epoch, Age; mandatory, used to index - // optional, max one of those (for now) Proposer uint64 - Group uint64 + Slot uint64 // same as Age + Block uint64 Status uint64 Reward decimal.Decimal } From cf997c42bd18f72054a4cdd86765b30c93befbdb Mon Sep 17 00:00:00 2001 From: remoterami <142154971+remoterami@users.noreply.github.com> Date: Fri, 18 Oct 2024 14:44:09 +0200 Subject: [PATCH 3/7] union past and scheduled blocks, syntax fixed --- backend/pkg/api/data_access/general.go | 13 +- backend/pkg/api/data_access/vdb_blocks.go | 224 +++++++----------- .../api/enums/validator_dashboard_enums.go | 2 +- backend/pkg/api/types/data_access.go | 13 +- .../dashboard/table/DashboardTableBlocks.vue | 2 +- 5 files changed, 108 insertions(+), 146 deletions(-) diff --git a/backend/pkg/api/data_access/general.go b/backend/pkg/api/data_access/general.go index 3c94db8a1..26d051dc9 100644 --- a/backend/pkg/api/data_access/general.go +++ b/backend/pkg/api/data_access/general.go @@ -62,6 +62,9 @@ func applySortAndPagination(defaultColumns []types.SortColumn, primary types.Sor if primary.Offset == nil { queryOrderColumns[0].Offset = column.Offset } + if len(primary.Table) == 0 { + queryOrderColumns[0].Table = column.Table + } continue } queryOrderColumns = append(queryOrderColumns, column) @@ -74,9 +77,9 @@ func applySortAndPagination(defaultColumns []types.SortColumn, primary types.Sor if cursor.IsReverse() { column.Desc = !column.Desc } - colOrder := goqu.C(column.Column).Asc() + colOrder := column.Expr().Asc() if column.Desc { - colOrder = goqu.C(column.Column).Desc() + colOrder = column.Expr().Desc() } queryOrder = append(queryOrder, colOrder) } @@ -87,15 +90,15 @@ func applySortAndPagination(defaultColumns []types.SortColumn, primary types.Sor // reverse order to nest conditions for i := len(queryOrderColumns) - 1; i >= 0; i-- { column := queryOrderColumns[i] - colWhere := goqu.C(column.Column).Gt(column.Offset) + colWhere := column.Expr().Gt(column.Offset) if column.Desc { - colWhere = goqu.C(column.Column).Lt(column.Offset) + colWhere = column.Expr().Lt(column.Offset) } if queryWhere == nil { queryWhere = colWhere } else { - queryWhere = goqu.And(goqu.C(column.Column).Eq(column.Offset), queryWhere) + queryWhere = goqu.And(column.Expr().Eq(column.Offset), queryWhere) queryWhere = goqu.Or(colWhere, queryWhere) } } diff --git a/backend/pkg/api/data_access/vdb_blocks.go b/backend/pkg/api/data_access/vdb_blocks.go index 409099ebc..f45ec2ac1 100644 --- a/backend/pkg/api/data_access/vdb_blocks.go +++ b/backend/pkg/api/data_access/vdb_blocks.go @@ -22,21 +22,6 @@ import ( "github.com/shopspring/decimal" ) -type table string - -// Stringer interface -func (t table) String() string { - return string(t) -} - -//func (t table) C(column string) exp.IdentifierExpression { -// return goqu.I(string(t) + "." + column) -//} - -func (t table) C(column string) string { - return string(t) + "." + column -} - func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, dashboardId t.VDBId, cursor string, colSort t.Sort[enums.VDBBlocksColumn], search string, limit uint64, protocolModes t.VDBProtocolModes) ([]t.VDBBlocksTableRow, *t.Paging, error) { // @DATA-ACCESS incorporate protocolModes @@ -48,9 +33,9 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das if err != nil { return nil, nil, err } - validators := table("validators") - blocks := table("blocks") - groups := table("goups") + validators := goqu.T("users_val_dashboards_validators").As("validators") + blocks := goqu.T("blocks") + groups := goqu.T("goups") // TODO @LuccaBitfly move validation to handler? if cursor != "" { @@ -72,31 +57,24 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das var filteredValidators []validatorGroup validatorsDs := goqu.Dialect("postgres"). Select( - validators.C("validator_index"), + "validator_index", ) if dashboardId.Validators == nil { validatorsDs = validatorsDs. - From( - goqu.T("users_val_dashboards_validators").As(validators), - ). - /*Select( - // TODO mustn't be here, can be done further down - validators.C("group_id"), - ).*/ - Where(goqu.Ex{validators.C("dashboard_id"): dashboardId.Id}) - + From(validators). + Where(validators.Col("dashboard_id").Eq(dashboardId.Id)) // apply search filters if searchIndex { - validatorsDs = validatorsDs.Where(goqu.Ex{validators.C("validator_index"): search}) + validatorsDs = validatorsDs.Where(validators.Col("validator_index").Eq(search)) } if searchGroup { validatorsDs = validatorsDs. InnerJoin(goqu.T("users_val_dashboards_groups").As(groups), goqu.On( - goqu.Ex{validators.C("dashboard_id"): groups.C("dashboard_id")}, - goqu.Ex{validators.C("group_id"): groups.C("id")}, + validators.Col("group_id").Eq(groups.Col("id")), + validators.Col("dashboard_id").Eq(groups.Col("dashboard_id")), )). Where( - goqu.L("LOWER(?)", groups.C("name")).Like(strings.Replace(search, "_", "\\_", -1) + "%"), + goqu.L("LOWER(?)", groups.Col("name")).Like(strings.Replace(search, "_", "\\_", -1) + "%"), ) } if searchPubkey { @@ -107,7 +85,7 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das } validatorsDs = validatorsDs. - Where(goqu.Ex{validators.C("validator_index"): index}) + Where(validators.Col("validator_index").Eq(index)) } } else { for _, validator := range dashboardId.Validators { @@ -126,7 +104,7 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das validatorsDs = validatorsDs. From( goqu.L("unnest(?)", pq.Array(filteredValidators)).As("validator_index"), - ).As(string(validators)) + ).As("validators") // TODO ? } if dashboardId.Validators == nil { @@ -180,51 +158,37 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das // Sorting and pagination if cursor is present defaultColumns := []t.SortColumn{ - {Column: enums.VDBBlocksColumns.Slot.ToString(), Desc: true, Offset: currentCursor.Slot}, + {Column: enums.VDBBlocksColumns.Slot.ToString(), Table: blocks.GetTable(), Desc: true, Offset: currentCursor.Slot}, } var offset any + var table string switch colSort.Column { case enums.VDBBlocksColumns.Proposer: offset = currentCursor.Proposer case enums.VDBBlocksColumns.Block: offset = currentCursor.Block + table = blocks.GetTable() case enums.VDBBlocksColumns.Status: offset = fmt.Sprintf("%d", currentCursor.Status) // type of 'status' column is text for some reason case enums.VDBBlocksColumns.ProposerReward: offset = currentCursor.Reward } - order, directions := applySortAndPagination(defaultColumns, t.SortColumn{Column: colSort.Column.ToString(), Desc: colSort.Desc, Offset: offset}, currentCursor.GenericCursor) + order, directions := applySortAndPagination(defaultColumns, t.SortColumn{Column: colSort.Column.ToString(), Table: table, Desc: colSort.Desc, Offset: offset}, currentCursor.GenericCursor) validatorsDs = validatorsDs.Order(order...) if directions != nil { validatorsDs = validatorsDs.Where(directions) } - // group id - if dashboardId.Validators == nil { - validatorsDs = validatorsDs.Select( - validators.C("group_id"), - ) - } else { - validatorsDs = validatorsDs.Select( - goqu.L("?", t.DefaultGroupId).As("group_id"), - ) - } - validatorsDs = validatorsDs. - Select( - blocks.C("proposer"), - blocks.C("epoch"), - blocks.C("slot"), - blocks.C("status"), - blocks.C("exec_block_number"), - blocks.C("graffiti_text"), - ). + InnerJoin(blocks, goqu.On( + blocks.Col("proposer").Eq(validators.Col("validator_index")), + )). LeftJoin(goqu.T("consensus_payloads").As("cp"), goqu.On( - goqu.Ex{blocks.C("slot"): goqu.I("cp.slot")}, + blocks.Col("slot").Eq(goqu.I("cp.slot")), )). LeftJoin(goqu.T("execution_payloads").As("ep"), goqu.On( - goqu.Ex{blocks.C("exec_block_hash"): goqu.I("ep.block_hash")}, + blocks.Col("exec_block_hash").Eq(goqu.I("ep.block_hash")), )). LeftJoin( // relay bribe deduplication; select most likely (=max) relay bribe value for each block @@ -232,101 +196,85 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das From(goqu.T("relays_blocks")). Select( goqu.I("relays_blocks.exec_block_hash"), + goqu.I("relays_blocks.proposer_fee_recipient"), goqu.MAX(goqu.I("relays_blocks.value")).As("value")). - // needed? TODO test - // Where(goqu.L("relays_blocks.exec_block_hash = blocks.exec_block_hash")). - GroupBy("exec_block_hash")).As("rb"), + GroupBy( + "exec_block_hash", + "proposer_fee_recipient", + )).As("rb"), goqu.On( - goqu.Ex{"rb.exec_block_hash": blocks.C("exec_block_hash")}, + goqu.I("rb.exec_block_hash").Eq(blocks.Col("exec_block_hash")), ), ). - Select( - goqu.COALESCE(goqu.I("rb.proposer_fee_recipient"), blocks.C("exec_fee_recipient")).As("fee_recipient"), + SelectAppend( + blocks.Col("epoch"), + blocks.Col("slot"), + blocks.Col("status"), + blocks.Col("exec_block_number"), + blocks.Col("graffiti_text"), + goqu.COALESCE(goqu.I("rb.proposer_fee_recipient"), blocks.Col("exec_fee_recipient")).As("fee_recipient"), goqu.COALESCE(goqu.L("rb.value / 1e18"), goqu.I("ep.fee_recipient_reward")).As("el_reward"), goqu.L("cp.cl_attestations_reward / 1e9 + cp.cl_sync_aggregate_reward / 1e9 + cp.cl_slashing_inclusion_reward / 1e9").As("cl_reward"), - ) - - // union scheduled blocks if present - // WIP - - params := make([]any, 0) - selectFields, where, orderBy, groupIdCol, sortColName := "", "", "", "", "" - cte := fmt.Sprintf(`WITH past_blocks AS (SELECT - %s - FROM blocks - `, selectFields) + ). + Limit(uint(limit + 1)) - if dashboardId.Validators == nil { - //cte += fmt.Sprintf(` - //INNER JOIN (%s) validators ON validators.validator_index = proposer`, filteredValidatorsQuery) - } else { - if len(where) == 0 { - where += `WHERE ` - } else { - where += `AND ` - } - where += `proposer = ANY($1) ` + // Group id + groupId := validators.Col("group_id") + if dashboardId.Validators != nil { + groupId = goqu.V(t.DefaultGroupId).As("group_id").GetAs() } + validatorsDs = validatorsDs.SelectAppend(groupId) - params = append(params, limit+1) - limitStr := fmt.Sprintf(` - LIMIT $%d - `, len(params)) + /* + if dashboardId.Validators == nil { + validatorsDs = validatorsDs.Select( + validators.Col("group_id"), + ) + } else { + validatorsDs = validatorsDs.Select( + goqu.L("?", t.DefaultGroupId).As("group_id"), + ) + }*/ - from := `past_blocks ` - selectStr := `SELECT * FROM ` + // union scheduled blocks if present + // WIP - query := selectStr + from + where + orderBy + limitStr - // supply scheduled proposals, if any + finalDs := validatorsDs if len(scheduledProposers) > 0 { - // distinct to filter out duplicates in an edge case (if dutiesInfo didn't update yet after a block was proposed, but the blocks table was) + scheduledDs := goqu.Dialect("postgres"). + From( + goqu.L("unnest(?, ?, ?) AS prov(validator_index, epoch, slot)", pq.Array(scheduledProposers), pq.Array(scheduledEpochs), pq.Array(scheduledSlots)), + ). + Select( + goqu.C("validator_index"), + goqu.C("epoch"), + goqu.C("slot"), + goqu.V("0").As("status"), + goqu.V(nil).As("exec_block_number"), + goqu.V(nil).As("fee_recipient"), + goqu.V(nil).As("el_reward"), + goqu.V(nil).As("cl_reward"), + goqu.V(nil).As("graffiti_text"), + ). + As("scheduled_blocks") + + // distinct + block number ordering to filter out duplicates in an edge case (if dutiesInfo didn't update yet after a block was proposed, but the blocks table was) // might be possible to remove this once the TODO in service_slot_viz.go:startSlotVizDataService is resolved - params = append(params, scheduledProposers) - params = append(params, scheduledEpochs) - params = append(params, scheduledSlots) - cte += fmt.Sprintf(`, - scheduled_blocks as ( - SELECT - prov.proposer, - prov.epoch, - prov.slot, - %s, - '0'::text AS status, - NULL::int AS exec_block_number, - ''::bytea AS fee_recipient, - NULL::float AS el_reward, - NULL::float AS cl_reward, - ''::text AS graffiti_text - FROM unnest($%d::int[], $%d::int[], $%d::int[]) AS prov(proposer, epoch, slot) - `, groupIdCol, len(params)-2, len(params)-1, len(params)) - if dashboardId.Validators == nil { - // add group id - cte += fmt.Sprintf(`INNER JOIN users_val_dashboards_validators validators - ON validators.dashboard_id = $1 - AND validators.validator_index = ANY($%d::int[]) - `, len(params)-2) - } - cte += `) ` - distinct := "slot" + finalDs = validatorsDs. + Union(scheduledDs). + Where(directions). + Order(order...). + OrderAppend(goqu.C("exec_block_number").Desc().NullsLast()). + Limit(uint(limit + 1)). + Distinct(blocks.Col("slot")) if !onlyPrimarySort { - distinct = sortColName + ", " + distinct + finalDs = finalDs. + Distinct(blocks.Col("slot"), blocks.Col("exec_block_number")) } - // keep all ordering, sorting etc - selectStr = `SELECT DISTINCT ON (` + distinct + `) * FROM ` - // encapsulate past blocks query to ensure performance - from = `( - ( ` + query + ` ) - UNION ALL - SELECT * FROM scheduled_blocks - ) as combined - ` - // make sure the distinct clause filters out the correct duplicated row (e.g. block=nil) - orderBy += `, exec_block_number NULLS LAST` - query = selectStr + from + where + orderBy + limitStr } var proposals []struct { - Proposer t.VDBValidator `db:"proposer"` + Proposer t.VDBValidator `db:"validator_index"` Group uint64 `db:"group_id"` Epoch uint64 `db:"epoch"` Slot uint64 `db:"slot"` @@ -341,11 +289,11 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das Reward decimal.Decimal } startTime := time.Now() - _, _, err = validatorsDs.Prepared(true).ToSQL() + query, args, err := finalDs.Prepared(true).ToSQL() if err != nil { return nil, nil, err } - err = d.alloyReader.SelectContext(ctx, &proposals, cte+query, params...) + err = d.alloyReader.SelectContext(ctx, &proposals, query, args...) log.Debugf("=== getting past blocks took %s", time.Since(startTime)) if err != nil { return nil, nil, err @@ -389,11 +337,11 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das } graffiti := proposal.GraffitiText data[i].Graffiti = &graffiti + block := uint64(proposal.Block.Int64) + data[i].Block = &block if proposal.Status == 3 { continue } - block := uint64(proposal.Block.Int64) - data[i].Block = &block var reward t.ClElValue[decimal.Decimal] if proposal.ElReward.Valid { rewardRecp := t.Address{ diff --git a/backend/pkg/api/enums/validator_dashboard_enums.go b/backend/pkg/api/enums/validator_dashboard_enums.go index 2646244e8..228b224dc 100644 --- a/backend/pkg/api/enums/validator_dashboard_enums.go +++ b/backend/pkg/api/enums/validator_dashboard_enums.go @@ -163,7 +163,7 @@ func (c VDBBlocksColumn) ToString() string { case VDBBlockSlot: return "slot" case VDBBlockBlock: - return "block" + return "exec_block_number" case VDBBlockStatus: return "status" case VDBBlockProposerReward: diff --git a/backend/pkg/api/types/data_access.go b/backend/pkg/api/types/data_access.go index 3b1d77c16..70961c5f2 100644 --- a/backend/pkg/api/types/data_access.go +++ b/backend/pkg/api/types/data_access.go @@ -1,8 +1,11 @@ package types import ( + "database/sql" "time" + "github.com/doug-martin/goqu/v9" + "github.com/doug-martin/goqu/v9/exp" "github.com/gobitfly/beaconchain/pkg/api/enums" "github.com/gobitfly/beaconchain/pkg/consapi/types" "github.com/gobitfly/beaconchain/pkg/monitoring/constants" @@ -24,11 +27,19 @@ type Sort[T enums.Enum] struct { type SortColumn struct { Column string + Table string // optional Desc bool // represents value from cursor Offset any } +func (s SortColumn) Expr() exp.IdentifierExpression { + if s.Table != "" { + return goqu.T(s.Table).Col(s.Column) + } + return goqu.C(s.Column) +} + type VDBIdPrimary int type VDBIdPublic string type VDBIdValidatorSet []VDBValidator @@ -167,7 +178,7 @@ type BlocksCursor struct { Proposer uint64 Slot uint64 // same as Age - Block uint64 + Block sql.NullInt64 Status uint64 Reward decimal.Decimal } diff --git a/frontend/components/dashboard/table/DashboardTableBlocks.vue b/frontend/components/dashboard/table/DashboardTableBlocks.vue index daa3a6e2f..01227c95f 100644 --- a/frontend/components/dashboard/table/DashboardTableBlocks.vue +++ b/frontend/components/dashboard/table/DashboardTableBlocks.vue @@ -52,7 +52,7 @@ const loadData = (query?: TableQueryParams) => { if (!query) { query = { limit: pageSize.value, - sort: 'block:desc', + sort: 'slot:desc', } } setQuery(query, true, true) From 327a88e4c8b90275a8b72d355269d17281f83cc3 Mon Sep 17 00:00:00 2001 From: remoterami <142154971+remoterami@users.noreply.github.com> Date: Mon, 21 Oct 2024 13:47:15 +0200 Subject: [PATCH 4/7] extended sorting/paging features, fixed blocks query --- backend/pkg/api/data_access/general.go | 19 +- backend/pkg/api/data_access/notifications.go | 38 +-- backend/pkg/api/data_access/vdb_blocks.go | 282 +++++++++--------- backend/pkg/api/enums/notifications_enums.go | 50 ++-- .../api/enums/validator_dashboard_enums.go | 25 +- backend/pkg/api/types/data_access.go | 16 +- 6 files changed, 223 insertions(+), 207 deletions(-) diff --git a/backend/pkg/api/data_access/general.go b/backend/pkg/api/data_access/general.go index 26d051dc9..d629d52df 100644 --- a/backend/pkg/api/data_access/general.go +++ b/backend/pkg/api/data_access/general.go @@ -62,9 +62,6 @@ func applySortAndPagination(defaultColumns []types.SortColumn, primary types.Sor if primary.Offset == nil { queryOrderColumns[0].Offset = column.Offset } - if len(primary.Table) == 0 { - queryOrderColumns[0].Table = column.Table - } continue } queryOrderColumns = append(queryOrderColumns, column) @@ -77,9 +74,9 @@ func applySortAndPagination(defaultColumns []types.SortColumn, primary types.Sor if cursor.IsReverse() { column.Desc = !column.Desc } - colOrder := column.Expr().Asc() + colOrder := column.Column.Asc() if column.Desc { - colOrder = column.Expr().Desc() + colOrder = column.Column.Desc() } queryOrder = append(queryOrder, colOrder) } @@ -90,15 +87,21 @@ func applySortAndPagination(defaultColumns []types.SortColumn, primary types.Sor // reverse order to nest conditions for i := len(queryOrderColumns) - 1; i >= 0; i-- { column := queryOrderColumns[i] - colWhere := column.Expr().Gt(column.Offset) + var colWhere exp.Expression + + // current convention is the psql default (ASC: nulls last, DESC: nulls first) + colWhere = goqu.Or(column.Column.Gt(column.Offset), column.Column.IsNull()) if column.Desc { - colWhere = column.Expr().Lt(column.Offset) + colWhere = column.Column.Lt(column.Offset) + if column.Offset == nil { + colWhere = goqu.Or(colWhere, column.Column.IsNull()) + } } if queryWhere == nil { queryWhere = colWhere } else { - queryWhere = goqu.And(column.Expr().Eq(column.Offset), queryWhere) + queryWhere = goqu.And(column.Column.Eq(column.Offset), queryWhere) queryWhere = goqu.Or(colWhere, queryWhere) } } diff --git a/backend/pkg/api/data_access/notifications.go b/backend/pkg/api/data_access/notifications.go index 776308329..7351de0c6 100644 --- a/backend/pkg/api/data_access/notifications.go +++ b/backend/pkg/api/data_access/notifications.go @@ -332,14 +332,14 @@ func (d *DataAccessService) GetDashboardNotifications(ctx context.Context, userI // sorting defaultColumns := []t.SortColumn{ - {Column: enums.NotificationsDashboardsColumns.Timestamp.ToString(), Desc: true, Offset: currentCursor.Epoch}, - {Column: enums.NotificationsDashboardsColumns.DashboardName.ToString(), Desc: false, Offset: currentCursor.DashboardName}, - {Column: enums.NotificationsDashboardsColumns.DashboardId.ToString(), Desc: false, Offset: currentCursor.DashboardId}, - {Column: enums.NotificationsDashboardsColumns.GroupName.ToString(), Desc: false, Offset: currentCursor.GroupName}, - {Column: enums.NotificationsDashboardsColumns.GroupId.ToString(), Desc: false, Offset: currentCursor.GroupId}, - {Column: enums.NotificationsDashboardsColumns.ChainId.ToString(), Desc: true, Offset: currentCursor.ChainId}, - } - order, directions := applySortAndPagination(defaultColumns, t.SortColumn{Column: colSort.Column.ToString(), Desc: colSort.Desc}, currentCursor.GenericCursor) + {Column: enums.NotificationsDashboardsColumns.Timestamp.ToExpr(), Desc: true, Offset: currentCursor.Epoch}, + {Column: enums.NotificationsDashboardsColumns.DashboardName.ToExpr(), Desc: false, Offset: currentCursor.DashboardName}, + {Column: enums.NotificationsDashboardsColumns.DashboardId.ToExpr(), Desc: false, Offset: currentCursor.DashboardId}, + {Column: enums.NotificationsDashboardsColumns.GroupName.ToExpr(), Desc: false, Offset: currentCursor.GroupName}, + {Column: enums.NotificationsDashboardsColumns.GroupId.ToExpr(), Desc: false, Offset: currentCursor.GroupId}, + {Column: enums.NotificationsDashboardsColumns.ChainId.ToExpr(), Desc: true, Offset: currentCursor.ChainId}, + } + order, directions := applySortAndPagination(defaultColumns, t.SortColumn{Column: colSort.Column.ToExpr(), Desc: colSort.Desc}, currentCursor.GenericCursor) unionQuery = unionQuery.Order(order...) if directions != nil { unionQuery = unionQuery.Where(directions) @@ -659,9 +659,9 @@ func (d *DataAccessService) GetMachineNotifications(ctx context.Context, userId // Sorting and limiting if cursor is present defaultColumns := []t.SortColumn{ - {Column: enums.NotificationsMachinesColumns.Timestamp.ToString(), Desc: true, Offset: currentCursor.Epoch}, - {Column: enums.NotificationsMachinesColumns.MachineId.ToString(), Desc: false, Offset: currentCursor.MachineId}, - {Column: enums.NotificationsMachinesColumns.EventType.ToString(), Desc: false, Offset: currentCursor.EventType}, + {Column: enums.NotificationsMachinesColumns.Timestamp.ToExpr(), Desc: true, Offset: currentCursor.Epoch}, + {Column: enums.NotificationsMachinesColumns.MachineId.ToExpr(), Desc: false, Offset: currentCursor.MachineId}, + {Column: enums.NotificationsMachinesColumns.EventType.ToExpr(), Desc: false, Offset: currentCursor.EventType}, } var offset interface{} switch colSort.Column { @@ -671,7 +671,7 @@ func (d *DataAccessService) GetMachineNotifications(ctx context.Context, userId offset = currentCursor.EventThreshold } - order, directions := applySortAndPagination(defaultColumns, t.SortColumn{Column: colSort.Column.ToString(), Desc: colSort.Desc, Offset: offset}, currentCursor.GenericCursor) + order, directions := applySortAndPagination(defaultColumns, t.SortColumn{Column: colSort.Column.ToExpr(), Desc: colSort.Desc, Offset: offset}, currentCursor.GenericCursor) ds = ds.Order(order...) if directions != nil { ds = ds.Where(directions) @@ -780,10 +780,10 @@ func (d *DataAccessService) GetClientNotifications(ctx context.Context, userId u // Sorting and limiting if cursor is present // Rows can be uniquely identified by (epoch, client) defaultColumns := []t.SortColumn{ - {Column: enums.NotificationsClientsColumns.Timestamp.ToString(), Desc: true, Offset: currentCursor.Epoch}, - {Column: enums.NotificationsClientsColumns.ClientName.ToString(), Desc: false, Offset: currentCursor.Client}, + {Column: enums.NotificationsClientsColumns.Timestamp.ToExpr(), Desc: true, Offset: currentCursor.Epoch}, + {Column: enums.NotificationsClientsColumns.ClientName.ToExpr(), Desc: false, Offset: currentCursor.Client}, } - order, directions := applySortAndPagination(defaultColumns, t.SortColumn{Column: colSort.Column.ToString(), Desc: colSort.Desc}, currentCursor.GenericCursor) + order, directions := applySortAndPagination(defaultColumns, t.SortColumn{Column: colSort.Column.ToExpr(), Desc: colSort.Desc}, currentCursor.GenericCursor) ds = ds.Order(order...) if directions != nil { ds = ds.Where(directions) @@ -1071,11 +1071,11 @@ func (d *DataAccessService) GetNetworkNotifications(ctx context.Context, userId // Sorting and limiting if cursor is present // Rows can be uniquely identified by (epoch, network, event_type) defaultColumns := []t.SortColumn{ - {Column: enums.NotificationNetworksColumns.Timestamp.ToString(), Desc: true, Offset: currentCursor.Epoch}, - {Column: enums.NotificationNetworksColumns.Network.ToString(), Desc: false, Offset: currentCursor.Network}, - {Column: enums.NotificationNetworksColumns.EventType.ToString(), Desc: false, Offset: currentCursor.EventType}, + {Column: enums.NotificationNetworksColumns.Timestamp.ToExpr(), Desc: true, Offset: currentCursor.Epoch}, + {Column: enums.NotificationNetworksColumns.Network.ToExpr(), Desc: false, Offset: currentCursor.Network}, + {Column: enums.NotificationNetworksColumns.EventType.ToExpr(), Desc: false, Offset: currentCursor.EventType}, } - order, directions := applySortAndPagination(defaultColumns, t.SortColumn{Column: colSort.Column.ToString(), Desc: colSort.Desc}, currentCursor.GenericCursor) + order, directions := applySortAndPagination(defaultColumns, t.SortColumn{Column: colSort.Column.ToExpr(), Desc: colSort.Desc}, currentCursor.GenericCursor) ds = ds.Order(order...) if directions != nil { ds = ds.Where(directions) diff --git a/backend/pkg/api/data_access/vdb_blocks.go b/backend/pkg/api/data_access/vdb_blocks.go index f45ec2ac1..cb9e6ef86 100644 --- a/backend/pkg/api/data_access/vdb_blocks.go +++ b/backend/pkg/api/data_access/vdb_blocks.go @@ -33,9 +33,6 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das if err != nil { return nil, nil, err } - validators := goqu.T("users_val_dashboards_validators").As("validators") - blocks := goqu.T("blocks") - groups := goqu.T("goups") // TODO @LuccaBitfly move validation to handler? if cursor != "" { @@ -48,27 +45,34 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das searchGroup := regexp.MustCompile(`^[a-zA-Z0-9_\-.\ ]+$`).MatchString(search) searchIndex := regexp.MustCompile(`^[0-9]+$`).MatchString(search) - // ------------------------------------- - // Goqu Query: Determine validators filtered by search + validators := goqu.T("users_val_dashboards_validators").As("validators") + blocks := goqu.T("blocks") + groups := goqu.T("groups") + type validatorGroup struct { Validator t.VDBValidator `db:"validator_index"` Group uint64 `db:"group_id"` } + + // ------------------------------------- + // Goqu Query to determine validators filtered by search + var filteredValidatorsDs *goqu.SelectDataset var filteredValidators []validatorGroup - validatorsDs := goqu.Dialect("postgres"). + + filteredValidatorsDs = goqu.Dialect("postgres"). Select( "validator_index", ) if dashboardId.Validators == nil { - validatorsDs = validatorsDs. + filteredValidatorsDs = filteredValidatorsDs. From(validators). Where(validators.Col("dashboard_id").Eq(dashboardId.Id)) // apply search filters if searchIndex { - validatorsDs = validatorsDs.Where(validators.Col("validator_index").Eq(search)) + filteredValidatorsDs = filteredValidatorsDs.Where(validators.Col("validator_index").Eq(search)) } if searchGroup { - validatorsDs = validatorsDs. + filteredValidatorsDs = filteredValidatorsDs. InnerJoin(goqu.T("users_val_dashboards_groups").As(groups), goqu.On( validators.Col("group_id").Eq(groups.Col("id")), validators.Col("dashboard_id").Eq(groups.Col("dashboard_id")), @@ -84,7 +88,7 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das return make([]t.VDBBlocksTableRow, 0), &t.Paging{}, nil } - validatorsDs = validatorsDs. + filteredValidatorsDs = filteredValidatorsDs. Where(validators.Col("validator_index").Eq(index)) } } else { @@ -101,86 +105,18 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das break } } - validatorsDs = validatorsDs. + filteredValidatorsDs = filteredValidatorsDs. From( goqu.L("unnest(?)", pq.Array(filteredValidators)).As("validator_index"), ).As("validators") // TODO ? } - if dashboardId.Validators == nil { - validatorsQuery, validatorsArgs, err := validatorsDs.Prepared(true).ToSQL() - if err != nil { - return nil, nil, err - } - if err = d.alloyReader.SelectContext(ctx, &filteredValidators, validatorsQuery, validatorsArgs...); err != nil { - return nil, nil, err - } - } - if len(filteredValidators) == 0 { - return make([]t.VDBBlocksTableRow, 0), &t.Paging{}, nil - } - // ------------------------------------- - // Gather scheduled blocks - // found in dutiesInfo; pass results to final query later and let db do the sorting etc - validatorSet := make(map[t.VDBValidator]bool) - for _, v := range filteredValidators { - validatorSet[v.Validator] = true - } - var scheduledProposers []t.VDBValidator - var scheduledEpochs []uint64 - var scheduledSlots []uint64 - // don't need if requested slots are in the past - latestSlot := cache.LatestSlot.Get() - onlyPrimarySort := colSort.Column == enums.VDBBlockSlot || colSort.Column == enums.VDBBlockBlock - if !onlyPrimarySort || !currentCursor.IsValid() || - currentCursor.Slot > latestSlot+1 && currentCursor.Reverse != colSort.Desc || - currentCursor.Slot < latestSlot+1 && currentCursor.Reverse == colSort.Desc { - dutiesInfo, err := d.services.GetCurrentDutiesInfo() - if err == nil { - for slot, vali := range dutiesInfo.PropAssignmentsForSlot { - // only gather scheduled slots - if _, ok := dutiesInfo.SlotStatus[slot]; ok { - continue - } - // only gather slots scheduled for our validators - if _, ok := validatorSet[vali]; !ok { - continue - } - scheduledProposers = append(scheduledProposers, dutiesInfo.PropAssignmentsForSlot[slot]) - scheduledEpochs = append(scheduledEpochs, slot/utils.Config.Chain.ClConfig.SlotsPerEpoch) - scheduledSlots = append(scheduledSlots, slot) - } - } else { - log.Debugf("duties info not available, skipping scheduled slots: %s", err) - } - } - - // Sorting and pagination if cursor is present - defaultColumns := []t.SortColumn{ - {Column: enums.VDBBlocksColumns.Slot.ToString(), Table: blocks.GetTable(), Desc: true, Offset: currentCursor.Slot}, - } - var offset any - var table string - switch colSort.Column { - case enums.VDBBlocksColumns.Proposer: - offset = currentCursor.Proposer - case enums.VDBBlocksColumns.Block: - offset = currentCursor.Block - table = blocks.GetTable() - case enums.VDBBlocksColumns.Status: - offset = fmt.Sprintf("%d", currentCursor.Status) // type of 'status' column is text for some reason - case enums.VDBBlocksColumns.ProposerReward: - offset = currentCursor.Reward - } + // Constuct final query + var blocksDs *goqu.SelectDataset - order, directions := applySortAndPagination(defaultColumns, t.SortColumn{Column: colSort.Column.ToString(), Table: table, Desc: colSort.Desc, Offset: offset}, currentCursor.GenericCursor) - validatorsDs = validatorsDs.Order(order...) - if directions != nil { - validatorsDs = validatorsDs.Where(directions) - } - - validatorsDs = validatorsDs. + // 1. Tables + blocksDs = filteredValidatorsDs. InnerJoin(blocks, goqu.On( blocks.Col("proposer").Eq(validators.Col("validator_index")), )). @@ -205,7 +141,10 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das goqu.On( goqu.I("rb.exec_block_hash").Eq(blocks.Col("exec_block_hash")), ), - ). + ) + + // 2. Selects + blocksDs = blocksDs. SelectAppend( blocks.Col("epoch"), blocks.Col("slot"), @@ -215,64 +154,128 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das goqu.COALESCE(goqu.I("rb.proposer_fee_recipient"), blocks.Col("exec_fee_recipient")).As("fee_recipient"), goqu.COALESCE(goqu.L("rb.value / 1e18"), goqu.I("ep.fee_recipient_reward")).As("el_reward"), goqu.L("cp.cl_attestations_reward / 1e9 + cp.cl_sync_aggregate_reward / 1e9 + cp.cl_slashing_inclusion_reward / 1e9").As("cl_reward"), - ). - Limit(uint(limit + 1)) + ) - // Group id groupId := validators.Col("group_id") if dashboardId.Validators != nil { groupId = goqu.V(t.DefaultGroupId).As("group_id").GetAs() } - validatorsDs = validatorsDs.SelectAppend(groupId) + blocksDs = blocksDs.SelectAppend(groupId) - /* - if dashboardId.Validators == nil { - validatorsDs = validatorsDs.Select( - validators.Col("group_id"), - ) - } else { - validatorsDs = validatorsDs.Select( - goqu.L("?", t.DefaultGroupId).As("group_id"), - ) - }*/ + // 3. Sorting and pagination + defaultColumns := []t.SortColumn{ + {Column: enums.VDBBlocksColumns.Slot.ToExpr(), Desc: true, Offset: currentCursor.Slot}, + } + var offset any + switch colSort.Column { + case enums.VDBBlocksColumns.Proposer: + offset = currentCursor.Proposer + case enums.VDBBlocksColumns.Block: + offset = currentCursor.Block + if !currentCursor.Block.Valid { + offset = nil + } + case enums.VDBBlocksColumns.Status: + offset = fmt.Sprintf("%d", currentCursor.Status) // type of 'status' column is text for some reason + case enums.VDBBlocksColumns.ProposerReward: + offset = currentCursor.Reward + } - // union scheduled blocks if present - // WIP + order, directions := applySortAndPagination(defaultColumns, t.SortColumn{Column: colSort.Column.ToExpr(), Desc: colSort.Desc, Offset: offset}, currentCursor.GenericCursor) + blocksDs = goqu.From(blocksDs). // encapsulate so we can use selected fields + Order(order...) + if directions != nil { + blocksDs = blocksDs.Where(directions) + } - finalDs := validatorsDs - if len(scheduledProposers) > 0 { - scheduledDs := goqu.Dialect("postgres"). - From( - goqu.L("unnest(?, ?, ?) AS prov(validator_index, epoch, slot)", pq.Array(scheduledProposers), pq.Array(scheduledEpochs), pq.Array(scheduledSlots)), - ). - Select( - goqu.C("validator_index"), - goqu.C("epoch"), - goqu.C("slot"), - goqu.V("0").As("status"), - goqu.V(nil).As("exec_block_number"), - goqu.V(nil).As("fee_recipient"), - goqu.V(nil).As("el_reward"), - goqu.V(nil).As("cl_reward"), - goqu.V(nil).As("graffiti_text"), - ). - As("scheduled_blocks") + // 4. Limit + blocksDs = blocksDs.Limit(uint(limit + 1)) + + // 5. Gather and supply scheduled blocks to let db do the sorting etc + latestSlot := cache.LatestSlot.Get() + onlyPrimarySort := colSort.Column == enums.VDBBlockSlot + if !(onlyPrimarySort || colSort.Column == enums.VDBBlockBlock) || !currentCursor.IsValid() || + currentCursor.Slot > latestSlot+1 && currentCursor.Reverse != colSort.Desc || + currentCursor.Slot < latestSlot+1 && currentCursor.Reverse == colSort.Desc { + dutiesInfo, err := d.services.GetCurrentDutiesInfo() + if err == nil { + if dashboardId.Validators == nil { + // fetch filtered validators if not done yet + validatorsQuery, validatorsArgs, err := filteredValidatorsDs.Prepared(true).ToSQL() + if err != nil { + return nil, nil, err + } + if err = d.alloyReader.SelectContext(ctx, &filteredValidators, validatorsQuery, validatorsArgs...); err != nil { + return nil, nil, err + } + } + if len(filteredValidators) == 0 { + return make([]t.VDBBlocksTableRow, 0), &t.Paging{}, nil + } + + validatorSet := make(map[t.VDBValidator]bool) + for _, v := range filteredValidators { + validatorSet[v.Validator] = true + } + var scheduledProposers []t.VDBValidator + var scheduledEpochs []uint64 + var scheduledSlots []uint64 + // don't need if requested slots are in the past + for slot, vali := range dutiesInfo.PropAssignmentsForSlot { + // only gather scheduled slots + if _, ok := dutiesInfo.SlotStatus[slot]; ok { + continue + } + // only gather slots scheduled for our validators + if _, ok := validatorSet[vali]; !ok { + continue + } + scheduledProposers = append(scheduledProposers, dutiesInfo.PropAssignmentsForSlot[slot]) + scheduledEpochs = append(scheduledEpochs, slot/utils.Config.Chain.ClConfig.SlotsPerEpoch) + scheduledSlots = append(scheduledSlots, slot) + } - // distinct + block number ordering to filter out duplicates in an edge case (if dutiesInfo didn't update yet after a block was proposed, but the blocks table was) - // might be possible to remove this once the TODO in service_slot_viz.go:startSlotVizDataService is resolved - finalDs = validatorsDs. - Union(scheduledDs). - Where(directions). - Order(order...). - OrderAppend(goqu.C("exec_block_number").Desc().NullsLast()). - Limit(uint(limit + 1)). - Distinct(blocks.Col("slot")) - if !onlyPrimarySort { - finalDs = finalDs. - Distinct(blocks.Col("slot"), blocks.Col("exec_block_number")) + scheduledDs := goqu.Dialect("postgres"). + From( + goqu.L("unnest(?::int[], ?::int[], ?::int[]) AS prov(validator_index, epoch, slot)", pq.Array(scheduledProposers), pq.Array(scheduledEpochs), pq.Array(scheduledSlots)), + ). + Select( + goqu.C("validator_index"), + goqu.C("epoch"), + goqu.C("slot"), + goqu.V("0").As("status"), + goqu.V(nil).As("exec_block_number"), + goqu.V(nil).As("fee_recipient"), + goqu.V(nil).As("el_reward"), + goqu.V(nil).As("cl_reward"), + goqu.V(nil).As("graffiti_text"), + goqu.V(t.DefaultGroupId).As("group_id"), + ). + As("scheduled_blocks") + + // Supply to result query + // distinct + block number ordering to filter out duplicates in an edge case (if dutiesInfo didn't update yet after a block was proposed, but the blocks table was) + // might be possible to remove this once the TODO in service_slot_viz.go:startSlotVizDataService is resolved + blocksDs = goqu.Dialect("Postgres"). + From(blocksDs.Union(scheduledDs)). // wrap union to apply order + Order(order...). + OrderAppend(goqu.C("exec_block_number").Desc().NullsLast()). + Limit(uint(limit + 1)). + Distinct(enums.VDBBlocksColumns.Slot.ToExpr()) + if directions != nil { + blocksDs = blocksDs.Where(directions) + } + if !onlyPrimarySort { + blocksDs = blocksDs. + Distinct(colSort.Column.ToExpr(), enums.VDBBlocksColumns.Slot.ToExpr()) + } + } else { + log.Warnf("Error getting scheduled proposals, DutiesInfo not available in Redis: %s", err) } } + // ------------------------------------- + // Execute query var proposals []struct { Proposer t.VDBValidator `db:"validator_index"` Group uint64 `db:"group_id"` @@ -283,13 +286,13 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das FeeRecipient []byte `db:"fee_recipient"` ElReward decimal.NullDecimal `db:"el_reward"` ClReward decimal.NullDecimal `db:"cl_reward"` - GraffitiText string `db:"graffiti_text"` + GraffitiText sql.NullString `db:"graffiti_text"` // for cursor only Reward decimal.Decimal } startTime := time.Now() - query, args, err := finalDs.Prepared(true).ToSQL() + query, args, err := blocksDs.Prepared(true).ToSQL() if err != nil { return nil, nil, err } @@ -301,6 +304,9 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das if len(proposals) == 0 { return make([]t.VDBBlocksTableRow, 0), &t.Paging{}, nil } + + // ------------------------------------- + // Prepare result moreDataFlag := len(proposals) > int(limit) if moreDataFlag { proposals = proposals[:len(proposals)-1] @@ -335,10 +341,14 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das if proposal.Status == 0 || proposal.Status == 2 { continue } - graffiti := proposal.GraffitiText - data[i].Graffiti = &graffiti - block := uint64(proposal.Block.Int64) - data[i].Block = &block + if proposal.GraffitiText.Valid { + graffiti := proposal.GraffitiText.String + data[i].Graffiti = &graffiti + } + if proposal.Block.Valid { + block := uint64(proposal.Block.Int64) + data[i].Block = &block + } if proposal.Status == 3 { continue } diff --git a/backend/pkg/api/enums/notifications_enums.go b/backend/pkg/api/enums/notifications_enums.go index 1fb78529e..9af65cb1a 100644 --- a/backend/pkg/api/enums/notifications_enums.go +++ b/backend/pkg/api/enums/notifications_enums.go @@ -1,5 +1,7 @@ package enums +import "github.com/doug-martin/goqu/v9" + // ------------------------------------------------------------ // Notifications Dashboard Table Columns @@ -34,22 +36,22 @@ func (NotificationDashboardsColumn) NewFromString(s string) NotificationDashboar } // internal use, used to map to query column names -func (c NotificationDashboardsColumn) ToString() string { +func (c NotificationDashboardsColumn) ToExpr() OrderableSortable { switch c { case NotificationDashboardChainId: - return "chain_id" + return goqu.C("chain_id") case NotificationDashboardEpoch: - return "epoch" + return goqu.C("epoch") case NotificationDashboardDashboardName: - return "dashboard_name" + return goqu.C("dashboard_name") case NotificationDashboardDashboardId: - return "dashboard_id" + return goqu.C("dashboard_id") case NotificationDashboardGroupName: - return "group_name" + return goqu.C("group_name") case NotificationDashboardGroupId: - return "group_id" + return goqu.C("group_id") default: - return "" + return nil } } @@ -104,20 +106,20 @@ func (NotificationMachinesColumn) NewFromString(s string) NotificationMachinesCo } // internal use, used to map to query column names -func (c NotificationMachinesColumn) ToString() string { +func (c NotificationMachinesColumn) ToExpr() OrderableSortable { switch c { case NotificationMachineId: - return "machine_id" + return goqu.C("machine_id") case NotificationMachineName: - return "machine_name" + return goqu.C("machine_name") case NotificationMachineThreshold: - return "threshold" + return goqu.C("threshold") case NotificationMachineEventType: - return "event_type" + return goqu.C("event_type") case NotificationMachineTimestamp: - return "epoch" + return goqu.C("epoch") default: - return "" + return nil } } @@ -163,14 +165,14 @@ func (NotificationClientsColumn) NewFromString(s string) NotificationClientsColu } // internal use, used to map to query column names -func (c NotificationClientsColumn) ToString() string { +func (c NotificationClientsColumn) ToExpr() OrderableSortable { switch c { case NotificationClientName: - return "client_name" + return goqu.C("client_name") case NotificationClientTimestamp: - return "epoch" + return goqu.C("epoch") default: - return "" + return nil } } @@ -251,16 +253,16 @@ func (NotificationNetworksColumn) NewFromString(s string) NotificationNetworksCo } // internal use, used to map to query column names -func (c NotificationNetworksColumn) ToString() string { +func (c NotificationNetworksColumn) ToExpr() OrderableSortable { switch c { case NotificationNetworkTimestamp: - return "epoch" + return goqu.C("epoch") case NotificationNetworkNetwork: - return "network" + return goqu.C("network") case NotificationNetworkEventType: - return "event_type" + return goqu.C("event_type") default: - return "" + return nil } } diff --git a/backend/pkg/api/enums/validator_dashboard_enums.go b/backend/pkg/api/enums/validator_dashboard_enums.go index 228b224dc..c241ef98e 100644 --- a/backend/pkg/api/enums/validator_dashboard_enums.go +++ b/backend/pkg/api/enums/validator_dashboard_enums.go @@ -1,5 +1,10 @@ package enums +import ( + "github.com/doug-martin/goqu/v9" + "github.com/doug-martin/goqu/v9/exp" +) + // ---------------- // Validator Dashboard Summary Table @@ -156,20 +161,26 @@ func (VDBBlocksColumn) NewFromString(s string) VDBBlocksColumn { } } -func (c VDBBlocksColumn) ToString() string { +type OrderableSortable interface { + exp.Orderable + exp.Comparable + exp.Isable +} + +func (c VDBBlocksColumn) ToExpr() OrderableSortable { switch c { case VDBBlockProposer: - return "proposer" + return goqu.C("validator_index") case VDBBlockSlot: - return "slot" + return goqu.C("slot") case VDBBlockBlock: - return "exec_block_number" + return goqu.C("exec_block_number") case VDBBlockStatus: - return "status" + return goqu.C("status") case VDBBlockProposerReward: - return "reward" + return goqu.L("el_reward + cl_reward") default: - return "" + return nil } } diff --git a/backend/pkg/api/types/data_access.go b/backend/pkg/api/types/data_access.go index 70961c5f2..6bcf30472 100644 --- a/backend/pkg/api/types/data_access.go +++ b/backend/pkg/api/types/data_access.go @@ -4,8 +4,6 @@ import ( "database/sql" "time" - "github.com/doug-martin/goqu/v9" - "github.com/doug-martin/goqu/v9/exp" "github.com/gobitfly/beaconchain/pkg/api/enums" "github.com/gobitfly/beaconchain/pkg/consapi/types" "github.com/gobitfly/beaconchain/pkg/monitoring/constants" @@ -26,18 +24,10 @@ type Sort[T enums.Enum] struct { } type SortColumn struct { - Column string - Table string // optional + // defaults + Column enums.OrderableSortable Desc bool - // represents value from cursor - Offset any -} - -func (s SortColumn) Expr() exp.IdentifierExpression { - if s.Table != "" { - return goqu.T(s.Table).Col(s.Column) - } - return goqu.C(s.Column) + Offset any // nil to indicate null value } type VDBIdPrimary int From 3d4b61409466bfe9428f4b3299a373dc74ef9913 Mon Sep 17 00:00:00 2001 From: remoterami <142154971+remoterami@users.noreply.github.com> Date: Mon, 21 Oct 2024 14:11:46 +0200 Subject: [PATCH 5/7] goqu quirck workaround --- backend/pkg/api/data_access/vdb_blocks.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/pkg/api/data_access/vdb_blocks.go b/backend/pkg/api/data_access/vdb_blocks.go index cb9e6ef86..00495e265 100644 --- a/backend/pkg/api/data_access/vdb_blocks.go +++ b/backend/pkg/api/data_access/vdb_blocks.go @@ -162,7 +162,10 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das } blocksDs = blocksDs.SelectAppend(groupId) - // 3. Sorting and pagination + // 3. Limit + blocksDs = blocksDs.Limit(uint(limit + 1)) + + // 4. Sorting and pagination defaultColumns := []t.SortColumn{ {Column: enums.VDBBlocksColumns.Slot.ToExpr(), Desc: true, Offset: currentCursor.Slot}, } @@ -188,9 +191,6 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das blocksDs = blocksDs.Where(directions) } - // 4. Limit - blocksDs = blocksDs.Limit(uint(limit + 1)) - // 5. Gather and supply scheduled blocks to let db do the sorting etc latestSlot := cache.LatestSlot.Get() onlyPrimarySort := colSort.Column == enums.VDBBlockSlot From 4340a77bf2645063e7ef22063ff3f4d21e284d74 Mon Sep 17 00:00:00 2001 From: remoterami <142154971+remoterami@users.noreply.github.com> Date: Thu, 24 Oct 2024 13:56:31 +0200 Subject: [PATCH 6/7] applied CR feedback --- backend/pkg/api/data_access/general.go | 12 ++-- backend/pkg/api/data_access/vdb_blocks.go | 80 ++++++++++++++--------- 2 files changed, 54 insertions(+), 38 deletions(-) diff --git a/backend/pkg/api/data_access/general.go b/backend/pkg/api/data_access/general.go index d629d52df..9d7ee0d97 100644 --- a/backend/pkg/api/data_access/general.go +++ b/backend/pkg/api/data_access/general.go @@ -74,9 +74,9 @@ func applySortAndPagination(defaultColumns []types.SortColumn, primary types.Sor if cursor.IsReverse() { column.Desc = !column.Desc } - colOrder := column.Column.Asc() + colOrder := column.Column.Asc().NullsFirst() if column.Desc { - colOrder = column.Column.Desc() + colOrder = column.Column.Desc().NullsLast() } queryOrder = append(queryOrder, colOrder) } @@ -89,10 +89,10 @@ func applySortAndPagination(defaultColumns []types.SortColumn, primary types.Sor column := queryOrderColumns[i] var colWhere exp.Expression - // current convention is the psql default (ASC: nulls last, DESC: nulls first) - colWhere = goqu.Or(column.Column.Gt(column.Offset), column.Column.IsNull()) - if column.Desc { - colWhere = column.Column.Lt(column.Offset) + // current convention is opposite of the psql default (ASC: nulls first, DESC: nulls last) + colWhere = goqu.Or(column.Column.Lt(column.Offset), column.Column.IsNull()) + if !column.Desc { + colWhere = column.Column.Gt(column.Offset) if column.Offset == nil { colWhere = goqu.Or(colWhere, column.Column.IsNull()) } diff --git a/backend/pkg/api/data_access/vdb_blocks.go b/backend/pkg/api/data_access/vdb_blocks.go index 00495e265..b8c1ba352 100644 --- a/backend/pkg/api/data_access/vdb_blocks.go +++ b/backend/pkg/api/data_access/vdb_blocks.go @@ -10,6 +10,7 @@ import ( "time" "github.com/doug-martin/goqu/v9" + "github.com/doug-martin/goqu/v9/exp" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/gobitfly/beaconchain/pkg/api/enums" t "github.com/gobitfly/beaconchain/pkg/api/types" @@ -45,7 +46,7 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das searchGroup := regexp.MustCompile(`^[a-zA-Z0-9_\-.\ ]+$`).MatchString(search) searchIndex := regexp.MustCompile(`^[0-9]+$`).MatchString(search) - validators := goqu.T("users_val_dashboards_validators").As("validators") + validators := goqu.T("validators") blocks := goqu.T("blocks") groups := goqu.T("groups") @@ -65,21 +66,22 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das ) if dashboardId.Validators == nil { filteredValidatorsDs = filteredValidatorsDs. - From(validators). + From(goqu.T("users_val_dashboards_validators").As(validators.GetTable())). Where(validators.Col("dashboard_id").Eq(dashboardId.Id)) // apply search filters + searches := []exp.Expression{} if searchIndex { - filteredValidatorsDs = filteredValidatorsDs.Where(validators.Col("validator_index").Eq(search)) + searches = append(searches, validators.Col("validator_index").Eq(search)) } if searchGroup { filteredValidatorsDs = filteredValidatorsDs. InnerJoin(goqu.T("users_val_dashboards_groups").As(groups), goqu.On( validators.Col("group_id").Eq(groups.Col("id")), validators.Col("dashboard_id").Eq(groups.Col("dashboard_id")), - )). - Where( - goqu.L("LOWER(?)", groups.Col("name")).Like(strings.Replace(search, "_", "\\_", -1) + "%"), - ) + )) + searches = append(searches, + goqu.L("LOWER(?)", groups.Col("name")).Like(strings.Replace(strings.ToLower(search), "_", "\\_", -1)+"%"), + ) } if searchPubkey { index, ok := validatorMapping.ValidatorIndices[search] @@ -87,11 +89,15 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das // searched pubkey doesn't exist, don't even need to query anything return make([]t.VDBBlocksTableRow, 0), &t.Paging{}, nil } - - filteredValidatorsDs = filteredValidatorsDs. - Where(validators.Col("validator_index").Eq(index)) + searches = append(searches, + validators.Col("validator_index").Eq(index), + ) + } + if len(searches) > 0 { + filteredValidatorsDs = filteredValidatorsDs.Where(goqu.Or(searches...)) } } else { + validatorList := make([]t.VDBValidator, 0, len(dashboardId.Validators)) for _, validator := range dashboardId.Validators { if searchIndex && fmt.Sprint(validator) != search || searchPubkey && validator != validatorMapping.ValidatorIndices[search] { @@ -101,14 +107,19 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das Validator: validator, Group: t.DefaultGroupId, }) + validatorList = append(validatorList, validator) if searchIndex || searchPubkey { break } } filteredValidatorsDs = filteredValidatorsDs. From( - goqu.L("unnest(?)", pq.Array(filteredValidators)).As("validator_index"), - ).As("validators") // TODO ? + goqu.Dialect("postgres"). + From( + goqu.L("unnest(?::int[])", pq.Array(validatorList)).As("validator_index"), + ). + As(validators.GetTable()), + ) } // ------------------------------------- @@ -144,10 +155,17 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das ) // 2. Selects + groupIdQ := goqu.C("group_id").(exp.Aliaseable) + if dashboardId.Validators != nil { + groupIdQ = exp.NewLiteralExpression("?::int", t.DefaultGroupId) + } + groupId := groupIdQ.As("group_id") + blocksDs = blocksDs. SelectAppend( blocks.Col("epoch"), blocks.Col("slot"), + groupId, blocks.Col("status"), blocks.Col("exec_block_number"), blocks.Col("graffiti_text"), @@ -156,16 +174,7 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das goqu.L("cp.cl_attestations_reward / 1e9 + cp.cl_sync_aggregate_reward / 1e9 + cp.cl_slashing_inclusion_reward / 1e9").As("cl_reward"), ) - groupId := validators.Col("group_id") - if dashboardId.Validators != nil { - groupId = goqu.V(t.DefaultGroupId).As("group_id").GetAs() - } - blocksDs = blocksDs.SelectAppend(groupId) - - // 3. Limit - blocksDs = blocksDs.Limit(uint(limit + 1)) - - // 4. Sorting and pagination + // 3. Sorting and pagination defaultColumns := []t.SortColumn{ {Column: enums.VDBBlocksColumns.Slot.ToExpr(), Desc: true, Offset: currentCursor.Slot}, } @@ -185,18 +194,23 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das } order, directions := applySortAndPagination(defaultColumns, t.SortColumn{Column: colSort.Column.ToExpr(), Desc: colSort.Desc, Offset: offset}, currentCursor.GenericCursor) - blocksDs = goqu.From(blocksDs). // encapsulate so we can use selected fields - Order(order...) + blocksDs = goqu.Dialect("postgres").From(goqu.T("past_blocks_cte")). + With("past_blocks_cte", blocksDs). // encapsulate so we can use selected fields + Order(order...) if directions != nil { blocksDs = blocksDs.Where(directions) } + // 4. Limit + blocksDs = blocksDs.Limit(uint(limit + 1)) + // 5. Gather and supply scheduled blocks to let db do the sorting etc latestSlot := cache.LatestSlot.Get() onlyPrimarySort := colSort.Column == enums.VDBBlockSlot - if !(onlyPrimarySort || colSort.Column == enums.VDBBlockBlock) || !currentCursor.IsValid() || - currentCursor.Slot > latestSlot+1 && currentCursor.Reverse != colSort.Desc || - currentCursor.Slot < latestSlot+1 && currentCursor.Reverse == colSort.Desc { + if !(onlyPrimarySort || colSort.Column == enums.VDBBlockBlock) || + !currentCursor.IsValid() || + currentCursor.Slot > latestSlot+1 || + colSort.Desc == currentCursor.Reverse { dutiesInfo, err := d.services.GetCurrentDutiesInfo() if err == nil { if dashboardId.Validators == nil { @@ -213,11 +227,12 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das return make([]t.VDBBlocksTableRow, 0), &t.Paging{}, nil } - validatorSet := make(map[t.VDBValidator]bool) + validatorSet := make(map[t.VDBValidator]uint64) for _, v := range filteredValidators { - validatorSet[v.Validator] = true + validatorSet[v.Validator] = v.Group } var scheduledProposers []t.VDBValidator + var scheduledGroups []uint64 var scheduledEpochs []uint64 var scheduledSlots []uint64 // don't need if requested slots are in the past @@ -231,25 +246,26 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das continue } scheduledProposers = append(scheduledProposers, dutiesInfo.PropAssignmentsForSlot[slot]) + scheduledGroups = append(scheduledGroups, validatorSet[vali]) scheduledEpochs = append(scheduledEpochs, slot/utils.Config.Chain.ClConfig.SlotsPerEpoch) scheduledSlots = append(scheduledSlots, slot) } scheduledDs := goqu.Dialect("postgres"). From( - goqu.L("unnest(?::int[], ?::int[], ?::int[]) AS prov(validator_index, epoch, slot)", pq.Array(scheduledProposers), pq.Array(scheduledEpochs), pq.Array(scheduledSlots)), + goqu.L("unnest(?::int[], ?::int[], ?::int[], ?::int[]) AS prov(validator_index, group_id, epoch, slot)", pq.Array(scheduledProposers), pq.Array(scheduledGroups), pq.Array(scheduledEpochs), pq.Array(scheduledSlots)), ). Select( goqu.C("validator_index"), goqu.C("epoch"), goqu.C("slot"), + groupId, goqu.V("0").As("status"), goqu.V(nil).As("exec_block_number"), + goqu.V(nil).As("graffiti_text"), goqu.V(nil).As("fee_recipient"), goqu.V(nil).As("el_reward"), goqu.V(nil).As("cl_reward"), - goqu.V(nil).As("graffiti_text"), - goqu.V(t.DefaultGroupId).As("group_id"), ). As("scheduled_blocks") From fcdea3682269fbbd98b4e0a2849695438c1478e0 Mon Sep 17 00:00:00 2001 From: remoterami <142154971+remoterami@users.noreply.github.com> Date: Mon, 28 Oct 2024 10:51:47 +0100 Subject: [PATCH 7/7] CR feedback --- backend/pkg/api/data_access/vdb_blocks.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/pkg/api/data_access/vdb_blocks.go b/backend/pkg/api/data_access/vdb_blocks.go index b8c1ba352..47bbd651f 100644 --- a/backend/pkg/api/data_access/vdb_blocks.go +++ b/backend/pkg/api/data_access/vdb_blocks.go @@ -46,7 +46,7 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das searchGroup := regexp.MustCompile(`^[a-zA-Z0-9_\-.\ ]+$`).MatchString(search) searchIndex := regexp.MustCompile(`^[0-9]+$`).MatchString(search) - validators := goqu.T("validators") + validators := goqu.T("validators") // could adapt data type to make handling as table/alias less confusing blocks := goqu.T("blocks") groups := goqu.T("groups") @@ -215,6 +215,8 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das if err == nil { if dashboardId.Validators == nil { // fetch filtered validators if not done yet + filteredValidatorsDs = filteredValidatorsDs. + SelectAppend(groupIdQ) validatorsQuery, validatorsArgs, err := filteredValidatorsDs.Prepared(true).ToSQL() if err != nil { return nil, nil, err