From defa284471250077b4af6a6aa3a0e7b617b0217f Mon Sep 17 00:00:00 2001 From: invis-bitfly <162128378+invis-bitfly@users.noreply.github.com> Date: Wed, 8 Jan 2025 18:10:11 +0100 Subject: [PATCH 1/9] fix(services): report as running before updating avg eff data --- backend/pkg/api/services/service_average_network_efficiency.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/pkg/api/services/service_average_network_efficiency.go b/backend/pkg/api/services/service_average_network_efficiency.go index 083b06dc2..88c20ae7f 100644 --- a/backend/pkg/api/services/service_average_network_efficiency.go +++ b/backend/pkg/api/services/service_average_network_efficiency.go @@ -28,8 +28,8 @@ func (s *Services) startEfficiencyDataService(wg *sync.WaitGroup) { startTime := time.Now() delay := time.Duration(utils.Config.Chain.ClConfig.SlotsPerEpoch*utils.Config.Chain.ClConfig.SecondsPerSlot) * time.Second r := services.NewStatusReport("api_service_avg_efficiency", constants.Default, delay) - err := s.updateEfficiencyData() // TODO: only update data if something has changed (new head epoch) r(constants.Running, nil) + err := s.updateEfficiencyData() // TODO: only update data if something has changed (new head epoch) if err != nil { log.Error(err, "error updating average network efficiency data", 0) r(constants.Failure, map[string]string{"error": err.Error()}) From eb94303bc8b160268b450d298b29dea2093d0af8 Mon Sep 17 00:00:00 2001 From: invis-bitfly <162128378+invis-bitfly@users.noreply.github.com> Date: Thu, 9 Jan 2025 12:38:23 +0100 Subject: [PATCH 2/9] feat(exporter): add finalized rewards table --- backend/cmd/exporter/main.go | 1 + .../modules/execution_rewards_finalizer.go | 139 ++++++++++++++++++ 2 files changed, 140 insertions(+) create mode 100644 backend/pkg/exporter/modules/execution_rewards_finalizer.go diff --git a/backend/cmd/exporter/main.go b/backend/cmd/exporter/main.go index 6f3cf7c72..dbf0e5ba6 100644 --- a/backend/cmd/exporter/main.go +++ b/backend/cmd/exporter/main.go @@ -202,6 +202,7 @@ func Run() { modules.NewSlotExporter(context), modules.NewExecutionDepositsExporter(context), modules.NewExecutionPayloadsExporter(context), + modules.NewExecutionRewardFinalizer(context), ) } diff --git a/backend/pkg/exporter/modules/execution_rewards_finalizer.go b/backend/pkg/exporter/modules/execution_rewards_finalizer.go new file mode 100644 index 000000000..0765b8d33 --- /dev/null +++ b/backend/pkg/exporter/modules/execution_rewards_finalizer.go @@ -0,0 +1,139 @@ +package modules + +import ( + "fmt" + "sync" + "time" + + "github.com/doug-martin/goqu/v9" + "github.com/gobitfly/beaconchain/pkg/commons/db" + "github.com/gobitfly/beaconchain/pkg/commons/log" + constypes "github.com/gobitfly/beaconchain/pkg/consapi/types" +) + +type executionRewardsFinalizer struct { + ModuleContext + ExportMutex *sync.Mutex +} + +func NewExecutionRewardFinalizer(moduleContext ModuleContext) ModuleInterface { + return &executionRewardsFinalizer{ + ModuleContext: moduleContext, + ExportMutex: &sync.Mutex{}, + } +} + +func (d *executionRewardsFinalizer) Init() error { + log.Infof("hi") + return nil +} + +func (d *executionRewardsFinalizer) GetName() string { + return "ExecutionRewards-Finalizer" +} + +func (d *executionRewardsFinalizer) OnChainReorg(event *constypes.StandardEventChainReorg) (err error) { + return nil // nop +} + +func (d *executionRewardsFinalizer) OnFinalizedCheckpoint(event *constypes.StandardFinalizedCheckpointResponse) (err error) { + + return nil // nop +} + +func (d *executionRewardsFinalizer) OnHead(event *constypes.StandardEventHeadResponse) (err error) { + // if mutex is locked, return early + if !d.ExportMutex.TryLock() { + log.Infof("execution rewards finalizer is already running") + return nil + } + defer d.ExportMutex.Unlock() + err = d.maintainTable() + if err != nil { + return fmt.Errorf("error maintaining table: %w", err) + } + return nil +} + +func (d *executionRewardsFinalizer) maintainTable() (err error) { + var lastExportedSlot int64 + err = db.ReaderDb.Get(&lastExportedSlot, ` + SELECT + coalesce(MAX(slot), -1) + FROM + execution_rewards_finalized + `) + if err != nil { + return fmt.Errorf("error getting last exported slot: %w", err) + } + // get latest finalized slot + var latestFinalizedSlot int64 + err = db.ReaderDb.Get(&latestFinalizedSlot, ` + SELECT + max(slot) + FROM + blocks + WHERE + status = '1' AND finalized = true + `) + + if err != nil { + return fmt.Errorf("error getting finalized-slot: %w", err) + } + + // limit to prevent overloading + if latestFinalizedSlot-lastExportedSlot > 250_000 { + latestFinalizedSlot = lastExportedSlot + 250_000 + } + + if latestFinalizedSlot <= lastExportedSlot { + log.Debugf("no new finalized slots to export") + return nil + } + log.Infof("finalized rewards = last exported slot: %v, latest finalized slot: %v", lastExportedSlot, latestFinalizedSlot) + + start := time.Now() + ds := goqu.Dialect("postgres").Insert("execution_rewards_finalized").FromQuery( + goqu.From(goqu.T("blocks").As("b")). + LeftJoin( + goqu.T("execution_payloads").As("ep"), + goqu.On(goqu.I("ep.block_hash").Eq(goqu.I("b.exec_block_hash"))), + ). + LeftJoin( + goqu.T("relays_blocks").As("rb"), + goqu.On(goqu.I("rb.exec_block_hash").Eq(goqu.I("b.exec_block_hash"))), + ). + Select( + goqu.I("b.epoch").As("epoch"), + goqu.I("b.slot").As("slot"), + goqu.I("b.proposer").As("proposer"), + goqu.Func("sum", goqu.COALESCE(goqu.I("rb.value"), goqu.L("ep.fee_recipient_reward * '10e18'::numeric"), goqu.L("0::numeric"))).As("value"), + ). + Where( + goqu.I("b.slot").Gt(lastExportedSlot), + goqu.I("b.slot").Lte(latestFinalizedSlot), + goqu.I("b.status").Eq("1"), + ). + GroupBy( + goqu.I("b.epoch"), goqu.I("b.slot"), goqu.I("b.proposer"), + ), + ).OnConflict(goqu.DoUpdate("slot", goqu.Record{ + "value": goqu.I("excluded.value"), + "proposer": goqu.I("excluded.proposer"), + })) + + log.Debugf("writing execution rewards finalized data") + + query, args, err := ds.Prepared(true).ToSQL() + if err != nil { + return fmt.Errorf("error preparing query: %w", err) + } + _, err = db.WriterDb.Exec(query, args...) + + if err != nil { + return fmt.Errorf("error inserting data: %w", err) + } + log.Infof("execution rewards finalized data written in %v", time.Since(start)) + + return nil +} From 60de5952b646cfcc11196e45be003ccf3bb46e60 Mon Sep 17 00:00:00 2001 From: invis-bitfly <162128378+invis-bitfly@users.noreply.github.com> Date: Thu, 9 Jan 2025 12:38:51 +0100 Subject: [PATCH 3/9] refactor(exporter): remove unused materialized view updater --- .../modules/execution_payloads_exporter.go | 52 ++----------------- 1 file changed, 4 insertions(+), 48 deletions(-) diff --git a/backend/pkg/exporter/modules/execution_payloads_exporter.go b/backend/pkg/exporter/modules/execution_payloads_exporter.go index e7f80bef1..cd456c5b6 100644 --- a/backend/pkg/exporter/modules/execution_payloads_exporter.go +++ b/backend/pkg/exporter/modules/execution_payloads_exporter.go @@ -18,16 +18,14 @@ import ( ) type executionPayloadsExporter struct { - ModuleContext ModuleContext - ExportMutex *sync.Mutex - CachedViewMutex *sync.Mutex + ModuleContext ModuleContext + ExportMutex *sync.Mutex } func NewExecutionPayloadsExporter(moduleContext ModuleContext) ModuleInterface { return &executionPayloadsExporter{ - ModuleContext: moduleContext, - ExportMutex: &sync.Mutex{}, - CachedViewMutex: &sync.Mutex{}, + ModuleContext: moduleContext, + ExportMutex: &sync.Mutex{}, } } @@ -59,51 +57,9 @@ func (d *executionPayloadsExporter) OnChainReorg(event *constypes.StandardEventC // can take however long it wants to run, is run in a separate goroutine, so no need to worry about blocking func (d *executionPayloadsExporter) OnFinalizedCheckpoint(event *constypes.StandardFinalizedCheckpointResponse) (err error) { - // if mutex is locked, return early - if !d.CachedViewMutex.TryLock() { - log.Infof("execution payloads exporter is already running") - return nil - } - defer d.CachedViewMutex.Unlock() - - start := time.Now() - // update cached view - err = d.updateCachedView() - if err != nil { - return err - } - - log.Infof("updating execution payloads cached view took %v", time.Since(start)) return nil } -func (d *executionPayloadsExporter) updateCachedView() (err error) { - err = db.CacheQuery(` - SELECT DISTINCT ON (uvdv.dashboard_id, uvdv.group_id, b.slot) - uvdv.dashboard_id, - uvdv.group_id, - b.slot, - coalesce(cp.cl_attestations_reward / 1e9, 0) + coalesce(cp.cl_sync_aggregate_reward / 1e9, 0) + coalesce(cp.cl_slashing_inclusion_reward / 1e9, 0) + coalesce(rb.value / 1e18, ep.fee_recipient_reward) as reward, - coalesce(rb.proposer_fee_recipient, b.exec_fee_recipient) as fee_recipient, - rb.value IS NOT NULL AS is_mev - FROM - blocks b - INNER JOIN execution_payloads ep ON ep.block_hash = b.exec_block_hash - INNER JOIN consensus_payloads cp ON cp.slot = b.slot - INNER JOIN users_val_dashboards_validators uvdv ON b.proposer = uvdv.validator_index - LEFT JOIN relays_blocks rb ON rb.exec_block_hash = b.exec_block_hash - WHERE - b.status = '1' - AND b.exec_block_hash IS NOT NULL AND ep.fee_recipient_reward IS NOT NULL - ORDER BY - dashboard_id, - group_id, - slot DESC, - rb.value DESC; - `, "cached_proposal_rewards", []string{"dashboard_id", "slot"}, []string{"dashboard_id", "reward"}, []string{"dashboard_id"}) - return err -} - func (d *executionPayloadsExporter) maintainTable() (err error) { blocks := struct { MinBlock sql.NullInt64 `db:"min"` From 7e40dc8dd9c50f48a56e7cddea339905ef34662e Mon Sep 17 00:00:00 2001 From: invis-bitfly <162128378+invis-bitfly@users.noreply.github.com> Date: Thu, 9 Jan 2025 12:39:32 +0100 Subject: [PATCH 4/9] refactor(api): use new finalized rewards table --- backend/pkg/api/data_access/vdb_rewards.go | 65 ++++------------------ backend/pkg/api/data_access/vdb_summary.go | 31 ++--------- 2 files changed, 15 insertions(+), 81 deletions(-) diff --git a/backend/pkg/api/data_access/vdb_rewards.go b/backend/pkg/api/data_access/vdb_rewards.go index 4d4e4e266..8b253bf50 100644 --- a/backend/pkg/api/data_access/vdb_rewards.go +++ b/backend/pkg/api/data_access/vdb_rewards.go @@ -100,21 +100,11 @@ func (d *DataAccessService) GetValidatorDashboardRewards(ctx context.Context, da elDs := goqu.Dialect("postgres"). Select( goqu.L("b.epoch"), - goqu.L("SUM(COALESCE(rb.value, ep.fee_recipient_reward * 1e18, 0)) AS el_rewards")). + goqu.SUM(goqu.I("value")).As("el_rewards")). From(goqu.L("users_val_dashboards_validators v")). Where(goqu.L("b.epoch >= ?", startEpoch)). - LeftJoin(goqu.L("blocks b"), goqu.On(goqu.L("v.validator_index = b.proposer AND b.status = '1'"))). - LeftJoin(goqu.L("execution_payloads ep"), goqu.On(goqu.L("ep.block_hash = b.exec_block_hash"))). - LeftJoin( - goqu.Lateral(goqu.Dialect("postgres"). - From("relays_blocks"). - Select( - goqu.L("exec_block_hash"), - goqu.MAX("value").As("value")). - Where(goqu.L("relays_blocks.exec_block_hash = b.exec_block_hash")). - GroupBy("exec_block_hash")).As("rb"), - goqu.On(goqu.L("rb.exec_block_hash = b.exec_block_hash")), - ) + LeftJoin(goqu.I("execution_rewards_finalized").As('b'), goqu.On(goqu.L("v.validator_index = b.proposer"))). + GroupBy(goqu.L("b.epoch")) if dashboardId.Validators == nil { rewardsDs = rewardsDs. @@ -557,20 +547,9 @@ func (d *DataAccessService) GetValidatorDashboardGroupRewards(ctx context.Contex elDs := goqu.Dialect("postgres"). Select( - goqu.L("COALESCE(SUM(COALESCE(rb.value, ep.fee_recipient_reward * 1e18, 0)), 0) AS blocks_el_reward")). + goqu.SUM(goqu.I("value")).As("blocks_el_rewards")). From(goqu.L("users_val_dashboards_validators v")). - LeftJoin(goqu.L("blocks b"), goqu.On(goqu.L("v.validator_index = b.proposer AND b.status = '1'"))). - LeftJoin(goqu.L("execution_payloads ep"), goqu.On(goqu.L("ep.block_hash = b.exec_block_hash"))). - LeftJoin( - goqu.Lateral(goqu.Dialect("postgres"). - From("relays_blocks"). - Select( - goqu.L("exec_block_hash"), - goqu.MAX("value").As("value")). - Where(goqu.L("relays_blocks.exec_block_hash = b.exec_block_hash")). - GroupBy("exec_block_hash")).As("rb"), - goqu.On(goqu.L("rb.exec_block_hash = b.exec_block_hash")), - ). + LeftJoin(goqu.I("execution_rewards_finalized").As('b'), goqu.On(goqu.L("v.validator_index = b.proposer"))). Where(goqu.L("b.epoch = ?", epoch)) // handle the case when we have a list of validators @@ -733,21 +712,11 @@ func (d *DataAccessService) GetValidatorDashboardRewardsChart(ctx context.Contex elDs := goqu.Dialect("postgres"). Select( goqu.L("b.epoch"), - goqu.L("SUM(COALESCE(rb.value, ep.fee_recipient_reward * 1e18, 0)) AS el_rewards")). + goqu.SUM(goqu.I("value")).As("el_rewards")). From(goqu.L("users_val_dashboards_validators v")). - LeftJoin(goqu.L("blocks b"), goqu.On(goqu.L("v.validator_index = b.proposer AND b.status = '1'"))). - LeftJoin(goqu.L("execution_payloads ep"), goqu.On(goqu.L("ep.block_hash = b.exec_block_hash"))). - LeftJoin( - goqu.Lateral(goqu.Dialect("postgres"). - From("relays_blocks"). - Select( - goqu.L("exec_block_hash"), - goqu.MAX("value").As("value")). - Where(goqu.L("relays_blocks.exec_block_hash = b.exec_block_hash")). - GroupBy("exec_block_hash")).As("rb"), - goqu.On(goqu.L("rb.exec_block_hash = b.exec_block_hash")), - ). - Where(goqu.L("b.epoch >= ?", startEpoch)) + Where(goqu.L("b.epoch >= ?", startEpoch)). + LeftJoin(goqu.I("execution_rewards_finalized").As('b'), goqu.On(goqu.L("v.validator_index = b.proposer"))). + GroupBy(goqu.L("b.epoch")) if dashboardId.Validators == nil { rewardsDs = rewardsDs. @@ -986,21 +955,9 @@ func (d *DataAccessService) GetValidatorDashboardDuties(ctx context.Context, das elDs := goqu.Dialect("postgres"). Select( goqu.L("b.proposer"), - goqu.L("SUM(COALESCE(rb.value, ep.fee_recipient_reward * 1e18, 0)) AS el_rewards")). - From(goqu.L("blocks b")). - LeftJoin(goqu.L("execution_payloads ep"), goqu.On(goqu.L("ep.block_hash = b.exec_block_hash"))). - LeftJoin( - goqu.Lateral(goqu.Dialect("postgres"). - From("relays_blocks"). - Select( - goqu.L("exec_block_hash"), - goqu.MAX("value").As("value")). - Where(goqu.L("relays_blocks.exec_block_hash = b.exec_block_hash")). - GroupBy("exec_block_hash")).As("rb"), - goqu.On(goqu.L("rb.exec_block_hash = b.exec_block_hash")), - ). + goqu.SUM(goqu.I("value")).As("el_rewards")). + From(goqu.I("execution_rewards_finalized").As('b')). Where(goqu.L("b.epoch = ?", epoch)). - Where(goqu.L("b.status = '1'")). GroupBy(goqu.L("b.proposer")) // ------------------------------------------------------------------------------------------------------------------ diff --git a/backend/pkg/api/data_access/vdb_summary.go b/backend/pkg/api/data_access/vdb_summary.go index da45182d4..adde58e63 100644 --- a/backend/pkg/api/data_access/vdb_summary.go +++ b/backend/pkg/api/data_access/vdb_summary.go @@ -195,19 +195,8 @@ func (d *DataAccessService) GetValidatorDashboardSummary(ctx context.Context, da elRewards := make(map[int64]decimal.Decimal) ds = goqu.Dialect("postgres"). Select( - goqu.L("SUM(COALESCE(rb.value, ep.fee_recipient_reward * 1e18, 0)) AS el_rewards")). - From(goqu.L("blocks b")). - LeftJoin(goqu.L("execution_payloads ep"), goqu.On(goqu.L("ep.block_hash = b.exec_block_hash"))). - LeftJoin( - goqu.Lateral(goqu.Dialect("postgres"). - From("relays_blocks"). - Select( - goqu.L("exec_block_hash"), - goqu.MAX("value").As("value")). - Where(goqu.L("relays_blocks.exec_block_hash = b.exec_block_hash")). - GroupBy("exec_block_hash")).As("rb"), - goqu.On(goqu.L("rb.exec_block_hash = b.exec_block_hash")), - ). + goqu.SUM(goqu.I("value")).As("el_rewards")). + From(goqu.I("execution_rewards_finalized").As('b')). Where(goqu.L("b.epoch >= ? AND b.epoch <= ? AND b.status = '1'", epochMin, epochMax)). GroupBy(goqu.L("result_group_id")) @@ -929,20 +918,8 @@ func (d *DataAccessService) internal_getElClAPR(ctx context.Context, dashboardId } elDs := goqu.Dialect("postgres"). - Select(goqu.L("COALESCE(SUM(COALESCE(rb.value / 1e18, fee_recipient_reward)), 0) AS el_reward")). - From(goqu.L("blocks AS b")). - LeftJoin(goqu.L("execution_payloads AS ep"), goqu.On(goqu.L("b.exec_block_hash = ep.block_hash"))). - LeftJoin( - goqu.Lateral(goqu.Dialect("postgres"). - From("relays_blocks"). - Select( - goqu.L("exec_block_hash"), - goqu.MAX("value").As("value")). - Where(goqu.L("relays_blocks.exec_block_hash = b.exec_block_hash")). - GroupBy("exec_block_hash")).As("rb"), - goqu.On(goqu.L("rb.exec_block_hash = b.exec_block_hash")), - ). - Where(goqu.L("b.status = '1'")) + Select(goqu.SUM(goqu.I("value"))). + From(goqu.I("execution_rewards_finalized")).As("b") if len(dashboardId.Validators) > 0 { elDs = elDs. From ab9b0482eb1599e90a74524668692052c9d52f02 Mon Sep 17 00:00:00 2001 From: invis-bitfly <162128378+invis-bitfly@users.noreply.github.com> Date: Thu, 9 Jan 2025 12:42:11 +0100 Subject: [PATCH 5/9] fix(exporter): remove unused import --- backend/pkg/exporter/modules/execution_payloads_exporter.go | 1 - backend/pkg/exporter/modules/execution_rewards_finalizer.go | 2 -- 2 files changed, 3 deletions(-) diff --git a/backend/pkg/exporter/modules/execution_payloads_exporter.go b/backend/pkg/exporter/modules/execution_payloads_exporter.go index cd456c5b6..7266821cc 100644 --- a/backend/pkg/exporter/modules/execution_payloads_exporter.go +++ b/backend/pkg/exporter/modules/execution_payloads_exporter.go @@ -6,7 +6,6 @@ import ( "fmt" "math/big" "sync" - "time" "github.com/gobitfly/beaconchain/pkg/commons/db" "github.com/gobitfly/beaconchain/pkg/commons/log" diff --git a/backend/pkg/exporter/modules/execution_rewards_finalizer.go b/backend/pkg/exporter/modules/execution_rewards_finalizer.go index 0765b8d33..c2cbf20a1 100644 --- a/backend/pkg/exporter/modules/execution_rewards_finalizer.go +++ b/backend/pkg/exporter/modules/execution_rewards_finalizer.go @@ -24,7 +24,6 @@ func NewExecutionRewardFinalizer(moduleContext ModuleContext) ModuleInterface { } func (d *executionRewardsFinalizer) Init() error { - log.Infof("hi") return nil } @@ -37,7 +36,6 @@ func (d *executionRewardsFinalizer) OnChainReorg(event *constypes.StandardEventC } func (d *executionRewardsFinalizer) OnFinalizedCheckpoint(event *constypes.StandardFinalizedCheckpointResponse) (err error) { - return nil // nop } From d97b1723c737be0b466ee5cc2483592d35801e65 Mon Sep 17 00:00:00 2001 From: invis-bitfly <162128378+invis-bitfly@users.noreply.github.com> Date: Thu, 9 Jan 2025 13:31:27 +0100 Subject: [PATCH 6/9] fix(api): 'b' is a rune, not a string --- backend/pkg/api/data_access/vdb_rewards.go | 8 ++++---- backend/pkg/api/data_access/vdb_summary.go | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/backend/pkg/api/data_access/vdb_rewards.go b/backend/pkg/api/data_access/vdb_rewards.go index 8b253bf50..a71695d1a 100644 --- a/backend/pkg/api/data_access/vdb_rewards.go +++ b/backend/pkg/api/data_access/vdb_rewards.go @@ -103,7 +103,7 @@ func (d *DataAccessService) GetValidatorDashboardRewards(ctx context.Context, da goqu.SUM(goqu.I("value")).As("el_rewards")). From(goqu.L("users_val_dashboards_validators v")). Where(goqu.L("b.epoch >= ?", startEpoch)). - LeftJoin(goqu.I("execution_rewards_finalized").As('b'), goqu.On(goqu.L("v.validator_index = b.proposer"))). + LeftJoin(goqu.I("execution_rewards_finalized").As("b"), goqu.On(goqu.L("v.validator_index = b.proposer"))). GroupBy(goqu.L("b.epoch")) if dashboardId.Validators == nil { @@ -549,7 +549,7 @@ func (d *DataAccessService) GetValidatorDashboardGroupRewards(ctx context.Contex Select( goqu.SUM(goqu.I("value")).As("blocks_el_rewards")). From(goqu.L("users_val_dashboards_validators v")). - LeftJoin(goqu.I("execution_rewards_finalized").As('b'), goqu.On(goqu.L("v.validator_index = b.proposer"))). + LeftJoin(goqu.I("execution_rewards_finalized").As("b"), goqu.On(goqu.L("v.validator_index = b.proposer"))). Where(goqu.L("b.epoch = ?", epoch)) // handle the case when we have a list of validators @@ -715,7 +715,7 @@ func (d *DataAccessService) GetValidatorDashboardRewardsChart(ctx context.Contex goqu.SUM(goqu.I("value")).As("el_rewards")). From(goqu.L("users_val_dashboards_validators v")). Where(goqu.L("b.epoch >= ?", startEpoch)). - LeftJoin(goqu.I("execution_rewards_finalized").As('b'), goqu.On(goqu.L("v.validator_index = b.proposer"))). + LeftJoin(goqu.I("execution_rewards_finalized").As("b"), goqu.On(goqu.L("v.validator_index = b.proposer"))). GroupBy(goqu.L("b.epoch")) if dashboardId.Validators == nil { @@ -956,7 +956,7 @@ func (d *DataAccessService) GetValidatorDashboardDuties(ctx context.Context, das Select( goqu.L("b.proposer"), goqu.SUM(goqu.I("value")).As("el_rewards")). - From(goqu.I("execution_rewards_finalized").As('b')). + From(goqu.I("execution_rewards_finalized").As("b")). Where(goqu.L("b.epoch = ?", epoch)). GroupBy(goqu.L("b.proposer")) diff --git a/backend/pkg/api/data_access/vdb_summary.go b/backend/pkg/api/data_access/vdb_summary.go index adde58e63..4319a9a95 100644 --- a/backend/pkg/api/data_access/vdb_summary.go +++ b/backend/pkg/api/data_access/vdb_summary.go @@ -196,7 +196,7 @@ func (d *DataAccessService) GetValidatorDashboardSummary(ctx context.Context, da ds = goqu.Dialect("postgres"). Select( goqu.SUM(goqu.I("value")).As("el_rewards")). - From(goqu.I("execution_rewards_finalized").As('b')). + From(goqu.I("execution_rewards_finalized").As("b")). Where(goqu.L("b.epoch >= ? AND b.epoch <= ? AND b.status = '1'", epochMin, epochMax)). GroupBy(goqu.L("result_group_id")) From d115d8957b20d89d3a072e0b60c2fea075de957f Mon Sep 17 00:00:00 2001 From: invis-bitfly <162128378+invis-bitfly@users.noreply.github.com> Date: Thu, 9 Jan 2025 13:37:18 +0100 Subject: [PATCH 7/9] fix(api): copy&paste errors --- backend/pkg/api/data_access/vdb_rewards.go | 8 ++++---- backend/pkg/api/data_access/vdb_summary.go | 8 ++++---- .../pkg/exporter/modules/execution_rewards_finalizer.go | 2 +- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/backend/pkg/api/data_access/vdb_rewards.go b/backend/pkg/api/data_access/vdb_rewards.go index a71695d1a..aacafada5 100644 --- a/backend/pkg/api/data_access/vdb_rewards.go +++ b/backend/pkg/api/data_access/vdb_rewards.go @@ -100,7 +100,7 @@ func (d *DataAccessService) GetValidatorDashboardRewards(ctx context.Context, da elDs := goqu.Dialect("postgres"). Select( goqu.L("b.epoch"), - goqu.SUM(goqu.I("value")).As("el_rewards")). + goqu.COALESCE(goqu.SUM(goqu.I("value")), 0).As("el_rewards")). From(goqu.L("users_val_dashboards_validators v")). Where(goqu.L("b.epoch >= ?", startEpoch)). LeftJoin(goqu.I("execution_rewards_finalized").As("b"), goqu.On(goqu.L("v.validator_index = b.proposer"))). @@ -547,7 +547,7 @@ func (d *DataAccessService) GetValidatorDashboardGroupRewards(ctx context.Contex elDs := goqu.Dialect("postgres"). Select( - goqu.SUM(goqu.I("value")).As("blocks_el_rewards")). + goqu.COALESCE(goqu.SUM(goqu.I("value")), 0).As("blocks_el_rewards")). From(goqu.L("users_val_dashboards_validators v")). LeftJoin(goqu.I("execution_rewards_finalized").As("b"), goqu.On(goqu.L("v.validator_index = b.proposer"))). Where(goqu.L("b.epoch = ?", epoch)) @@ -712,7 +712,7 @@ func (d *DataAccessService) GetValidatorDashboardRewardsChart(ctx context.Contex elDs := goqu.Dialect("postgres"). Select( goqu.L("b.epoch"), - goqu.SUM(goqu.I("value")).As("el_rewards")). + goqu.COALESCE(goqu.SUM(goqu.I("value")), 0).As("el_rewards")). From(goqu.L("users_val_dashboards_validators v")). Where(goqu.L("b.epoch >= ?", startEpoch)). LeftJoin(goqu.I("execution_rewards_finalized").As("b"), goqu.On(goqu.L("v.validator_index = b.proposer"))). @@ -955,7 +955,7 @@ func (d *DataAccessService) GetValidatorDashboardDuties(ctx context.Context, das elDs := goqu.Dialect("postgres"). Select( goqu.L("b.proposer"), - goqu.SUM(goqu.I("value")).As("el_rewards")). + goqu.COALESCE(goqu.SUM(goqu.I("value")), 0).As("el_rewards")). From(goqu.I("execution_rewards_finalized").As("b")). Where(goqu.L("b.epoch = ?", epoch)). GroupBy(goqu.L("b.proposer")) diff --git a/backend/pkg/api/data_access/vdb_summary.go b/backend/pkg/api/data_access/vdb_summary.go index 4319a9a95..aefa97396 100644 --- a/backend/pkg/api/data_access/vdb_summary.go +++ b/backend/pkg/api/data_access/vdb_summary.go @@ -195,9 +195,9 @@ func (d *DataAccessService) GetValidatorDashboardSummary(ctx context.Context, da elRewards := make(map[int64]decimal.Decimal) ds = goqu.Dialect("postgres"). Select( - goqu.SUM(goqu.I("value")).As("el_rewards")). + goqu.COALESCE(goqu.SUM(goqu.I("value")), 0).As("el_rewards")). From(goqu.I("execution_rewards_finalized").As("b")). - Where(goqu.L("b.epoch >= ? AND b.epoch <= ? AND b.status = '1'", epochMin, epochMax)). + Where(goqu.L("b.epoch >= ? AND b.epoch <= ?", epochMin, epochMax)). GroupBy(goqu.L("result_group_id")) if len(validators) > 0 { @@ -918,8 +918,8 @@ func (d *DataAccessService) internal_getElClAPR(ctx context.Context, dashboardId } elDs := goqu.Dialect("postgres"). - Select(goqu.SUM(goqu.I("value"))). - From(goqu.I("execution_rewards_finalized")).As("b") + Select(goqu.COALESCE(goqu.SUM(goqu.L("value / 1e18")), 0)). + From(goqu.I("execution_rewards_finalized").As("b")) if len(dashboardId.Validators) > 0 { elDs = elDs. diff --git a/backend/pkg/exporter/modules/execution_rewards_finalizer.go b/backend/pkg/exporter/modules/execution_rewards_finalizer.go index c2cbf20a1..34c6a115d 100644 --- a/backend/pkg/exporter/modules/execution_rewards_finalizer.go +++ b/backend/pkg/exporter/modules/execution_rewards_finalizer.go @@ -105,7 +105,7 @@ func (d *executionRewardsFinalizer) maintainTable() (err error) { goqu.I("b.epoch").As("epoch"), goqu.I("b.slot").As("slot"), goqu.I("b.proposer").As("proposer"), - goqu.Func("sum", goqu.COALESCE(goqu.I("rb.value"), goqu.L("ep.fee_recipient_reward * '10e18'::numeric"), goqu.L("0::numeric"))).As("value"), + goqu.Func("sum", goqu.COALESCE(goqu.I("rb.value"), goqu.L("ep.fee_recipient_reward * '1e18'::numeric"), goqu.L("0::numeric"))).As("value"), ). Where( goqu.I("b.slot").Gt(lastExportedSlot), From 2ae1aae41043d06bfa05ee7a99bb4448fc2d5d3a Mon Sep 17 00:00:00 2001 From: invis-bitfly <162128378+invis-bitfly@users.noreply.github.com> Date: Thu, 9 Jan 2025 14:46:02 +0100 Subject: [PATCH 8/9] fix(exporter): use max instead of sum for el rewards finalizer --- backend/pkg/exporter/modules/execution_rewards_finalizer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/pkg/exporter/modules/execution_rewards_finalizer.go b/backend/pkg/exporter/modules/execution_rewards_finalizer.go index 34c6a115d..13a6529af 100644 --- a/backend/pkg/exporter/modules/execution_rewards_finalizer.go +++ b/backend/pkg/exporter/modules/execution_rewards_finalizer.go @@ -105,7 +105,7 @@ func (d *executionRewardsFinalizer) maintainTable() (err error) { goqu.I("b.epoch").As("epoch"), goqu.I("b.slot").As("slot"), goqu.I("b.proposer").As("proposer"), - goqu.Func("sum", goqu.COALESCE(goqu.I("rb.value"), goqu.L("ep.fee_recipient_reward * '1e18'::numeric"), goqu.L("0::numeric"))).As("value"), + goqu.Func("max", goqu.COALESCE(goqu.I("rb.value"), goqu.L("ep.fee_recipient_reward * '1e18'::numeric"), goqu.L("0::numeric"))).As("value"), ). Where( goqu.I("b.slot").Gt(lastExportedSlot), From 975fedef3f01d5429aa6a960870b12800574372b Mon Sep 17 00:00:00 2001 From: invis-bitfly <162128378+invis-bitfly@users.noreply.github.com> Date: Thu, 9 Jan 2025 14:49:04 +0100 Subject: [PATCH 9/9] feat(migrations): add finalized el rewards table --- ..._add_finalized_execution_rewards_table.sql | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) create mode 100644 backend/pkg/commons/db/migrations/postgres/20250109144744_add_finalized_execution_rewards_table.sql diff --git a/backend/pkg/commons/db/migrations/postgres/20250109144744_add_finalized_execution_rewards_table.sql b/backend/pkg/commons/db/migrations/postgres/20250109144744_add_finalized_execution_rewards_table.sql new file mode 100644 index 000000000..4e48f83c7 --- /dev/null +++ b/backend/pkg/commons/db/migrations/postgres/20250109144744_add_finalized_execution_rewards_table.sql @@ -0,0 +1,21 @@ +-- +goose Up +-- +goose StatementBegin +CREATE TABLE execution_rewards_finalized ( + epoch int4 NOT NULL, + slot int4 NOT NULL, + proposer int4 NOT NULL, + value numeric NOT NULL, + CONSTRAINT finalized_execution_rewards_pk PRIMARY KEY (slot) +); +-- +goose StatementEnd +-- +goose StatementBegin +CREATE INDEX finalized_execution_rewards_epoch_idx ON execution_rewards_finalized USING btree (epoch); +-- +goose StatementEnd +-- +goose StatementBegin +CREATE UNIQUE INDEX finalized_execution_rewards_proposer_idx ON execution_rewards_finalized USING btree (proposer, slot); +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +DROP TABLE execution_rewards_finalized; +-- +goose StatementEnd