Bids 3236/ch for summary #657
Conversation
| @@ -732,62 +738,138 @@ func (d *DataAccessService) GetValidatorDashboardGroupSummary(ctx context.Contex | |||
| return ret, nil | |||
| } | |||
|
|
|||
| func (d *DataAccessService) internal_getElClAPR(ctx context.Context, validators []t.VDBValidator, days int) (elIncome decimal.Decimal, elAPR float64, clIncome decimal.Decimal, clAPR float64, err error) { | |||
There was a problem hiding this comment.
The reason I had to change the whole method from a validator list validators to the dashboardId (+group filter) is because I could not change the query for clickhouse to
WHERE validator_index IN ($1)
For a large dashboard, e.g. 250,000 validators, you get a "Max query size exceeded" error.
Deploying beaconchain with
|
| Latest commit: |
f5606e4
|
| Status: | 🚫 Build failed. |
| With("validators", goqu.L("(SELECT dashboard_id, group_id, validator_index FROM users_val_dashboards_validators WHERE dashboard_id = ?)", dashboardId.Id)). | ||
| Select( | ||
| goqu.L("ARRAY_AGG(r.validator_index) AS validator_indices"), | ||
| goqu.L("SUM(COALESCE(r.attestations_reward, 0) + COALESCE(r.blocks_cl_reward, 0) + COALESCE(r.sync_rewards, 0) + COALESCE(r.blocks_cl_slasher_reward, 0)) AS cl_rewards"), |
There was a problem hiding this comment.
| goqu.L("SUM(COALESCE(r.attestations_reward, 0) + COALESCE(r.blocks_cl_reward, 0) + COALESCE(r.sync_rewards, 0) + COALESCE(r.blocks_cl_slasher_reward, 0)) AS cl_rewards"), | |
| goqu.L("COALESCE(SUM(r.attestations_reward + r.blocks_cl_reward + r.sync_rewards + r.blocks_cl_slasher_reward), 0) AS cl_rewards"), |
these columns arent nullable in clickhouse so this should work
| if row.MaxEpochEnd > epochMax { | ||
| epochMax = row.MaxEpochEnd | ||
| } | ||
| } | ||
| // ------------------------------------------------------------------------------------------------------------------ | ||
| // Get the EL rewards |
There was a problem hiding this comment.
validator_dashboard_view_execution_rewards_epoch is a proxied view that exists in clickhouse that exposes value proposer and epoch - in theory this can even be combined in the above query.
but not really important enough to have this block the pr
There was a problem hiding this comment.
I will keep it in mind for the future.
| rewardsDs = rewardsDs. | ||
| InnerJoin(goqu.L("users_val_dashboards_validators v"), goqu.On(goqu.L("r.validator_index = v.validator_index"))). | ||
| Where(goqu.L("v.dashboard_id = ?", dashboardId.Id)) |
There was a problem hiding this comment.
this is prob slow as it isn't using the with subquery optimization that gets used above
There was a problem hiding this comment.
I changed it to the subquery for here and also the validator slashing list which should now be all clickhouse queries.
However I also removed the WHERE condition after the INNER JOIN since it was redudant, or was there an unknown performance trick that needed it?
There was a problem hiding this comment.
However I also removed the WHERE condition after the INNER JOIN since it was redudant, or was there an unknown performance trick that needed it?
doesnt seem like the clickhouse query planner is smart enough to optimize the join that way. without the where filter the query always scans the complete from table
There was a problem hiding this comment.
Ah, I see. I reverted the removal of the WHERE.
| LEFT JOIN relays_blocks rb ON blocks.exec_block_hash = rb.exec_block_hash | ||
| WHERE proposer = ANY($1) AND status = '1' AND slot >= (SELECT MIN(epoch_start) * $2 FROM %s WHERE validator_index = ANY($1));` | ||
| err = db.AlloyReader.GetContext(ctx, &elIncome, fmt.Sprintf(query, table), validators, utils.Config.Chain.ClConfig.SlotsPerEpoch) | ||
| err = d.alloyReader.GetContext(ctx, &elIncome, query, args...) |
There was a problem hiding this comment.
same as above, could be rewriten to use the proxy view in clickhouse
No description provided.