Skip to content

Commit

Permalink
Merge branch 'main' into NOBIDS/public-vali-dashboard-api
Browse files Browse the repository at this point in the history
# Conflicts:
#	backend/pkg/api/data_access/user.go
#	backend/pkg/api/handlers/internal.go
#	backend/pkg/api/router.go
  • Loading branch information
LuccaBitfly committed Jun 6, 2024
2 parents ae3751c + 2d99529 commit 1545394
Show file tree
Hide file tree
Showing 161 changed files with 4,851 additions and 2,459 deletions.
43 changes: 33 additions & 10 deletions backend/db_migrations/20240304094628_dashboard_data.sql
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,8 @@ CREATE TABLE IF NOT EXISTS validator_dashboard_data_epoch (
withdrawals_count smallint,
withdrawals_amount BIGINT,
inclusion_delay_sum smallint,
block_chance double precision,
blocks_expected double precision,
sync_committees_expected double precision,
attestations_scheduled smallint,
attestations_executed smallint,
attestation_head_executed smallint,
Expand All @@ -52,6 +53,8 @@ CREATE TABLE IF NOT EXISTS validator_dashboard_data_epoch (
slashed_violation smallint, -- 0: attestation, 1: block
slasher_reward BIGINT, -- gwei
last_executed_duty_epoch int,
blocks_cl_attestations_reward BIGINT, -- gwei
blocks_cl_sync_aggregate_reward BIGINT, -- gwei
primary key (validator_index, epoch)
) PARTITION BY range (epoch);

Expand Down Expand Up @@ -89,7 +92,8 @@ CREATE TABLE IF NOT EXISTS validator_dashboard_data_hourly (
withdrawals_count smallint,
withdrawals_amount BIGINT,
inclusion_delay_sum int,
block_chance double precision,
blocks_expected double precision,
sync_committees_expected double precision,
attestations_scheduled smallint,
attestations_executed smallint,
attestation_head_executed smallint,
Expand All @@ -100,10 +104,11 @@ CREATE TABLE IF NOT EXISTS validator_dashboard_data_hourly (
slashed_violation smallint, -- 0: attestation, 1: block
slasher_reward BIGINT, -- gwei
last_executed_duty_epoch int,
blocks_cl_attestations_reward BIGINT, -- gwei
blocks_cl_sync_aggregate_reward BIGINT, -- gwei
primary key (epoch_start, validator_index)
) PARTITION BY range(epoch_start);


CREATE TABLE IF NOT EXISTS validator_dashboard_data_daily (
validator_index int NOT NULL,
day date NOT NULL,
Expand Down Expand Up @@ -136,7 +141,8 @@ CREATE TABLE IF NOT EXISTS validator_dashboard_data_daily (
withdrawals_count smallint,
withdrawals_amount BIGINT,
inclusion_delay_sum int,
block_chance double precision,
blocks_expected double precision,
sync_committees_expected double precision,
attestations_scheduled smallint,
attestations_executed smallint,
attestation_head_executed smallint,
Expand All @@ -147,13 +153,15 @@ CREATE TABLE IF NOT EXISTS validator_dashboard_data_daily (
slashed_violation smallint, -- 0: attestation, 1: block
slasher_reward BIGINT, -- gwei
last_executed_duty_epoch int,
blocks_cl_attestations_reward BIGINT, -- gwei
blocks_cl_sync_aggregate_reward BIGINT, -- gwei
primary key (day, validator_index)
) PARTITION BY range(day);


CREATE TABLE IF NOT EXISTS validator_dashboard_data_rolling_daily (
validator_index int NOT NULL,
epoch_start int NOT NULL, -- incl
epoch_start int NOT NULL, -- incl. Do not rely on the epoch_start to be true in every case. Validators that join on current day but get activated later may have an out of bounds epoch_start but this is intended.
epoch_end int NOT NULL, -- excl
attestations_source_reward int ,
attestations_target_reward int,
Expand Down Expand Up @@ -182,7 +190,8 @@ CREATE TABLE IF NOT EXISTS validator_dashboard_data_rolling_daily (
withdrawals_count smallint,
withdrawals_amount BIGINT,
inclusion_delay_sum int,
block_chance double precision,
blocks_expected double precision,
sync_committees_expected double precision,
attestations_scheduled smallint,
attestations_executed smallint,
attestation_head_executed smallint,
Expand All @@ -193,6 +202,8 @@ CREATE TABLE IF NOT EXISTS validator_dashboard_data_rolling_daily (
slashed_violation smallint, -- 0: attestation, 1: block
slasher_reward BIGINT, -- gwei
last_executed_duty_epoch int,
blocks_cl_attestations_reward BIGINT, -- gwei
blocks_cl_sync_aggregate_reward BIGINT, -- gwei
primary key (validator_index)
);

Expand Down Expand Up @@ -227,7 +238,8 @@ CREATE TABLE IF NOT EXISTS validator_dashboard_data_rolling_weekly (
withdrawals_count int,
withdrawals_amount BIGINT,
inclusion_delay_sum int,
block_chance double precision,
blocks_expected double precision,
sync_committees_expected double precision,
attestations_scheduled int,
attestations_executed int,
attestation_head_executed int,
Expand All @@ -238,6 +250,8 @@ CREATE TABLE IF NOT EXISTS validator_dashboard_data_rolling_weekly (
slashed_violation smallint, -- 0: attestation, 1: block
slasher_reward BIGINT, -- gwei
last_executed_duty_epoch int,
blocks_cl_attestations_reward BIGINT, -- gwei
blocks_cl_sync_aggregate_reward BIGINT, -- gwei
primary key (validator_index)
);

Expand Down Expand Up @@ -272,7 +286,8 @@ CREATE TABLE IF NOT EXISTS validator_dashboard_data_rolling_monthly (
withdrawals_count int,
withdrawals_amount BIGINT,
inclusion_delay_sum int,
block_chance double precision,
blocks_expected double precision,
sync_committees_expected double precision,
attestations_scheduled int,
attestations_executed int,
attestation_head_executed int,
Expand All @@ -283,6 +298,8 @@ CREATE TABLE IF NOT EXISTS validator_dashboard_data_rolling_monthly (
slashed_violation smallint, -- 0: attestation, 1: block
slasher_reward BIGINT, -- gwei
last_executed_duty_epoch int,
blocks_cl_attestations_reward BIGINT, -- gwei
blocks_cl_sync_aggregate_reward BIGINT, -- gwei
primary key (validator_index)
);

Expand Down Expand Up @@ -317,7 +334,8 @@ CREATE TABLE IF NOT EXISTS validator_dashboard_data_rolling_90d (
withdrawals_count BIGINT,
withdrawals_amount BIGINT,
inclusion_delay_sum BIGINT,
block_chance double precision,
blocks_expected double precision,
sync_committees_expected double precision,
attestations_scheduled int,
attestations_executed int,
attestation_head_executed int,
Expand All @@ -328,6 +346,8 @@ CREATE TABLE IF NOT EXISTS validator_dashboard_data_rolling_90d (
slashed_violation smallint, -- 0: attestation, 1: block
slasher_reward BIGINT, -- gwei
last_executed_duty_epoch int,
blocks_cl_attestations_reward BIGINT, -- gwei
blocks_cl_sync_aggregate_reward BIGINT, -- gwei
primary key (validator_index)
);

Expand Down Expand Up @@ -362,7 +382,8 @@ CREATE TABLE IF NOT EXISTS validator_dashboard_data_rolling_total (
withdrawals_count BIGINT,
withdrawals_amount BIGINT,
inclusion_delay_sum BIGINT,
block_chance double precision,
blocks_expected double precision,
sync_committees_expected double precision,
attestations_scheduled int,
attestations_executed int,
attestation_head_executed int,
Expand All @@ -373,6 +394,8 @@ CREATE TABLE IF NOT EXISTS validator_dashboard_data_rolling_total (
slashed_violation smallint, -- 0: attestation, 1: block
slasher_reward BIGINT, -- gwei
last_executed_duty_epoch int,
blocks_cl_attestations_reward BIGINT, -- gwei
blocks_cl_sync_aggregate_reward BIGINT, -- gwei
primary key (validator_index)
);

Expand Down
91 changes: 91 additions & 0 deletions backend/db_migrations/20240412154432_blocksdeposits_validators.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
-- +goose Up
-- +goose StatementBegin
CREATE TABLE IF NOT EXISTS
blocks_deposits (
block_slot INT NOT NULL,
block_index INT NOT NULL,
block_root bytea NOT NULL DEFAULT '',
proof bytea[],
publickey bytea NOT NULL,
withdrawalcredentials bytea NOT NULL,
amount BIGINT NOT NULL,
signature bytea NOT NULL,
valid_signature bool NOT NULL DEFAULT TRUE,
PRIMARY KEY (block_slot, block_index)
);
CREATE INDEX IF NOT EXISTS idx_blocks_deposits_publickey ON blocks_deposits (publickey);
CREATE INDEX IF NOT EXISTS idx_blocks_deposits_block_slot_block_root ON public.blocks_deposits USING btree (block_slot, block_root);
CREATE INDEX IF NOT EXISTS idx_blocks_deposits_block_root_publickey ON public.blocks_deposits USING btree (block_root, publickey);

DROP TABLE IF EXISTS validators;
CREATE TABLE IF NOT EXISTS
validators (
validatorindex INT NOT NULL,
pubkey bytea NOT NULL,
pubkeyhex TEXT NOT NULL DEFAULT '',
withdrawableepoch BIGINT NOT NULL,
withdrawalcredentials bytea NOT NULL,
balance BIGINT NOT NULL,
balanceactivation BIGINT,
effectivebalance BIGINT NOT NULL,
slashed bool NOT NULL,
activationeligibilityepoch BIGINT NOT NULL,
activationepoch BIGINT NOT NULL,
exitepoch BIGINT NOT NULL,
lastattestationslot BIGINT,
status VARCHAR(20) NOT NULL DEFAULT '',
PRIMARY KEY (validatorindex)
);
CREATE INDEX IF NOT EXISTS idx_validators_pubkey ON validators (pubkey);
CREATE INDEX IF NOT EXISTS idx_validators_pubkeyhex ON validators (pubkeyhex);
CREATE INDEX IF NOT EXISTS idx_validators_pubkeyhex_pattern_pos ON validators (pubkeyhex varchar_pattern_ops);
CREATE INDEX IF NOT EXISTS idx_validators_status ON validators (status);
CREATE INDEX IF NOT EXISTS idx_validators_balanceactivation ON validators (balanceactivation);
CREATE INDEX IF NOT EXISTS idx_validators_activationepoch ON validators (activationepoch);
CREATE INDEX IF NOT EXISTS validators_is_offline_vali_idx ON validators (validatorindex, lastattestationslot, pubkey);
CREATE INDEX IF NOT EXISTS idx_validators_withdrawalcredentials ON validators (withdrawalcredentials, validatorindex);
CREATE INDEX IF NOT EXISTS idx_validators_exitepoch ON validators (exitepoch);
CREATE INDEX IF NOT EXISTS idx_validators_withdrawableepoch ON validators (withdrawableepoch);
CREATE INDEX IF NOT EXISTS idx_validators_lastattestationslot ON validators (lastattestationslot);
CREATE INDEX IF NOT EXISTS idx_validators_activationepoch_status ON public.validators USING btree (activationepoch, status);
CREATE INDEX IF NOT EXISTS idx_validators_activationeligibilityepoch ON public.validators USING btree (activationeligibilityepoch);

do
$$
begin
if not exists (select * from pg_roles where rolname = 'alloydbsuperuser') then
create role alloydbsuperuser;
end if;
if not exists (select * from pg_roles where rolname = 'readaccess') then
create role readaccess;
end if;
end
$$
;

-- +goose StatementEnd

-- +goose Down
-- +goose StatementBegin

DROP TABLE IF EXISTS blocks_deposits;
DROP INDEX IF EXISTS idx_blocks_deposits_publickey;
DROP INDEX IF EXISTS idx_blocks_deposits_block_slot_block_root;
DROP INDEX IF EXISTS idx_blocks_deposits_block_root_publickey;

DROP TABLE IF EXISTS validators;
DROP INDEX IF EXISTS idx_validators_pubkey;
DROP INDEX IF EXISTS idx_validators_pubkeyhex;
DROP INDEX IF EXISTS idx_validators_pubkeyhex_pattern_pos;
DROP INDEX IF EXISTS idx_validators_status;
DROP INDEX IF EXISTS idx_validators_balanceactivation;
DROP INDEX IF EXISTS idx_validators_activationepoch;
DROP INDEX IF EXISTS validators_is_offline_vali_idx;
DROP INDEX IF EXISTS idx_validators_withdrawalcredentials;
DROP INDEX IF EXISTS idx_validators_exitepoch;
DROP INDEX IF EXISTS idx_validators_withdrawableepoch;
DROP INDEX IF EXISTS idx_validators_lastattestationslot;
DROP INDEX IF EXISTS idx_validators_activationepoch_status;
DROP INDEX IF EXISTS idx_validators_activationeligibilityepoch;

-- +goose StatementEnd
3 changes: 2 additions & 1 deletion backend/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ require (
github.com/gorilla/handlers v1.5.2
github.com/gorilla/mux v1.8.1
github.com/gorilla/websocket v1.5.0
github.com/gtuk/discordwebhook v1.2.0
github.com/gzuidhof/tygo v0.2.13
github.com/hashicorp/go-version v1.6.0
github.com/hashicorp/golang-lru v1.0.2
Expand Down Expand Up @@ -134,7 +135,6 @@ require (
github.com/gorilla/csrf v1.7.2 // indirect
github.com/gorilla/securecookie v1.1.2 // indirect
github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1 // indirect
github.com/gtuk/discordwebhook v1.2.0 // indirect
github.com/herumi/bls-eth-go-binary v1.31.0 // indirect
github.com/holiman/uint256 v1.2.4 // indirect
github.com/huandu/go-clone v1.6.0 // indirect
Expand Down Expand Up @@ -197,6 +197,7 @@ require (
github.com/sethvargo/go-retry v0.2.4 // indirect
github.com/shirou/gopsutil v3.21.11+incompatible // indirect
github.com/spaolacci/murmur3 v1.1.0 // indirect
github.com/stripe/stripe-go/v72 v72.122.0 // indirect
github.com/supranational/blst v0.3.11 // indirect
github.com/thomaso-mirodin/intmath v0.0.0-20160323211736-5dc6d854e46e // indirect
github.com/tklauser/go-sysconf v0.3.12 // indirect
Expand Down
3 changes: 3 additions & 0 deletions backend/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -838,6 +838,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/stripe/stripe-go/v72 v72.122.0 h1:eRXWqnEwGny6dneQ5BsxGzUCED5n180u8n665JHlut8=
github.com/stripe/stripe-go/v72 v72.122.0/go.mod h1:QwqJQtduHubZht9mek5sds9CtQcKFdsykV9ZepRWwo0=
github.com/supranational/blst v0.3.11 h1:LyU6FolezeWAhvQk0k6O/d49jqgO52MSDDfYgbeoEm4=
github.com/supranational/blst v0.3.11/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw=
github.com/syndtr/goleveldb v1.0.1-0.20220614013038-64ee5596c38a h1:1ur3QoCqvE5fl+nylMaIr9PVV1w343YRDtsy+Rwu7XI=
Expand Down Expand Up @@ -986,6 +988,7 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191116160921-f9c825593386/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
Expand Down
5 changes: 0 additions & 5 deletions backend/local_deployment/.env

This file was deleted.

3 changes: 3 additions & 0 deletions backend/local_deployment/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
.env
config.yml

6 changes: 3 additions & 3 deletions backend/local_deployment/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ Before continuing, restarting your computer now might save you from unexplained
```
cd ~
git clone https://github.com/gobitfly/eth2-beaconchain-explorer.git
cd eth2-beaconchain-explorer
cd eth2-beaconchain-explorer/backend
```

# Build the explorer binaries
Expand All @@ -59,15 +59,15 @@ make all

# Start postgres, redis, little_bigtable & the eth test network
```
cd ~/eth2-beaconchain-explorer/local_deployment/
cd ~/eth2-beaconchain-explorer/backend/local_deployment/
kurtosis clean -a && kurtosis run --enclave my-testnet . "$(cat network-params.json)"
```
Later in your developer life (after having started Kurtosis and stopped it a few times), if you encounter an error at this step, you might need to clean up bugged cache files from previous runs that Kurtosis or Docker left behind.
The `./stop` script [in this repository](https://github.com/thib-wien/scripts-localnetworkandexplorer) gathers cleaning commands which worked for their author (it might save you hours of browsing Stack Overflow and GitHub's issues).

# Generate the explorer config file for the deployed testnet
```
cd ~/eth2-beaconchain-explorer/local_deployment/
cd ~/eth2-beaconchain-explorer/backend/local_deployment/
bash provision-explorer-config.sh
```
This will generate a _config.yml_ to be used by the explorer and then create the bigtable & postgres schema.
Expand Down
Loading

0 comments on commit 1545394

Please sign in to comment.