Merge pull request #1016 from gobitfly/BEDS-397/group_efficiency_noti…

…fications Beds 397/group efficiency notifications
gobitfly · Oct 22, 2024 · 03ccfa1 · 03ccfa1
2 parents 9e091f2 + 3e209ab
commit 03ccfa1
Show file tree

Hide file tree

Showing 14 changed files with 426 additions and 118 deletions.
diff --git a/backend/cmd/misc/main.go b/backend/cmd/misc/main.go
@@ -247,27 +247,27 @@ func Run() {
 	}
 
 	// clickhouse
-	// db.ClickHouseWriter, db.ClickHouseReader = db.MustInitDB(&types.DatabaseConfig{
-	// 	Username:     cfg.ClickHouse.WriterDatabase.Username,
-	// 	Password:     cfg.ClickHouse.WriterDatabase.Password,
-	// 	Name:         cfg.ClickHouse.WriterDatabase.Name,
-	// 	Host:         cfg.ClickHouse.WriterDatabase.Host,
-	// 	Port:         cfg.ClickHouse.WriterDatabase.Port,
-	// 	MaxOpenConns: cfg.ClickHouse.WriterDatabase.MaxOpenConns,
-	// 	SSL:          true,
-	// 	MaxIdleConns: cfg.ClickHouse.WriterDatabase.MaxIdleConns,
-	// }, &types.DatabaseConfig{
-	// 	Username:     cfg.ClickHouse.ReaderDatabase.Username,
-	// 	Password:     cfg.ClickHouse.ReaderDatabase.Password,
-	// 	Name:         cfg.ClickHouse.ReaderDatabase.Name,
-	// 	Host:         cfg.ClickHouse.ReaderDatabase.Host,
-	// 	Port:         cfg.ClickHouse.ReaderDatabase.Port,
-	// 	MaxOpenConns: cfg.ClickHouse.ReaderDatabase.MaxOpenConns,
-	// 	SSL:          true,
-	// 	MaxIdleConns: cfg.ClickHouse.ReaderDatabase.MaxIdleConns,
-	// }, "clickhouse", "clickhouse")
-	// defer db.ClickHouseReader.Close()
-	// defer db.ClickHouseWriter.Close()
+	db.ClickHouseWriter, db.ClickHouseReader = db.MustInitDB(&types.DatabaseConfig{
+		Username:     cfg.ClickHouse.WriterDatabase.Username,
+		Password:     cfg.ClickHouse.WriterDatabase.Password,
+		Name:         cfg.ClickHouse.WriterDatabase.Name,
+		Host:         cfg.ClickHouse.WriterDatabase.Host,
+		Port:         cfg.ClickHouse.WriterDatabase.Port,
+		MaxOpenConns: cfg.ClickHouse.WriterDatabase.MaxOpenConns,
+		SSL:          true,
+		MaxIdleConns: cfg.ClickHouse.WriterDatabase.MaxIdleConns,
+	}, &types.DatabaseConfig{
+		Username:     cfg.ClickHouse.ReaderDatabase.Username,
+		Password:     cfg.ClickHouse.ReaderDatabase.Password,
+		Name:         cfg.ClickHouse.ReaderDatabase.Name,
+		Host:         cfg.ClickHouse.ReaderDatabase.Host,
+		Port:         cfg.ClickHouse.ReaderDatabase.Port,
+		MaxOpenConns: cfg.ClickHouse.ReaderDatabase.MaxOpenConns,
+		SSL:          true,
+		MaxIdleConns: cfg.ClickHouse.ReaderDatabase.MaxIdleConns,
+	}, "clickhouse", "clickhouse")
+	defer db.ClickHouseReader.Close()
+	defer db.ClickHouseWriter.Close()
 
 	// Initialize the persistent redis client
 	if requires.Redis {

diff --git a/backend/cmd/notification_collector/main.go b/backend/cmd/notification_collector/main.go
@@ -150,6 +150,31 @@ func Run() {
 		}, "pgx", "postgres")
 	}()
 
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		// clickhouse
+		db.ClickHouseWriter, db.ClickHouseReader = db.MustInitDB(&types.DatabaseConfig{
+			Username:     cfg.ClickHouse.WriterDatabase.Username,
+			Password:     cfg.ClickHouse.WriterDatabase.Password,
+			Name:         cfg.ClickHouse.WriterDatabase.Name,
+			Host:         cfg.ClickHouse.WriterDatabase.Host,
+			Port:         cfg.ClickHouse.WriterDatabase.Port,
+			MaxOpenConns: cfg.ClickHouse.WriterDatabase.MaxOpenConns,
+			SSL:          true,
+			MaxIdleConns: cfg.ClickHouse.WriterDatabase.MaxIdleConns,
+		}, &types.DatabaseConfig{
+			Username:     cfg.ClickHouse.ReaderDatabase.Username,
+			Password:     cfg.ClickHouse.ReaderDatabase.Password,
+			Name:         cfg.ClickHouse.ReaderDatabase.Name,
+			Host:         cfg.ClickHouse.ReaderDatabase.Host,
+			Port:         cfg.ClickHouse.ReaderDatabase.Port,
+			MaxOpenConns: cfg.ClickHouse.ReaderDatabase.MaxOpenConns,
+			SSL:          true,
+			MaxIdleConns: cfg.ClickHouse.ReaderDatabase.MaxIdleConns,
+		}, "clickhouse", "clickhouse")
+	}()
+
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
@@ -184,6 +209,8 @@ func Run() {
 	defer db.FrontendWriterDB.Close()
 	defer db.AlloyReader.Close()
 	defer db.AlloyWriter.Close()
+	defer db.ClickHouseReader.Close()
+	defer db.ClickHouseWriter.Close()
 	defer db.BigtableClient.Close()
 
 	log.Infof("database connection established")

diff --git a/backend/pkg/api/data_access/mobile.go b/backend/pkg/api/data_access/mobile.go
@@ -171,7 +171,7 @@ func (d *DataAccessService) GetValidatorDashboardMobileWidget(ctx context.Contex
 	if err != nil {
 		return nil, fmt.Errorf("error retrieving validator dashboard overview data: %w", err)
 	}
-	data.NetworkEfficiency = d.calculateTotalEfficiency(
+	data.NetworkEfficiency = utils.CalculateTotalEfficiency(
 		efficiency.AttestationEfficiency[enums.AllTime], efficiency.ProposalEfficiency[enums.AllTime], efficiency.SyncEfficiency[enums.AllTime])
 
 	// Validator status
@@ -327,7 +327,7 @@ func (d *DataAccessService) GetValidatorDashboardMobileWidget(ctx context.Contex
 				syncEfficiency.Float64 = float64(queryResult.SyncExecuted) / float64(queryResult.SyncScheduled)
 				syncEfficiency.Valid = true
 			}
-			*efficiency = d.calculateTotalEfficiency(attestationEfficiency, proposerEfficiency, syncEfficiency)
+			*efficiency = utils.CalculateTotalEfficiency(attestationEfficiency, proposerEfficiency, syncEfficiency)
 
 			return nil
 		})

diff --git a/backend/pkg/api/data_access/notifications.go b/backend/pkg/api/data_access/notifications.go
@@ -63,6 +63,7 @@ func (*DataAccessService) registerNotificationInterfaceTypes() {
 	once.Do(func() {
 		gob.Register(&n.ValidatorProposalNotification{})
 		gob.Register(&n.ValidatorUpcomingProposalNotification{})
+		gob.Register(&n.ValidatorGroupEfficiencyNotification{})
 		gob.Register(&n.ValidatorAttestationNotification{})
 		gob.Register(&n.ValidatorIsOfflineNotification{})
 		gob.Register(&n.ValidatorIsOnlineNotification{})

diff --git a/backend/pkg/api/data_access/vdb_helpers.go b/backend/pkg/api/data_access/vdb_helpers.go
@@ -41,28 +41,6 @@ func (d DataAccessService) getDashboardValidators(ctx context.Context, dashboard
 	return dashboardId.Validators, nil
 }
 
-func (d DataAccessService) calculateTotalEfficiency(attestationEff, proposalEff, syncEff sql.NullFloat64) float64 {
-	efficiency := float64(0)
-
-	if !attestationEff.Valid && !proposalEff.Valid && !syncEff.Valid {
-		efficiency = 0
-	} else if attestationEff.Valid && !proposalEff.Valid && !syncEff.Valid {
-		efficiency = attestationEff.Float64 * 100.0
-	} else if attestationEff.Valid && proposalEff.Valid && !syncEff.Valid {
-		efficiency = ((56.0 / 64.0 * attestationEff.Float64) + (8.0 / 64.0 * proposalEff.Float64)) * 100.0
-	} else if attestationEff.Valid && !proposalEff.Valid && syncEff.Valid {
-		efficiency = ((62.0 / 64.0 * attestationEff.Float64) + (2.0 / 64.0 * syncEff.Float64)) * 100.0
-	} else {
-		efficiency = (((54.0 / 64.0) * attestationEff.Float64) + ((8.0 / 64.0) * proposalEff.Float64) + ((2.0 / 64.0) * syncEff.Float64)) * 100.0
-	}
-
-	if efficiency < 0 {
-		efficiency = 0
-	}
-
-	return efficiency
-}
-
 func (d DataAccessService) calculateChartEfficiency(efficiencyType enums.VDBSummaryChartEfficiencyType, row *t.VDBValidatorSummaryChartRow) (float64, error) {
 	efficiency := float64(0)
 	switch efficiencyType {
@@ -81,7 +59,7 @@ func (d DataAccessService) calculateChartEfficiency(efficiencyType enums.VDBSumm
 			syncEfficiency.Valid = true
 		}
 
-		efficiency = d.calculateTotalEfficiency(attestationEfficiency, proposerEfficiency, syncEfficiency)
+		efficiency = utils.CalculateTotalEfficiency(attestationEfficiency, proposerEfficiency, syncEfficiency)
 	case enums.VDBSummaryChartAttestation:
 		if row.AttestationIdealReward > 0 {
 			efficiency = (row.AttestationReward / row.AttestationIdealReward) * 100

diff --git a/backend/pkg/api/data_access/vdb_management.go b/backend/pkg/api/data_access/vdb_management.go
@@ -520,7 +520,7 @@ func (d *DataAccessService) GetValidatorDashboardOverview(ctx context.Context, d
 				syncEfficiency.Float64 = float64(queryResult.SyncExecuted) / float64(queryResult.SyncScheduled)
 				syncEfficiency.Valid = true
 			}
-			*efficiency = d.calculateTotalEfficiency(attestationEfficiency, proposerEfficiency, syncEfficiency)
+			*efficiency = utils.CalculateTotalEfficiency(attestationEfficiency, proposerEfficiency, syncEfficiency)
 
 			return nil
 		})

diff --git a/backend/pkg/api/data_access/vdb_summary.go b/backend/pkg/api/data_access/vdb_summary.go
@@ -90,7 +90,7 @@ func (d *DataAccessService) GetValidatorDashboardSummary(ctx context.Context, da
 	if err != nil {
 		return nil, nil, err
 	}
-	averageNetworkEfficiency := d.calculateTotalEfficiency(
+	averageNetworkEfficiency := utils.CalculateTotalEfficiency(
 		efficiency.AttestationEfficiency[period], efficiency.ProposalEfficiency[period], efficiency.SyncEfficiency[period])
 
 	// ------------------------------------------------------------------------------------------------------------------
@@ -366,7 +366,7 @@ func (d *DataAccessService) GetValidatorDashboardSummary(ctx context.Context, da
 			syncEfficiency.Float64 = float64(queryEntry.SyncExecuted) / float64(queryEntry.SyncScheduled)
 			syncEfficiency.Valid = true
 		}
-		resultEntry.Efficiency = d.calculateTotalEfficiency(attestationEfficiency, proposerEfficiency, syncEfficiency)
+		resultEntry.Efficiency = utils.CalculateTotalEfficiency(attestationEfficiency, proposerEfficiency, syncEfficiency)
 
 		// Add the duties info to the total
 		total.AttestationReward = total.AttestationReward.Add(queryEntry.AttestationReward)
@@ -486,7 +486,7 @@ func (d *DataAccessService) GetValidatorDashboardSummary(ctx context.Context, da
 			totalSyncEfficiency.Float64 = float64(total.SyncExecuted) / float64(total.SyncScheduled)
 			totalSyncEfficiency.Valid = true
 		}
-		totalEntry.Efficiency = d.calculateTotalEfficiency(totalAttestationEfficiency, totalProposerEfficiency, totalSyncEfficiency)
+		totalEntry.Efficiency = utils.CalculateTotalEfficiency(totalAttestationEfficiency, totalProposerEfficiency, totalSyncEfficiency)
 
 		result = append([]t.VDBSummaryTableRow{totalEntry}, result...)
 	}
@@ -1021,7 +1021,7 @@ func (d *DataAccessService) GetValidatorDashboardSummaryChart(ctx context.Contex
 		if err != nil {
 			return nil, err
 		}
-		averageNetworkEfficiency := d.calculateTotalEfficiency(
+		averageNetworkEfficiency := utils.CalculateTotalEfficiency(
 			efficiency.AttestationEfficiency[enums.Last24h], efficiency.ProposalEfficiency[enums.Last24h], efficiency.SyncEfficiency[enums.Last24h])
 
 		for ts := range tsMap {

diff --git a/backend/pkg/commons/types/frontend.go b/backend/pkg/commons/types/frontend.go
@@ -131,6 +131,7 @@ var EventSortOrder = []EventName{
 	SyncCommitteeSoonEventName,
 	ValidatorIsOfflineEventName,
 	ValidatorIsOnlineEventName,
+	ValidatorGroupEfficiencyEventName,
 	ValidatorReceivedWithdrawalEventName,
 	NetworkLivenessIncreasedEventName,
 	EthClientUpdateEventName,
@@ -174,6 +175,7 @@ var MachineEventsMap = map[EventName]struct{}{
 
 var LegacyEventLabel map[EventName]string = map[EventName]string{
 	ValidatorUpcomingProposalEventName:       "Your validator(s) will soon propose a block",
+	ValidatorGroupEfficiencyEventName:        "Your validator group efficiency is low",
 	ValidatorMissedProposalEventName:         "Your validator(s) missed a proposal",
 	ValidatorExecutedProposalEventName:       "Your validator(s) submitted a proposal",
 	ValidatorMissedAttestationEventName:      "Your validator(s) missed an attestation",
@@ -198,6 +200,7 @@ var LegacyEventLabel map[EventName]string = map[EventName]string{
 
 var EventLabel map[EventName]string = map[EventName]string{
 	ValidatorUpcomingProposalEventName:       "Upcoming block proposal",
+	ValidatorGroupEfficiencyEventName:        "Low validator group efficiency",
 	ValidatorMissedProposalEventName:         "Block proposal missed",
 	ValidatorExecutedProposalEventName:       "Block proposal submitted",
 	ValidatorMissedAttestationEventName:      "Attestation missed",
@@ -232,6 +235,7 @@ func IsMachineNotification(event EventName) bool {
 
 var EventNames = []EventName{
 	ValidatorExecutedProposalEventName,
+	ValidatorGroupEfficiencyEventName,
 	ValidatorMissedProposalEventName,
 	ValidatorMissedAttestationEventName,
 	ValidatorGotSlashedEventName,

diff --git a/backend/pkg/commons/utils/efficiency.go b/backend/pkg/commons/utils/efficiency.go
@@ -0,0 +1,25 @@
+package utils
+
+import "database/sql"
+
+func CalculateTotalEfficiency(attestationEff, proposalEff, syncEff sql.NullFloat64) float64 {
+	efficiency := float64(0)
+
+	if !attestationEff.Valid && !proposalEff.Valid && !syncEff.Valid {
+		efficiency = 0
+	} else if attestationEff.Valid && !proposalEff.Valid && !syncEff.Valid {
+		efficiency = attestationEff.Float64 * 100.0
+	} else if attestationEff.Valid && proposalEff.Valid && !syncEff.Valid {
+		efficiency = ((56.0 / 64.0 * attestationEff.Float64) + (8.0 / 64.0 * proposalEff.Float64)) * 100.0
+	} else if attestationEff.Valid && !proposalEff.Valid && syncEff.Valid {
+		efficiency = ((62.0 / 64.0 * attestationEff.Float64) + (2.0 / 64.0 * syncEff.Float64)) * 100.0
+	} else {
+		efficiency = (((54.0 / 64.0) * attestationEff.Float64) + ((8.0 / 64.0) * proposalEff.Float64) + ((2.0 / 64.0) * syncEff.Float64)) * 100.0
+	}
+
+	if efficiency < 0 {
+		efficiency = 0
+	}
+
+	return efficiency
+}