From e1ff24949f50eb2225fb24aa74e1d0b161e7fc7f Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Tue, 15 Oct 2024 07:33:05 +0000 Subject: [PATCH 001/124] feat(notifications): respect per user email limits --- backend/pkg/api/data_access/user.go | 452 +------------------------- backend/pkg/commons/db/user.go | 470 ++++++++++++++++++++++++++++ backend/pkg/commons/mail/mail.go | 45 +-- 3 files changed, 498 insertions(+), 469 deletions(-) create mode 100644 backend/pkg/commons/db/user.go diff --git a/backend/pkg/api/data_access/user.go b/backend/pkg/api/data_access/user.go index 906742ecf..0c7a64884 100644 --- a/backend/pkg/api/data_access/user.go +++ b/backend/pkg/api/data_access/user.go @@ -4,10 +4,10 @@ import ( "context" "database/sql" "fmt" - "math" "time" t "github.com/gobitfly/beaconchain/pkg/api/types" + "github.com/gobitfly/beaconchain/pkg/commons/db" "github.com/gobitfly/beaconchain/pkg/commons/utils" "github.com/pkg/errors" "golang.org/x/sync/errgroup" @@ -258,460 +258,16 @@ func (d *DataAccessService) GetUserIdByResetHash(ctx context.Context, hash strin return result, err } -var adminPerks = t.PremiumPerks{ - AdFree: false, // admins want to see ads to check ad configuration - ValidatorDashboards: maxJsInt, - ValidatorsPerDashboard: maxJsInt, - ValidatorGroupsPerDashboard: maxJsInt, - ShareCustomDashboards: true, - ManageDashboardViaApi: true, - BulkAdding: true, - ChartHistorySeconds: t.ChartHistorySeconds{ - Epoch: maxJsInt, - Hourly: maxJsInt, - Daily: maxJsInt, - Weekly: maxJsInt, - }, - EmailNotificationsPerDay: maxJsInt, - ConfigureNotificationsViaApi: true, - ValidatorGroupNotifications: maxJsInt, - WebhookEndpoints: maxJsInt, - MobileAppCustomThemes: true, - MobileAppWidget: true, - MonitorMachines: maxJsInt, - MachineMonitoringHistorySeconds: maxJsInt, - NotificationsMachineCustomThreshold: true, - NotificationsValidatorDashboardRealTimeMode: true, - NotificationsValidatorDashboardGroupOffline: true, -} - func (d *DataAccessService) GetUserInfo(ctx context.Context, userId uint64) (*t.UserInfo, error) { - // TODO @patrick post-beta improve and unmock - userInfo := &t.UserInfo{ - Id: userId, - ApiKeys: []string{}, - ApiPerks: t.ApiPerks{ - UnitsPerSecond: 10, - UnitsPerMonth: 10, - ApiKeys: 4, - ConsensusLayerAPI: true, - ExecutionLayerAPI: true, - Layer2API: true, - NoAds: true, - DiscordSupport: false, - }, - Subscriptions: []t.UserSubscription{}, - } - - productSummary, err := d.GetProductSummary(ctx) - if err != nil { - return nil, fmt.Errorf("error getting productSummary: %w", err) - } - - result := struct { - Email string `db:"email"` - UserGroup string `db:"user_group"` - }{} - err = d.userReader.GetContext(ctx, &result, `SELECT email, COALESCE(user_group, '') as user_group FROM users WHERE id = $1`, userId) - if err != nil { - if errors.Is(err, sql.ErrNoRows) { - return nil, fmt.Errorf("%w: user not found", ErrNotFound) - } - return nil, err - } - userInfo.Email = result.Email - userInfo.UserGroup = result.UserGroup - - userInfo.Email = utils.CensorEmail(userInfo.Email) - - err = d.userReader.SelectContext(ctx, &userInfo.ApiKeys, `SELECT api_key FROM api_keys WHERE user_id = $1`, userId) - if err != nil && err != sql.ErrNoRows { - return nil, fmt.Errorf("error getting userApiKeys for user %v: %w", userId, err) - } - - premiumProduct := struct { - ProductId string `db:"product_id"` - Store string `db:"store"` - Start time.Time `db:"start"` - End time.Time `db:"end"` - }{} - err = d.userReader.GetContext(ctx, &premiumProduct, ` - SELECT - COALESCE(uas.product_id, '') AS product_id, - COALESCE(uas.store, '') AS store, - COALESCE(to_timestamp((uss.payload->>'current_period_start')::bigint),uas.created_at) AS start, - COALESCE(to_timestamp((uss.payload->>'current_period_end')::bigint),uas.expires_at) AS end - FROM users_app_subscriptions uas - LEFT JOIN users_stripe_subscriptions uss ON uss.subscription_id = uas.subscription_id - WHERE uas.user_id = $1 AND uas.active = true AND product_id IN ('orca.yearly', 'orca', 'dolphin.yearly', 'dolphin', 'guppy.yearly', 'guppy', 'whale', 'goldfish', 'plankton') - ORDER BY CASE uas.product_id - WHEN 'orca.yearly' THEN 1 - WHEN 'orca' THEN 2 - WHEN 'dolphin.yearly' THEN 3 - WHEN 'dolphin' THEN 4 - WHEN 'guppy.yearly' THEN 5 - WHEN 'guppy' THEN 6 - WHEN 'whale' THEN 7 - WHEN 'goldfish' THEN 8 - WHEN 'plankton' THEN 9 - ELSE 10 -- For any other product_id values - END, uas.id DESC - LIMIT 1`, userId) - if err != nil { - if err != sql.ErrNoRows { - return nil, fmt.Errorf("error getting premiumProduct for userId %v: %w", userId, err) - } - premiumProduct.ProductId = "premium_free" - premiumProduct.Store = "" - } - - foundProduct := false - for _, p := range productSummary.PremiumProducts { - effectiveProductId := premiumProduct.ProductId - productName := p.ProductName - switch premiumProduct.ProductId { - case "whale": - effectiveProductId = "dolphin" - productName = "Whale" - case "goldfish": - effectiveProductId = "guppy" - productName = "Goldfish" - case "plankton": - effectiveProductId = "guppy" - productName = "Plankton" - } - if p.ProductIdMonthly == effectiveProductId || p.ProductIdYearly == effectiveProductId { - userInfo.PremiumPerks = p.PremiumPerks - foundProduct = true - - store := t.ProductStoreStripe - switch premiumProduct.Store { - case "ios-appstore": - store = t.ProductStoreIosAppstore - case "android-playstore": - store = t.ProductStoreAndroidPlaystore - case "ethpool": - store = t.ProductStoreEthpool - case "manuall": - store = t.ProductStoreCustom - } - - if effectiveProductId != "premium_free" { - userInfo.Subscriptions = append(userInfo.Subscriptions, t.UserSubscription{ - ProductId: premiumProduct.ProductId, - ProductName: productName, - ProductCategory: t.ProductCategoryPremium, - ProductStore: store, - Start: premiumProduct.Start.Unix(), - End: premiumProduct.End.Unix(), - }) - } - break - } - } - if !foundProduct { - return nil, fmt.Errorf("product %s not found", premiumProduct.ProductId) - } - - premiumAddons := []struct { - PriceId string `db:"price_id"` - Start time.Time `db:"start"` - End time.Time `db:"end"` - Quantity int `db:"quantity"` - }{} - err = d.userReader.SelectContext(ctx, &premiumAddons, ` - SELECT - price_id, - to_timestamp((uss.payload->>'current_period_start')::bigint) AS start, - to_timestamp((uss.payload->>'current_period_end')::bigint) AS end, - COALESCE((uss.payload->>'quantity')::int,1) AS quantity - FROM users_stripe_subscriptions uss - INNER JOIN users u ON u.stripe_customer_id = uss.customer_id - WHERE u.id = $1 AND uss.active = true AND uss.purchase_group = 'addon'`, userId) - if err != nil { - return nil, fmt.Errorf("error getting premiumAddons for userId %v: %w", userId, err) - } - for _, addon := range premiumAddons { - foundAddon := false - for _, p := range productSummary.ExtraDashboardValidatorsPremiumAddon { - if p.StripePriceIdMonthly == addon.PriceId || p.StripePriceIdYearly == addon.PriceId { - foundAddon = true - for i := 0; i < addon.Quantity; i++ { - userInfo.PremiumPerks.ValidatorsPerDashboard += p.ExtraDashboardValidators - userInfo.Subscriptions = append(userInfo.Subscriptions, t.UserSubscription{ - ProductId: utils.PriceIdToProductId(addon.PriceId), - ProductName: p.ProductName, - ProductCategory: t.ProductCategoryPremiumAddon, - ProductStore: t.ProductStoreStripe, - Start: addon.Start.Unix(), - End: addon.End.Unix(), - }) - } - } - } - if !foundAddon { - return nil, fmt.Errorf("addon not found: %v", addon.PriceId) - } - } - - if productSummary.ValidatorsPerDashboardLimit < userInfo.PremiumPerks.ValidatorsPerDashboard { - userInfo.PremiumPerks.ValidatorsPerDashboard = productSummary.ValidatorsPerDashboardLimit - } - - if userInfo.UserGroup == t.UserGroupAdmin { - userInfo.PremiumPerks = adminPerks - } - - return userInfo, nil -} - -const hour uint64 = 3600 -const day = 24 * hour -const week = 7 * day -const month = 30 * day -const maxJsInt uint64 = 9007199254740991 // 2^53-1 (max safe int in JS) - -var freeTierProduct t.PremiumProduct = t.PremiumProduct{ - ProductName: "Free", - PremiumPerks: t.PremiumPerks{ - AdFree: false, - ValidatorDashboards: 1, - ValidatorsPerDashboard: 20, - ValidatorGroupsPerDashboard: 1, - ShareCustomDashboards: false, - ManageDashboardViaApi: false, - BulkAdding: false, - ChartHistorySeconds: t.ChartHistorySeconds{ - Epoch: 0, - Hourly: 12 * hour, - Daily: 0, - Weekly: 0, - }, - EmailNotificationsPerDay: 5, - ConfigureNotificationsViaApi: false, - ValidatorGroupNotifications: 1, - WebhookEndpoints: 1, - MobileAppCustomThemes: false, - MobileAppWidget: false, - MonitorMachines: 1, - MachineMonitoringHistorySeconds: 3600 * 3, - NotificationsMachineCustomThreshold: false, - NotificationsValidatorDashboardRealTimeMode: false, - NotificationsValidatorDashboardGroupOffline: false, - }, - PricePerMonthEur: 0, - PricePerYearEur: 0, - ProductIdMonthly: "premium_free", - ProductIdYearly: "premium_free.yearly", + return db.GetUserInfo(ctx, userId) } func (d *DataAccessService) GetProductSummary(ctx context.Context) (*t.ProductSummary, error) { - // TODO @patrick post-beta put into db instead of hardcoding here and make it configurable - return &t.ProductSummary{ - ValidatorsPerDashboardLimit: 102_000, - StripePublicKey: utils.Config.Frontend.Stripe.PublicKey, - ApiProducts: []t.ApiProduct{ // TODO @patrick post-beta this data is not final yet - { - ProductId: "api_free", - ProductName: "Free", - PricePerMonthEur: 0, - PricePerYearEur: 0 * 12, - ApiPerks: t.ApiPerks{ - UnitsPerSecond: 10, - UnitsPerMonth: 10_000_000, - ApiKeys: 2, - ConsensusLayerAPI: true, - ExecutionLayerAPI: true, - Layer2API: true, - NoAds: true, - DiscordSupport: false, - }, - }, - { - ProductId: "iron", - ProductName: "Iron", - PricePerMonthEur: 1.99, - PricePerYearEur: math.Floor(1.99*12*0.9*100) / 100, - ApiPerks: t.ApiPerks{ - UnitsPerSecond: 20, - UnitsPerMonth: 20_000_000, - ApiKeys: 10, - ConsensusLayerAPI: true, - ExecutionLayerAPI: true, - Layer2API: true, - NoAds: true, - DiscordSupport: false, - }, - }, - { - ProductId: "silver", - ProductName: "Silver", - PricePerMonthEur: 2.99, - PricePerYearEur: math.Floor(2.99*12*0.9*100) / 100, - ApiPerks: t.ApiPerks{ - UnitsPerSecond: 30, - UnitsPerMonth: 100_000_000, - ApiKeys: 20, - ConsensusLayerAPI: true, - ExecutionLayerAPI: true, - Layer2API: true, - NoAds: true, - DiscordSupport: false, - }, - }, - { - ProductId: "gold", - ProductName: "Gold", - PricePerMonthEur: 3.99, - PricePerYearEur: math.Floor(3.99*12*0.9*100) / 100, - ApiPerks: t.ApiPerks{ - UnitsPerSecond: 40, - UnitsPerMonth: 200_000_000, - ApiKeys: 40, - ConsensusLayerAPI: true, - ExecutionLayerAPI: true, - Layer2API: true, - NoAds: true, - DiscordSupport: false, - }, - }, - }, - PremiumProducts: []t.PremiumProduct{ - freeTierProduct, - { - ProductName: "Guppy", - PremiumPerks: t.PremiumPerks{ - AdFree: true, - ValidatorDashboards: 1, - ValidatorsPerDashboard: 100, - ValidatorGroupsPerDashboard: 3, - ShareCustomDashboards: true, - ManageDashboardViaApi: false, - BulkAdding: true, - ChartHistorySeconds: t.ChartHistorySeconds{ - Epoch: day, - Hourly: 7 * day, - Daily: month, - Weekly: 0, - }, - EmailNotificationsPerDay: 15, - ConfigureNotificationsViaApi: false, - ValidatorGroupNotifications: 3, - WebhookEndpoints: 3, - MobileAppCustomThemes: true, - MobileAppWidget: true, - MonitorMachines: 2, - MachineMonitoringHistorySeconds: 3600 * 24 * 30, - NotificationsMachineCustomThreshold: true, - NotificationsValidatorDashboardRealTimeMode: true, - NotificationsValidatorDashboardGroupOffline: true, - }, - PricePerMonthEur: 9.99, - PricePerYearEur: 107.88, - ProductIdMonthly: "guppy", - ProductIdYearly: "guppy.yearly", - StripePriceIdMonthly: utils.Config.Frontend.Stripe.Guppy, - StripePriceIdYearly: utils.Config.Frontend.Stripe.GuppyYearly, - }, - { - ProductName: "Dolphin", - PremiumPerks: t.PremiumPerks{ - AdFree: true, - ValidatorDashboards: 2, - ValidatorsPerDashboard: 300, - ValidatorGroupsPerDashboard: 10, - ShareCustomDashboards: true, - ManageDashboardViaApi: false, - BulkAdding: true, - ChartHistorySeconds: t.ChartHistorySeconds{ - Epoch: 5 * day, - Hourly: month, - Daily: 2 * month, - Weekly: 8 * week, - }, - EmailNotificationsPerDay: 20, - ConfigureNotificationsViaApi: false, - ValidatorGroupNotifications: 10, - WebhookEndpoints: 10, - MobileAppCustomThemes: true, - MobileAppWidget: true, - MonitorMachines: 10, - MachineMonitoringHistorySeconds: 3600 * 24 * 30, - NotificationsMachineCustomThreshold: true, - NotificationsValidatorDashboardRealTimeMode: true, - NotificationsValidatorDashboardGroupOffline: true, - }, - PricePerMonthEur: 29.99, - PricePerYearEur: 311.88, - ProductIdMonthly: "dolphin", - ProductIdYearly: "dolphin.yearly", - StripePriceIdMonthly: utils.Config.Frontend.Stripe.Dolphin, - StripePriceIdYearly: utils.Config.Frontend.Stripe.DolphinYearly, - }, - { - ProductName: "Orca", - PremiumPerks: t.PremiumPerks{ - AdFree: true, - ValidatorDashboards: 2, - ValidatorsPerDashboard: 1000, - ValidatorGroupsPerDashboard: 30, - ShareCustomDashboards: true, - ManageDashboardViaApi: true, - BulkAdding: true, - ChartHistorySeconds: t.ChartHistorySeconds{ - Epoch: 3 * week, - Hourly: 6 * month, - Daily: 12 * month, - Weekly: maxJsInt, - }, - EmailNotificationsPerDay: 50, - ConfigureNotificationsViaApi: true, - ValidatorGroupNotifications: 60, - WebhookEndpoints: 30, - MobileAppCustomThemes: true, - MobileAppWidget: true, - MonitorMachines: 10, - MachineMonitoringHistorySeconds: 3600 * 24 * 30, - NotificationsMachineCustomThreshold: true, - NotificationsValidatorDashboardRealTimeMode: true, - NotificationsValidatorDashboardGroupOffline: true, - }, - PricePerMonthEur: 49.99, - PricePerYearEur: 479.88, - ProductIdMonthly: "orca", - ProductIdYearly: "orca.yearly", - StripePriceIdMonthly: utils.Config.Frontend.Stripe.Orca, - StripePriceIdYearly: utils.Config.Frontend.Stripe.OrcaYearly, - IsPopular: true, - }, - }, - ExtraDashboardValidatorsPremiumAddon: []t.ExtraDashboardValidatorsPremiumAddon{ - { - ProductName: "1k extra valis per dashboard", - ExtraDashboardValidators: 1000, - PricePerMonthEur: 74.99, - PricePerYearEur: 719.88, - ProductIdMonthly: "vdb_addon_1k", - ProductIdYearly: "vdb_addon_1k.yearly", - StripePriceIdMonthly: utils.Config.Frontend.Stripe.VdbAddon1k, - StripePriceIdYearly: utils.Config.Frontend.Stripe.VdbAddon1kYearly, - }, - { - ProductName: "10k extra valis per dashboard", - ExtraDashboardValidators: 10000, - PricePerMonthEur: 449.99, - PricePerYearEur: 4319.88, - ProductIdMonthly: "vdb_addon_10k", - ProductIdYearly: "vdb_addon_10k.yearly", - StripePriceIdMonthly: utils.Config.Frontend.Stripe.VdbAddon10k, - StripePriceIdYearly: utils.Config.Frontend.Stripe.VdbAddon10kYearly, - }, - }, - }, nil + return db.GetProductSummary(ctx) } func (d *DataAccessService) GetFreeTierPerks(ctx context.Context) (*t.PremiumPerks, error) { - return &freeTierProduct.PremiumPerks, nil + return db.GetFreeTierPerks(ctx) } func (d *DataAccessService) GetUserDashboards(ctx context.Context, userId uint64) (*t.UserDashboardsData, error) { diff --git a/backend/pkg/commons/db/user.go b/backend/pkg/commons/db/user.go new file mode 100644 index 000000000..c877a1379 --- /dev/null +++ b/backend/pkg/commons/db/user.go @@ -0,0 +1,470 @@ +package db + +import ( + "context" + "database/sql" + "errors" + "fmt" + "math" + "time" + + t "github.com/gobitfly/beaconchain/pkg/api/types" + "github.com/gobitfly/beaconchain/pkg/commons/utils" +) + +var ErrNotFound = errors.New("not found") + +const hour uint64 = 3600 +const day = 24 * hour +const week = 7 * day +const month = 30 * day +const maxJsInt uint64 = 9007199254740991 // 2^53-1 (max safe int in JS) + +var freeTierProduct t.PremiumProduct = t.PremiumProduct{ + ProductName: "Free", + PremiumPerks: t.PremiumPerks{ + AdFree: false, + ValidatorDashboards: 1, + ValidatorsPerDashboard: 20, + ValidatorGroupsPerDashboard: 1, + ShareCustomDashboards: false, + ManageDashboardViaApi: false, + BulkAdding: false, + ChartHistorySeconds: t.ChartHistorySeconds{ + Epoch: 0, + Hourly: 12 * hour, + Daily: 0, + Weekly: 0, + }, + EmailNotificationsPerDay: 5, + ConfigureNotificationsViaApi: false, + ValidatorGroupNotifications: 1, + WebhookEndpoints: 1, + MobileAppCustomThemes: false, + MobileAppWidget: false, + MonitorMachines: 1, + MachineMonitoringHistorySeconds: 3600 * 3, + NotificationsMachineCustomThreshold: false, + NotificationsValidatorDashboardRealTimeMode: false, + NotificationsValidatorDashboardGroupOffline: false, + }, + PricePerMonthEur: 0, + PricePerYearEur: 0, + ProductIdMonthly: "premium_free", + ProductIdYearly: "premium_free.yearly", +} + +var adminPerks = t.PremiumPerks{ + AdFree: false, // admins want to see ads to check ad configuration + ValidatorDashboards: maxJsInt, + ValidatorsPerDashboard: maxJsInt, + ValidatorGroupsPerDashboard: maxJsInt, + ShareCustomDashboards: true, + ManageDashboardViaApi: true, + BulkAdding: true, + ChartHistorySeconds: t.ChartHistorySeconds{ + Epoch: maxJsInt, + Hourly: maxJsInt, + Daily: maxJsInt, + Weekly: maxJsInt, + }, + EmailNotificationsPerDay: maxJsInt, + ConfigureNotificationsViaApi: true, + ValidatorGroupNotifications: maxJsInt, + WebhookEndpoints: maxJsInt, + MobileAppCustomThemes: true, + MobileAppWidget: true, + MonitorMachines: maxJsInt, + MachineMonitoringHistorySeconds: maxJsInt, + NotificationsMachineCustomThreshold: true, + NotificationsValidatorDashboardRealTimeMode: true, + NotificationsValidatorDashboardGroupOffline: true, +} + +func GetUserInfo(ctx context.Context, userId uint64) (*t.UserInfo, error) { + // TODO @patrick post-beta improve and unmock + userInfo := &t.UserInfo{ + Id: userId, + ApiKeys: []string{}, + ApiPerks: t.ApiPerks{ + UnitsPerSecond: 10, + UnitsPerMonth: 10, + ApiKeys: 4, + ConsensusLayerAPI: true, + ExecutionLayerAPI: true, + Layer2API: true, + NoAds: true, + DiscordSupport: false, + }, + Subscriptions: []t.UserSubscription{}, + } + + productSummary, err := GetProductSummary(ctx) + if err != nil { + return nil, fmt.Errorf("error getting productSummary: %w", err) + } + + result := struct { + Email string `db:"email"` + UserGroup string `db:"user_group"` + }{} + err = FrontendReaderDB.GetContext(ctx, &result, `SELECT email, COALESCE(user_group, '') as user_group FROM users WHERE id = $1`, userId) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return nil, fmt.Errorf("%w: user not found", ErrNotFound) + } + return nil, err + } + userInfo.Email = result.Email + userInfo.UserGroup = result.UserGroup + + userInfo.Email = utils.CensorEmail(userInfo.Email) + + err = FrontendReaderDB.SelectContext(ctx, &userInfo.ApiKeys, `SELECT api_key FROM api_keys WHERE user_id = $1`, userId) + if err != nil && err != sql.ErrNoRows { + return nil, fmt.Errorf("error getting userApiKeys for user %v: %w", userId, err) + } + + premiumProduct := struct { + ProductId string `db:"product_id"` + Store string `db:"store"` + Start time.Time `db:"start"` + End time.Time `db:"end"` + }{} + err = FrontendReaderDB.GetContext(ctx, &premiumProduct, ` + SELECT + COALESCE(uas.product_id, '') AS product_id, + COALESCE(uas.store, '') AS store, + COALESCE(to_timestamp((uss.payload->>'current_period_start')::bigint),uas.created_at) AS start, + COALESCE(to_timestamp((uss.payload->>'current_period_end')::bigint),uas.expires_at) AS end + FROM users_app_subscriptions uas + LEFT JOIN users_stripe_subscriptions uss ON uss.subscription_id = uas.subscription_id + WHERE uas.user_id = $1 AND uas.active = true AND product_id IN ('orca.yearly', 'orca', 'dolphin.yearly', 'dolphin', 'guppy.yearly', 'guppy', 'whale', 'goldfish', 'plankton') + ORDER BY CASE uas.product_id + WHEN 'orca.yearly' THEN 1 + WHEN 'orca' THEN 2 + WHEN 'dolphin.yearly' THEN 3 + WHEN 'dolphin' THEN 4 + WHEN 'guppy.yearly' THEN 5 + WHEN 'guppy' THEN 6 + WHEN 'whale' THEN 7 + WHEN 'goldfish' THEN 8 + WHEN 'plankton' THEN 9 + ELSE 10 -- For any other product_id values + END, uas.id DESC + LIMIT 1`, userId) + if err != nil { + if err != sql.ErrNoRows { + return nil, fmt.Errorf("error getting premiumProduct for userId %v: %w", userId, err) + } + premiumProduct.ProductId = "premium_free" + premiumProduct.Store = "" + } + + foundProduct := false + for _, p := range productSummary.PremiumProducts { + effectiveProductId := premiumProduct.ProductId + productName := p.ProductName + switch premiumProduct.ProductId { + case "whale": + effectiveProductId = "dolphin" + productName = "Whale" + case "goldfish": + effectiveProductId = "guppy" + productName = "Goldfish" + case "plankton": + effectiveProductId = "guppy" + productName = "Plankton" + } + if p.ProductIdMonthly == effectiveProductId || p.ProductIdYearly == effectiveProductId { + userInfo.PremiumPerks = p.PremiumPerks + foundProduct = true + + store := t.ProductStoreStripe + switch premiumProduct.Store { + case "ios-appstore": + store = t.ProductStoreIosAppstore + case "android-playstore": + store = t.ProductStoreAndroidPlaystore + case "ethpool": + store = t.ProductStoreEthpool + case "manuall": + store = t.ProductStoreCustom + } + + if effectiveProductId != "premium_free" { + userInfo.Subscriptions = append(userInfo.Subscriptions, t.UserSubscription{ + ProductId: premiumProduct.ProductId, + ProductName: productName, + ProductCategory: t.ProductCategoryPremium, + ProductStore: store, + Start: premiumProduct.Start.Unix(), + End: premiumProduct.End.Unix(), + }) + } + break + } + } + if !foundProduct { + return nil, fmt.Errorf("product %s not found", premiumProduct.ProductId) + } + + premiumAddons := []struct { + PriceId string `db:"price_id"` + Start time.Time `db:"start"` + End time.Time `db:"end"` + Quantity int `db:"quantity"` + }{} + err = FrontendReaderDB.SelectContext(ctx, &premiumAddons, ` + SELECT + price_id, + to_timestamp((uss.payload->>'current_period_start')::bigint) AS start, + to_timestamp((uss.payload->>'current_period_end')::bigint) AS end, + COALESCE((uss.payload->>'quantity')::int,1) AS quantity + FROM users_stripe_subscriptions uss + INNER JOIN users u ON u.stripe_customer_id = uss.customer_id + WHERE u.id = $1 AND uss.active = true AND uss.purchase_group = 'addon'`, userId) + if err != nil { + return nil, fmt.Errorf("error getting premiumAddons for userId %v: %w", userId, err) + } + for _, addon := range premiumAddons { + foundAddon := false + for _, p := range productSummary.ExtraDashboardValidatorsPremiumAddon { + if p.StripePriceIdMonthly == addon.PriceId || p.StripePriceIdYearly == addon.PriceId { + foundAddon = true + for i := 0; i < addon.Quantity; i++ { + userInfo.PremiumPerks.ValidatorsPerDashboard += p.ExtraDashboardValidators + userInfo.Subscriptions = append(userInfo.Subscriptions, t.UserSubscription{ + ProductId: utils.PriceIdToProductId(addon.PriceId), + ProductName: p.ProductName, + ProductCategory: t.ProductCategoryPremiumAddon, + ProductStore: t.ProductStoreStripe, + Start: addon.Start.Unix(), + End: addon.End.Unix(), + }) + } + } + } + if !foundAddon { + return nil, fmt.Errorf("addon not found: %v", addon.PriceId) + } + } + + if productSummary.ValidatorsPerDashboardLimit < userInfo.PremiumPerks.ValidatorsPerDashboard { + userInfo.PremiumPerks.ValidatorsPerDashboard = productSummary.ValidatorsPerDashboardLimit + } + + if userInfo.UserGroup == t.UserGroupAdmin { + userInfo.PremiumPerks = adminPerks + } + + return userInfo, nil +} + +func GetProductSummary(ctx context.Context) (*t.ProductSummary, error) { // TODO @patrick post-beta put into db instead of hardcoding here and make it configurable + return &t.ProductSummary{ + ValidatorsPerDashboardLimit: 102_000, + StripePublicKey: utils.Config.Frontend.Stripe.PublicKey, + ApiProducts: []t.ApiProduct{ // TODO @patrick post-beta this data is not final yet + { + ProductId: "api_free", + ProductName: "Free", + PricePerMonthEur: 0, + PricePerYearEur: 0 * 12, + ApiPerks: t.ApiPerks{ + UnitsPerSecond: 10, + UnitsPerMonth: 10_000_000, + ApiKeys: 2, + ConsensusLayerAPI: true, + ExecutionLayerAPI: true, + Layer2API: true, + NoAds: true, + DiscordSupport: false, + }, + }, + { + ProductId: "iron", + ProductName: "Iron", + PricePerMonthEur: 1.99, + PricePerYearEur: math.Floor(1.99*12*0.9*100) / 100, + ApiPerks: t.ApiPerks{ + UnitsPerSecond: 20, + UnitsPerMonth: 20_000_000, + ApiKeys: 10, + ConsensusLayerAPI: true, + ExecutionLayerAPI: true, + Layer2API: true, + NoAds: true, + DiscordSupport: false, + }, + }, + { + ProductId: "silver", + ProductName: "Silver", + PricePerMonthEur: 2.99, + PricePerYearEur: math.Floor(2.99*12*0.9*100) / 100, + ApiPerks: t.ApiPerks{ + UnitsPerSecond: 30, + UnitsPerMonth: 100_000_000, + ApiKeys: 20, + ConsensusLayerAPI: true, + ExecutionLayerAPI: true, + Layer2API: true, + NoAds: true, + DiscordSupport: false, + }, + }, + { + ProductId: "gold", + ProductName: "Gold", + PricePerMonthEur: 3.99, + PricePerYearEur: math.Floor(3.99*12*0.9*100) / 100, + ApiPerks: t.ApiPerks{ + UnitsPerSecond: 40, + UnitsPerMonth: 200_000_000, + ApiKeys: 40, + ConsensusLayerAPI: true, + ExecutionLayerAPI: true, + Layer2API: true, + NoAds: true, + DiscordSupport: false, + }, + }, + }, + PremiumProducts: []t.PremiumProduct{ + freeTierProduct, + { + ProductName: "Guppy", + PremiumPerks: t.PremiumPerks{ + AdFree: true, + ValidatorDashboards: 1, + ValidatorsPerDashboard: 100, + ValidatorGroupsPerDashboard: 3, + ShareCustomDashboards: true, + ManageDashboardViaApi: false, + BulkAdding: true, + ChartHistorySeconds: t.ChartHistorySeconds{ + Epoch: day, + Hourly: 7 * day, + Daily: month, + Weekly: 0, + }, + EmailNotificationsPerDay: 15, + ConfigureNotificationsViaApi: false, + ValidatorGroupNotifications: 3, + WebhookEndpoints: 3, + MobileAppCustomThemes: true, + MobileAppWidget: true, + MonitorMachines: 2, + MachineMonitoringHistorySeconds: 3600 * 24 * 30, + NotificationsMachineCustomThreshold: true, + NotificationsValidatorDashboardRealTimeMode: true, + NotificationsValidatorDashboardGroupOffline: true, + }, + PricePerMonthEur: 9.99, + PricePerYearEur: 107.88, + ProductIdMonthly: "guppy", + ProductIdYearly: "guppy.yearly", + StripePriceIdMonthly: utils.Config.Frontend.Stripe.Guppy, + StripePriceIdYearly: utils.Config.Frontend.Stripe.GuppyYearly, + }, + { + ProductName: "Dolphin", + PremiumPerks: t.PremiumPerks{ + AdFree: true, + ValidatorDashboards: 2, + ValidatorsPerDashboard: 300, + ValidatorGroupsPerDashboard: 10, + ShareCustomDashboards: true, + ManageDashboardViaApi: false, + BulkAdding: true, + ChartHistorySeconds: t.ChartHistorySeconds{ + Epoch: 5 * day, + Hourly: month, + Daily: 2 * month, + Weekly: 8 * week, + }, + EmailNotificationsPerDay: 20, + ConfigureNotificationsViaApi: false, + ValidatorGroupNotifications: 10, + WebhookEndpoints: 10, + MobileAppCustomThemes: true, + MobileAppWidget: true, + MonitorMachines: 10, + MachineMonitoringHistorySeconds: 3600 * 24 * 30, + NotificationsMachineCustomThreshold: true, + NotificationsValidatorDashboardRealTimeMode: true, + NotificationsValidatorDashboardGroupOffline: true, + }, + PricePerMonthEur: 29.99, + PricePerYearEur: 311.88, + ProductIdMonthly: "dolphin", + ProductIdYearly: "dolphin.yearly", + StripePriceIdMonthly: utils.Config.Frontend.Stripe.Dolphin, + StripePriceIdYearly: utils.Config.Frontend.Stripe.DolphinYearly, + }, + { + ProductName: "Orca", + PremiumPerks: t.PremiumPerks{ + AdFree: true, + ValidatorDashboards: 2, + ValidatorsPerDashboard: 1000, + ValidatorGroupsPerDashboard: 30, + ShareCustomDashboards: true, + ManageDashboardViaApi: true, + BulkAdding: true, + ChartHistorySeconds: t.ChartHistorySeconds{ + Epoch: 3 * week, + Hourly: 6 * month, + Daily: 12 * month, + Weekly: maxJsInt, + }, + EmailNotificationsPerDay: 50, + ConfigureNotificationsViaApi: true, + ValidatorGroupNotifications: 60, + WebhookEndpoints: 30, + MobileAppCustomThemes: true, + MobileAppWidget: true, + MonitorMachines: 10, + MachineMonitoringHistorySeconds: 3600 * 24 * 30, + NotificationsMachineCustomThreshold: true, + NotificationsValidatorDashboardRealTimeMode: true, + NotificationsValidatorDashboardGroupOffline: true, + }, + PricePerMonthEur: 49.99, + PricePerYearEur: 479.88, + ProductIdMonthly: "orca", + ProductIdYearly: "orca.yearly", + StripePriceIdMonthly: utils.Config.Frontend.Stripe.Orca, + StripePriceIdYearly: utils.Config.Frontend.Stripe.OrcaYearly, + IsPopular: true, + }, + }, + ExtraDashboardValidatorsPremiumAddon: []t.ExtraDashboardValidatorsPremiumAddon{ + { + ProductName: "1k extra valis per dashboard", + ExtraDashboardValidators: 1000, + PricePerMonthEur: 74.99, + PricePerYearEur: 719.88, + ProductIdMonthly: "vdb_addon_1k", + ProductIdYearly: "vdb_addon_1k.yearly", + StripePriceIdMonthly: utils.Config.Frontend.Stripe.VdbAddon1k, + StripePriceIdYearly: utils.Config.Frontend.Stripe.VdbAddon1kYearly, + }, + { + ProductName: "10k extra valis per dashboard", + ExtraDashboardValidators: 10000, + PricePerMonthEur: 449.99, + PricePerYearEur: 4319.88, + ProductIdMonthly: "vdb_addon_10k", + ProductIdYearly: "vdb_addon_10k.yearly", + StripePriceIdMonthly: utils.Config.Frontend.Stripe.VdbAddon10k, + StripePriceIdYearly: utils.Config.Frontend.Stripe.VdbAddon10kYearly, + }, + }, + }, nil +} + +func GetFreeTierPerks(ctx context.Context) (*t.PremiumPerks, error) { + return &freeTierProduct.PremiumPerks, nil +} diff --git a/backend/pkg/commons/mail/mail.go b/backend/pkg/commons/mail/mail.go index e2e9d08e2..966e29e52 100644 --- a/backend/pkg/commons/mail/mail.go +++ b/backend/pkg/commons/mail/mail.go @@ -73,32 +73,35 @@ func createTextMessage(msg types.Email) string { // SendMailRateLimited sends an email to a given address with the given message. // It will return a ratelimit-error if the configured ratelimit is exceeded. func SendMailRateLimited(content types.TransitEmailContent) error { - if utils.Config.Frontend.MaxMailsPerEmailPerDay > 0 { - now := time.Now() - count, err := db.CountSentMessage("n_mails", content.UserId) + now := time.Now() + count, err := db.CountSentMessage("n_mails", content.UserId) + if err != nil { + return err + } + + userInfo, err := db.GetUserInfo(context.Background(), uint64(content.UserId)) + if err != nil { + return err + } + timeLeft := now.Add(utils.Day).Truncate(utils.Day).Sub(now) + if count > int64(userInfo.PremiumPerks.EmailNotificationsPerDay) { + return &types.RateLimitError{TimeLeft: timeLeft} + } else if count == int64(userInfo.PremiumPerks.EmailNotificationsPerDay) { + // send an email if this was the last email for today + err := SendHTMLMail(content.Address, + "beaconcha.in - Email notification threshold limit reached", + types.Email{ + Title: "Email notification threshold limit reached", + //nolint: gosec + Body: template.HTML(fmt.Sprintf("You have reached the email notification threshold limit of %d emails per day. Further notification emails will be suppressed for %.1f hours.", utils.Config.Frontend.MaxMailsPerEmailPerDay, timeLeft.Hours())), + }, + []types.EmailAttachment{}) if err != nil { return err } - timeLeft := now.Add(utils.Day).Truncate(utils.Day).Sub(now) - if count > int64(utils.Config.Frontend.MaxMailsPerEmailPerDay) { - return &types.RateLimitError{TimeLeft: timeLeft} - } else if count == int64(utils.Config.Frontend.MaxMailsPerEmailPerDay) { - // send an email if this was the last email for today - err := SendHTMLMail(content.Address, - "beaconcha.in - Email notification threshold limit reached", - types.Email{ - Title: "Email notification threshold limit reached", - //nolint: gosec - Body: template.HTML(fmt.Sprintf("You have reached the email notification threshold limit of %d emails per day. Further notification emails will be suppressed for %.1f hours.", utils.Config.Frontend.MaxMailsPerEmailPerDay, timeLeft.Hours())), - }, - []types.EmailAttachment{}) - if err != nil { - return err - } - } } - err := SendHTMLMail(content.Address, content.Subject, content.Email, content.Attachments) + err = SendHTMLMail(content.Address, content.Subject, content.Email, content.Attachments) if err != nil { return err } From f584030e43e254177ea1280234a48be839c85d3c Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Wed, 16 Oct 2024 06:27:00 +0000 Subject: [PATCH 002/124] chore(modules): make module init more flexible --- backend/cmd/exporter/main.go | 14 +++++++++++++- backend/pkg/exporter/modules/base.go | 17 ++--------------- 2 files changed, 15 insertions(+), 16 deletions(-) diff --git a/backend/cmd/exporter/main.go b/backend/cmd/exporter/main.go index 2bf0a014e..6f3cf7c72 100644 --- a/backend/cmd/exporter/main.go +++ b/backend/cmd/exporter/main.go @@ -193,7 +193,19 @@ func Run() { go services.StartHistoricPriceService() } - go modules.StartAll(context) + usedModules := []modules.ModuleInterface{} + + if cfg.JustV2 { + usedModules = append(usedModules, modules.NewDashboardDataModule(context)) + } else { + usedModules = append(usedModules, + modules.NewSlotExporter(context), + modules.NewExecutionDepositsExporter(context), + modules.NewExecutionPayloadsExporter(context), + ) + } + + go modules.StartAll(context, usedModules, cfg.JustV2) // Keep the program alive until Ctrl+C is pressed utils.WaitForCtrlC() diff --git a/backend/pkg/exporter/modules/base.go b/backend/pkg/exporter/modules/base.go index 300f44229..1dba9ed01 100644 --- a/backend/pkg/exporter/modules/base.go +++ b/backend/pkg/exporter/modules/base.go @@ -32,8 +32,8 @@ type ModuleInterface interface { var Client *rpc.Client // Start will start the export of data from rpc into the database -func StartAll(context ModuleContext) { - if !utils.Config.JustV2 { +func StartAll(context ModuleContext, modules []ModuleInterface, justV2 bool) { + if !justV2 { go networkLivenessUpdater(context.ConsClient) go genesisDepositsExporter(context.ConsClient) go syncCommitteesExporter(context.ConsClient) @@ -65,19 +65,6 @@ func StartAll(context ModuleContext) { } // start subscription modules - - modules := []ModuleInterface{} - - if utils.Config.JustV2 { - modules = append(modules, NewDashboardDataModule(context)) - } else { - modules = append(modules, - NewSlotExporter(context), - NewExecutionDepositsExporter(context), - NewExecutionPayloadsExporter(context), - ) - } - startSubscriptionModules(&context, modules) } From 10152e640b70cc37feb5ad77b7cf9015b5151626 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Wed, 16 Oct 2024 06:27:56 +0000 Subject: [PATCH 003/124] chore(notification): bump daily emails for non-paying users to 10 --- backend/pkg/commons/db/user.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/pkg/commons/db/user.go b/backend/pkg/commons/db/user.go index c877a1379..1c25ff4d6 100644 --- a/backend/pkg/commons/db/user.go +++ b/backend/pkg/commons/db/user.go @@ -36,7 +36,7 @@ var freeTierProduct t.PremiumProduct = t.PremiumProduct{ Daily: 0, Weekly: 0, }, - EmailNotificationsPerDay: 5, + EmailNotificationsPerDay: 10, ConfigureNotificationsViaApi: false, ValidatorGroupNotifications: 1, WebhookEndpoints: 1, From 160be812ec1ef2967c9363c38f65a8aa8b377ac7 Mon Sep 17 00:00:00 2001 From: Stefan Pletka <124689083+Eisei24@users.noreply.github.com> Date: Wed, 16 Oct 2024 13:06:46 +0200 Subject: [PATCH 004/124] Removed SELECT DISTINCT for past blocks --- backend/pkg/api/data_access/vdb_blocks.go | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/backend/pkg/api/data_access/vdb_blocks.go b/backend/pkg/api/data_access/vdb_blocks.go index ca881c0fb..86abd17bd 100644 --- a/backend/pkg/api/data_access/vdb_blocks.go +++ b/backend/pkg/api/data_access/vdb_blocks.go @@ -284,15 +284,8 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das ) ` - distinct := "" - if !onlyPrimarySort { - distinct = sortColName - } from := `past_blocks ` - selectStr := `SELECT * FROM ` + from - if len(distinct) > 0 { - selectStr = `SELECT DISTINCT ON (` + distinct + `) * FROM ` + from - } + selectStr := `SELECT * FROM ` query := selectStr + from + where + orderBy + limitStr // supply scheduled proposals, if any @@ -325,11 +318,11 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das `, len(params)-2) } cte += `) ` - if len(distinct) != 0 { - distinct += ", " + distinct := "slot" + if !onlyPrimarySort { + distinct = sortColName + ", " + distinct } // keep all ordering, sorting etc - distinct += "slot" selectStr = `SELECT DISTINCT ON (` + distinct + `) * FROM ` // encapsulate past blocks query to ensure performance from = `( From 27ce5534e41e7a3d2c3e6b4097676d4fc17e6989 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Wed, 16 Oct 2024 11:25:46 +0000 Subject: [PATCH 005/124] feat(notifications): add upcoming block proposal notifications --- backend/pkg/notification/collection.go | 232 ++++++++++++------------- backend/pkg/notification/db.go | 213 ++++++++++++++++------- backend/pkg/notification/types.go | 35 ++++ 3 files changed, 304 insertions(+), 176 deletions(-) diff --git a/backend/pkg/notification/collection.go b/backend/pkg/notification/collection.go index 8c6918aac..a50c21326 100644 --- a/backend/pkg/notification/collection.go +++ b/backend/pkg/notification/collection.go @@ -21,6 +21,7 @@ import ( "github.com/gobitfly/beaconchain/pkg/commons/services" "github.com/gobitfly/beaconchain/pkg/commons/types" "github.com/gobitfly/beaconchain/pkg/commons/utils" + "github.com/gobitfly/beaconchain/pkg/exporter/modules" "github.com/lib/pq" "github.com/rocket-pool/rocketpool-go/utils/eth" ) @@ -185,126 +186,52 @@ func collectNotifications(epoch uint64) (types.NotificationsPerUserId, error) { } log.Infof("started collecting notifications") - - log.Infof("retrieving dashboard definitions") - // Retrieve all dashboard definitions to be able to retrieve validators included in - // the group notification subscriptions - // TODO: add a filter to retrieve only groups that have notifications enabled - // Needs a new field in the db - dashboardConfigRetrievalStartTs := time.Now() - type dashboardDefinitionRow struct { - DashboardId types.DashboardId `db:"dashboard_id"` - DashboardName string `db:"dashboard_name"` - UserId types.UserId `db:"user_id"` - GroupId types.DashboardGroupId `db:"group_id"` - GroupName string `db:"group_name"` - ValidatorIndex types.ValidatorIndex `db:"validator_index"` - WebhookTarget string `db:"webhook_target"` - WebhookFormat string `db:"webhook_format"` - } - var dashboardDefinitions []dashboardDefinitionRow - err = db.AlloyWriter.Select(&dashboardDefinitions, ` - SELECT - users_val_dashboards.id as dashboard_id, - users_val_dashboards.name as dashboard_name, - users_val_dashboards.user_id, - users_val_dashboards_groups.id as group_id, - users_val_dashboards_groups.name as group_name, - users_val_dashboards_validators.validator_index, - COALESCE(users_val_dashboards_groups.webhook_target, '') AS webhook_target, - COALESCE(users_val_dashboards_groups.webhook_format, '') AS webhook_format - FROM users_val_dashboards - LEFT JOIN users_val_dashboards_groups ON users_val_dashboards_groups.dashboard_id = users_val_dashboards.id - LEFT JOIN users_val_dashboards_validators ON users_val_dashboards_validators.dashboard_id = users_val_dashboards_groups.dashboard_id AND users_val_dashboards_validators.group_id = users_val_dashboards_groups.id - WHERE users_val_dashboards_validators.validator_index IS NOT NULL; - `) - if err != nil { - return nil, fmt.Errorf("error getting dashboard definitions: %v", err) - } - - // Now initialize the validator dashboard configuration map - validatorDashboardConfig := &types.ValidatorDashboardConfig{ - DashboardsById: make(map[types.DashboardId]*types.ValidatorDashboard), - RocketpoolNodeByPubkey: make(map[string]string), - } - for _, row := range dashboardDefinitions { - if validatorDashboardConfig.DashboardsById[row.DashboardId] == nil { - validatorDashboardConfig.DashboardsById[row.DashboardId] = &types.ValidatorDashboard{ - Name: row.DashboardName, - Groups: make(map[types.DashboardGroupId]*types.ValidatorDashboardGroup), - } - } - if validatorDashboardConfig.DashboardsById[row.DashboardId].Groups[row.GroupId] == nil { - validatorDashboardConfig.DashboardsById[row.DashboardId].Groups[row.GroupId] = &types.ValidatorDashboardGroup{ - Name: row.GroupName, - Validators: []uint64{}, - } - } - validatorDashboardConfig.DashboardsById[row.DashboardId].Groups[row.GroupId].Validators = append(validatorDashboardConfig.DashboardsById[row.DashboardId].Groups[row.GroupId].Validators, uint64(row.ValidatorIndex)) - } - - log.Infof("retrieving dashboard definitions took: %v", time.Since(dashboardConfigRetrievalStartTs)) - - // Now collect the mapping of rocketpool node addresses to validator pubkeys - // This is needed for the rocketpool notifications - type rocketpoolNodeRow struct { - Pubkey []byte `db:"pubkey"` - NodeAddress []byte `db:"node_address"` - } - - var rocketpoolNodes []rocketpoolNodeRow - err = db.AlloyWriter.Select(&rocketpoolNodes, ` - SELECT - pubkey, - node_address - FROM rocketpool_minipools;`) - if err != nil { - return nil, fmt.Errorf("error getting rocketpool node addresses: %v", err) - } - - for _, row := range rocketpoolNodes { - validatorDashboardConfig.RocketpoolNodeByPubkey[hex.EncodeToString(row.Pubkey)] = hex.EncodeToString(row.NodeAddress) - } - // The following functions will collect the notifications and add them to the // notificationsByUserID map. The notifications will be queued and sent later // by the notification sender process - err = collectAttestationAndOfflineValidatorNotifications(notificationsByUserID, epoch, validatorDashboardConfig) + err = collectUpcomingBlockProposalNotifications(notificationsByUserID) + if err != nil { + metrics.Errors.WithLabelValues("notifications_collect_upcoming_block_proposal").Inc() + return nil, fmt.Errorf("error collecting upcoming block proposal notifications: %v", err) + } + log.Infof("collecting attestation & offline notifications took: %v", time.Since(start)) + + err = collectAttestationAndOfflineValidatorNotifications(notificationsByUserID, epoch) if err != nil { metrics.Errors.WithLabelValues("notifications_collect_missed_attestation").Inc() return nil, fmt.Errorf("error collecting validator_attestation_missed notifications: %v", err) } log.Infof("collecting attestation & offline notifications took: %v", time.Since(start)) - err = collectBlockProposalNotifications(notificationsByUserID, 1, types.ValidatorExecutedProposalEventName, epoch, validatorDashboardConfig) + err = collectBlockProposalNotifications(notificationsByUserID, 1, types.ValidatorExecutedProposalEventName, epoch) if err != nil { metrics.Errors.WithLabelValues("notifications_collect_executed_block_proposal").Inc() return nil, fmt.Errorf("error collecting validator_proposal_submitted notifications: %v", err) } log.Infof("collecting block proposal proposed notifications took: %v", time.Since(start)) - err = collectBlockProposalNotifications(notificationsByUserID, 2, types.ValidatorMissedProposalEventName, epoch, validatorDashboardConfig) + err = collectBlockProposalNotifications(notificationsByUserID, 2, types.ValidatorMissedProposalEventName, epoch) if err != nil { metrics.Errors.WithLabelValues("notifications_collect_missed_block_proposal").Inc() return nil, fmt.Errorf("error collecting validator_proposal_missed notifications: %v", err) } log.Infof("collecting block proposal missed notifications took: %v", time.Since(start)) - err = collectBlockProposalNotifications(notificationsByUserID, 3, types.ValidatorMissedProposalEventName, epoch, validatorDashboardConfig) + err = collectBlockProposalNotifications(notificationsByUserID, 3, types.ValidatorMissedProposalEventName, epoch) if err != nil { metrics.Errors.WithLabelValues("notifications_collect_missed_orphaned_block_proposal").Inc() return nil, fmt.Errorf("error collecting validator_proposal_missed notifications for orphaned slots: %w", err) } log.Infof("collecting block proposal missed notifications for orphaned slots took: %v", time.Since(start)) - err = collectValidatorGotSlashedNotifications(notificationsByUserID, epoch, validatorDashboardConfig) + err = collectValidatorGotSlashedNotifications(notificationsByUserID, epoch) if err != nil { metrics.Errors.WithLabelValues("notifications_collect_validator_got_slashed").Inc() return nil, fmt.Errorf("error collecting validator_got_slashed notifications: %v", err) } log.Infof("collecting validator got slashed notifications took: %v", time.Since(start)) - err = collectWithdrawalNotifications(notificationsByUserID, epoch, validatorDashboardConfig) + err = collectWithdrawalNotifications(notificationsByUserID, epoch) if err != nil { metrics.Errors.WithLabelValues("notifications_collect_validator_withdrawal").Inc() return nil, fmt.Errorf("error collecting withdrawal notifications: %v", err) @@ -331,7 +258,7 @@ func collectNotifications(epoch uint64) (types.NotificationsPerUserId, error) { return nil, fmt.Errorf("error collecting rocketpool notifications: %v", err) } } else { - err = collectRocketpoolComissionNotifications(notificationsByUserID, validatorDashboardConfig) + err = collectRocketpoolComissionNotifications(notificationsByUserID) if err != nil { //nolint:misspell metrics.Errors.WithLabelValues("notifications_collect_rocketpool_comission").Inc() @@ -339,21 +266,21 @@ func collectNotifications(epoch uint64) (types.NotificationsPerUserId, error) { } log.Infof("collecting rocketpool commissions took: %v", time.Since(start)) - err = collectRocketpoolRewardClaimRoundNotifications(notificationsByUserID, validatorDashboardConfig) + err = collectRocketpoolRewardClaimRoundNotifications(notificationsByUserID) if err != nil { metrics.Errors.WithLabelValues("notifications_collect_rocketpool_reward_claim").Inc() return nil, fmt.Errorf("error collecting new rocketpool claim round: %v", err) } log.Infof("collecting rocketpool claim round took: %v", time.Since(start)) - err = collectRocketpoolRPLCollateralNotifications(notificationsByUserID, types.RocketpoolCollateralMaxReachedEventName, epoch, validatorDashboardConfig) + err = collectRocketpoolRPLCollateralNotifications(notificationsByUserID, types.RocketpoolCollateralMaxReachedEventName, epoch) if err != nil { metrics.Errors.WithLabelValues("notifications_collect_rocketpool_rpl_collateral_max_reached").Inc() return nil, fmt.Errorf("error collecting rocketpool max collateral: %v", err) } log.Infof("collecting rocketpool max collateral took: %v", time.Since(start)) - err = collectRocketpoolRPLCollateralNotifications(notificationsByUserID, types.RocketpoolCollateralMinReachedEventName, epoch, validatorDashboardConfig) + err = collectRocketpoolRPLCollateralNotifications(notificationsByUserID, types.RocketpoolCollateralMinReachedEventName, epoch) if err != nil { metrics.Errors.WithLabelValues("notifications_collect_rocketpool_rpl_collateral_min_reached").Inc() return nil, fmt.Errorf("error collecting rocketpool min collateral: %v", err) @@ -362,7 +289,7 @@ func collectNotifications(epoch uint64) (types.NotificationsPerUserId, error) { } } - err = collectSyncCommitteeNotifications(notificationsByUserID, epoch, validatorDashboardConfig) + err = collectSyncCommitteeNotifications(notificationsByUserID, epoch) if err != nil { metrics.Errors.WithLabelValues("notifications_collect_sync_committee").Inc() return nil, fmt.Errorf("error collecting sync committee: %v", err) @@ -421,7 +348,85 @@ func collectUserDbNotifications(epoch uint64) (types.NotificationsPerUserId, err return notificationsByUserID, nil } -func collectBlockProposalNotifications(notificationsByUserID types.NotificationsPerUserId, status uint64, eventName types.EventName, epoch uint64, validatorDashboardConfig *types.ValidatorDashboardConfig) error { +func collectUpcomingBlockProposalNotifications(notificationsByUserID types.NotificationsPerUserId) (err error) { + mc, err := modules.GetModuleContext() + if err != nil { + return fmt.Errorf("error getting module context: %w", err) + } + + // get the head epoch + head, err := mc.ConsClient.GetChainHead() + if err != nil { + return fmt.Errorf("error getting chain head: %w", err) + } + + headEpoch := head.HeadEpoch + nextEpoch := headEpoch + 1 + + var lastNotifiedEpoch uint64 + err = db.WriterDb.Get(&lastNotifiedEpoch, "SELECT COUNT(*) FROM epochs_notified_head WHERE epoch = $1 AND event_name = $2", nextEpoch, types.ValidatorUpcomingProposalEventName) + + if err != nil { + return fmt.Errorf("error checking if upcoming block proposal notifications for epoch %v have already been collected: %w", nextEpoch, err) + } + + if lastNotifiedEpoch > 0 { + log.Error(fmt.Errorf("upcoming block proposal notifications for epoch %v have already been collected", nextEpoch), "", 0) + return nil + } + + // todo: make sure not to collect notifications for the same epoch twice + assignments, err := mc.CL.GetPropoalAssignments(nextEpoch) + if err != nil { + return fmt.Errorf("error getting proposal assignments: %w", err) + } + + subs, err := GetSubsForEventFilter(types.ValidatorUpcomingProposalEventName, "", nil, nil) + if err != nil { + return fmt.Errorf("error getting subscriptions for upcoming block proposal notifications: %w", err) + } + + log.Infof("retrieved %d subscriptions for upcoming block proposal notifications", len(subs)) + if len(subs) == 0 { + return nil + } + + for _, assignment := range assignments.Data { + log.Infof("upcoming block proposal for validator %d in slot %d", assignment.ValidatorIndex, assignment.Slot) + for _, sub := range subs[hex.EncodeToString(assignment.Pubkey)] { + if sub.UserID == nil || sub.ID == nil { + return fmt.Errorf("error expected userId and subId to be defined but got user: %v, sub: %v", sub.UserID, sub.ID) + } + + log.Infof("creating %v notification for validator %v in epoch %v (dashboard: %v)", sub.EventName, assignment.ValidatorIndex, nextEpoch, sub.DashboardId != nil) + n := &ValidatorUpcomingProposalNotification{ + NotificationBaseImpl: types.NotificationBaseImpl{ + SubscriptionID: *sub.ID, + UserID: *sub.UserID, + Epoch: nextEpoch, + EventName: sub.EventName, + EventFilter: hex.EncodeToString(assignment.Pubkey), + DashboardId: sub.DashboardId, + DashboardName: sub.DashboardName, + DashboardGroupId: sub.DashboardGroupId, + DashboardGroupName: sub.DashboardGroupName, + }, + ValidatorIndex: assignment.ValidatorIndex, + Slot: uint64(assignment.Slot), + } + notificationsByUserID.AddNotification(n) + metrics.NotificationsCollected.WithLabelValues(string(n.GetEventName())).Inc() + } + } + + _, err = db.WriterDb.Exec("INSERT INTO epochs_notified_head (epoch, event_name) VALUES ($1, $2)", nextEpoch, types.ValidatorUpcomingProposalEventName) + if err != nil { + return fmt.Errorf("error marking notification status for epoch %v in db: %w", nextEpoch, err) + } + return nil +} + +func collectBlockProposalNotifications(notificationsByUserID types.NotificationsPerUserId, status uint64, eventName types.EventName, epoch uint64) error { type dbResult struct { Proposer uint64 `db:"proposer"` Status uint64 `db:"status"` @@ -430,7 +435,7 @@ func collectBlockProposalNotifications(notificationsByUserID types.Notifications ExecRewardETH float64 } - subMap, err := GetSubsForEventFilter(eventName, "", nil, nil, validatorDashboardConfig) + subMap, err := GetSubsForEventFilter(eventName, "", nil, nil) if err != nil { return fmt.Errorf("error getting subscriptions for (missed) block proposals %w", err) } @@ -529,9 +534,9 @@ func collectBlockProposalNotifications(notificationsByUserID types.Notifications } // collectAttestationAndOfflineValidatorNotifications collects notifications for missed attestations and offline validators -func collectAttestationAndOfflineValidatorNotifications(notificationsByUserID types.NotificationsPerUserId, epoch uint64, validatorDashboardConfig *types.ValidatorDashboardConfig) error { +func collectAttestationAndOfflineValidatorNotifications(notificationsByUserID types.NotificationsPerUserId, epoch uint64) error { // Retrieve subscriptions for missed attestations - subMapAttestationMissed, err := GetSubsForEventFilter(types.ValidatorMissedAttestationEventName, "", nil, nil, validatorDashboardConfig) + subMapAttestationMissed, err := GetSubsForEventFilter(types.ValidatorMissedAttestationEventName, "", nil, nil) if err != nil { return fmt.Errorf("error getting subscriptions for missted attestations %w", err) } @@ -703,7 +708,7 @@ func collectAttestationAndOfflineValidatorNotifications(notificationsByUserID ty return fmt.Errorf("retrieved more than %v online validators notifications: %v, exiting", onlineValidatorsLimit, len(onlineValidators)) } - subMapOnlineOffline, err := GetSubsForEventFilter(types.ValidatorIsOfflineEventName, "", nil, nil, validatorDashboardConfig) + subMapOnlineOffline, err := GetSubsForEventFilter(types.ValidatorIsOfflineEventName, "", nil, nil) if err != nil { return fmt.Errorf("failed to get subs for %v: %v", types.ValidatorIsOfflineEventName, err) } @@ -843,7 +848,7 @@ func collectAttestationAndOfflineValidatorNotifications(notificationsByUserID ty return nil } -func collectValidatorGotSlashedNotifications(notificationsByUserID types.NotificationsPerUserId, epoch uint64, validatorDashboardConfig *types.ValidatorDashboardConfig) error { +func collectValidatorGotSlashedNotifications(notificationsByUserID types.NotificationsPerUserId, epoch uint64) error { dbResult, err := db.GetValidatorsGotSlashed(epoch) if err != nil { return fmt.Errorf("error getting slashed validators from database, err: %w", err) @@ -854,7 +859,7 @@ func collectValidatorGotSlashedNotifications(notificationsByUserID types.Notific pubkeyToSlashingInfoMap[pubkeyStr] = event } - subscribedUsers, err := GetSubsForEventFilter(types.ValidatorGotSlashedEventName, "", nil, nil, validatorDashboardConfig) + subscribedUsers, err := GetSubsForEventFilter(types.ValidatorGotSlashedEventName, "", nil, nil) if err != nil { return fmt.Errorf("failed to get subs for %v: %v", types.ValidatorGotSlashedEventName, err) } @@ -893,9 +898,9 @@ func collectValidatorGotSlashedNotifications(notificationsByUserID types.Notific } // collectWithdrawalNotifications collects all notifications validator withdrawals -func collectWithdrawalNotifications(notificationsByUserID types.NotificationsPerUserId, epoch uint64, validatorDashboardConfig *types.ValidatorDashboardConfig) error { +func collectWithdrawalNotifications(notificationsByUserID types.NotificationsPerUserId, epoch uint64) error { // get all users that are subscribed to this event (scale: a few thousand rows depending on how many users we have) - subMap, err := GetSubsForEventFilter(types.ValidatorReceivedWithdrawalEventName, "", nil, nil, validatorDashboardConfig) + subMap, err := GetSubsForEventFilter(types.ValidatorReceivedWithdrawalEventName, "", nil, nil) if err != nil { return fmt.Errorf("error getting subscriptions for missed attestations %w", err) } @@ -966,8 +971,7 @@ func collectEthClientNotifications(notificationsByUserID types.NotificationsPerU types.EthClientUpdateEventName, "((last_sent_ts <= NOW() - INTERVAL '2 DAY' AND TO_TIMESTAMP(?) > last_sent_ts) OR last_sent_ts IS NULL)", []interface{}{client.Date.Unix()}, - []string{strings.ToLower(client.Name)}, - nil) + []string{strings.ToLower(client.Name)}) if err != nil { return err } @@ -1086,7 +1090,6 @@ func collectMonitoringMachine( "(created_epoch <= ? AND (last_sent_epoch < ? OR last_sent_epoch IS NULL))", []interface{}{epoch, int64(epoch) - int64(epochWaitInBetween)}, nil, - nil, ) // TODO: clarify why we need grouping here?! @@ -1134,8 +1137,8 @@ func collectMonitoringMachine( } //logrus.Infof("currentMachineData %v | %v | %v | %v", currentMachine.CurrentDataInsertTs, currentMachine.CompareDataInsertTs, currentMachine.UserID, currentMachine.Machine) - if notifyConditionFulfilled(&localData, currentMachineData) { - result = append(result, &localData) + if notifyConditionFulfilled(localData, currentMachineData) { + result = append(result, localData) } } } @@ -1234,7 +1237,6 @@ func collectTaxReportNotificationNotifications(notificationsByUserID types.Notif "(last_sent_ts < ? OR (last_sent_ts IS NULL AND created_ts < ?))", []interface{}{firstDayOfMonth, firstDayOfMonth}, nil, - nil, ) if err != nil { return err @@ -1286,7 +1288,6 @@ func collectNetworkNotifications(notificationsByUserID types.NotificationsPerUse "(last_sent_ts <= NOW() - INTERVAL '1 hour' OR last_sent_ts IS NULL)", nil, nil, - nil, ) if err != nil { return err @@ -1317,7 +1318,7 @@ func collectNetworkNotifications(notificationsByUserID types.NotificationsPerUse return nil } -func collectRocketpoolComissionNotifications(notificationsByUserID types.NotificationsPerUserId, validatorDashboardConfig *types.ValidatorDashboardConfig) error { +func collectRocketpoolComissionNotifications(notificationsByUserID types.NotificationsPerUserId) error { fee := 0.0 err := db.WriterDb.Get(&fee, ` select current_node_fee from rocketpool_network_stats order by id desc LIMIT 1; @@ -1340,7 +1341,6 @@ func collectRocketpoolComissionNotifications(notificationsByUserID types.Notific "(last_sent_ts <= NOW() - INTERVAL '8 hours' OR last_sent_ts IS NULL) AND (event_threshold <= ? OR (event_threshold < 0 AND event_threshold * -1 >= ?))", []interface{}{fee, fee}, nil, - validatorDashboardConfig, ) if err != nil { return err @@ -1372,7 +1372,7 @@ func collectRocketpoolComissionNotifications(notificationsByUserID types.Notific return nil } -func collectRocketpoolRewardClaimRoundNotifications(notificationsByUserID types.NotificationsPerUserId, validatorDashboardConfig *types.ValidatorDashboardConfig) error { +func collectRocketpoolRewardClaimRoundNotifications(notificationsByUserID types.NotificationsPerUserId) error { var ts int64 err := db.WriterDb.Get(&ts, ` select date_part('epoch', claim_interval_time_start)::int from rocketpool_network_stats order by id desc LIMIT 1; @@ -1397,7 +1397,6 @@ func collectRocketpoolRewardClaimRoundNotifications(notificationsByUserID types. "(last_sent_ts <= NOW() - INTERVAL '5 hours' OR last_sent_ts IS NULL)", nil, nil, - validatorDashboardConfig, ) if err != nil { return err @@ -1428,13 +1427,12 @@ func collectRocketpoolRewardClaimRoundNotifications(notificationsByUserID types. return nil } -func collectRocketpoolRPLCollateralNotifications(notificationsByUserID types.NotificationsPerUserId, eventName types.EventName, epoch uint64, validatorDashboardConfig *types.ValidatorDashboardConfig) error { +func collectRocketpoolRPLCollateralNotifications(notificationsByUserID types.NotificationsPerUserId, eventName types.EventName, epoch uint64) error { subMap, err := GetSubsForEventFilter( eventName, "(last_sent_ts <= NOW() - INTERVAL '24 hours' OR last_sent_ts IS NULL)", // send out this notification type only once per day nil, - nil, - validatorDashboardConfig) + nil) if err != nil { return fmt.Errorf("error getting subscriptions for RocketpoolRPLCollateral %w", err) } @@ -1570,7 +1568,7 @@ func collectRocketpoolRPLCollateralNotifications(notificationsByUserID types.Not return nil } -func collectSyncCommitteeNotifications(notificationsByUserID types.NotificationsPerUserId, epoch uint64, validatorDashboardConfig *types.ValidatorDashboardConfig) error { +func collectSyncCommitteeNotifications(notificationsByUserID types.NotificationsPerUserId, epoch uint64) error { slotsPerSyncCommittee := utils.SlotsPerSyncCommittee() currentPeriod := epoch * utils.Config.Chain.ClConfig.SlotsPerEpoch / slotsPerSyncCommittee nextPeriod := currentPeriod + 1 @@ -1594,7 +1592,7 @@ func collectSyncCommitteeNotifications(notificationsByUserID types.Notifications mapping[val.PubKey] = val.Index } - dbResult, err := GetSubsForEventFilter(types.SyncCommitteeSoonEventName, "(last_sent_ts <= NOW() - INTERVAL '26 hours' OR last_sent_ts IS NULL)", nil, nil, validatorDashboardConfig) + dbResult, err := GetSubsForEventFilter(types.SyncCommitteeSoonEventName, "(last_sent_ts <= NOW() - INTERVAL '26 hours' OR last_sent_ts IS NULL)", nil, nil) if err != nil { return err diff --git a/backend/pkg/notification/db.go b/backend/pkg/notification/db.go index 81b58bf16..4642d8373 100644 --- a/backend/pkg/notification/db.go +++ b/backend/pkg/notification/db.go @@ -5,6 +5,7 @@ import ( "fmt" "strconv" "strings" + "time" "github.com/doug-martin/goqu/v9" "github.com/gobitfly/beaconchain/pkg/commons/db" @@ -21,8 +22,8 @@ import ( // or a machine name for machine notifications or a eth client name for ethereum client update notifications // optionally it is possible to set a filter on the last sent ts and the event filter // fields -func GetSubsForEventFilter(eventName types.EventName, lastSentFilter string, lastSentFilterArgs []interface{}, eventFilters []string, validatorDashboardConfig *types.ValidatorDashboardConfig) (map[string][]types.Subscription, error) { - var subs []types.Subscription +func GetSubsForEventFilter(eventName types.EventName, lastSentFilter string, lastSentFilterArgs []interface{}, eventFilters []string) (map[string][]*types.Subscription, error) { + var subs []*types.Subscription // subQuery := ` // SELECT @@ -66,12 +67,15 @@ func GetSubsForEventFilter(eventName types.EventName, lastSentFilter string, las ds = ds.Where(goqu.L("(event_filter = ANY(?))", pq.StringArray(eventFilters))) } - query, args, err := ds.Prepared(true).ToSQL() + query, args, err := ds.Prepared(false).ToSQL() if err != nil { return nil, err } - subMap := make(map[string][]types.Subscription, 0) + log.Info(query) + log.Info(args) + + subMap := make(map[string][]*types.Subscription, 0) err = db.FrontendWriterDB.Select(&subs, query, args...) if err != nil { return nil, err @@ -79,6 +83,7 @@ func GetSubsForEventFilter(eventName types.EventName, lastSentFilter string, las log.Infof("found %d subscriptions for event %s", len(subs), eventName) + dashboardConfigsToFetch := make([]types.DashboardId, 0) for _, sub := range subs { sub.EventName = types.EventName(strings.Replace(string(sub.EventName), utils.GetNetwork()+":", "", 1)) // remove the network name from the event name if strings.HasPrefix(sub.EventFilter, "vdb:") { @@ -100,42 +105,158 @@ func GetSubsForEventFilter(eventName types.EventName, lastSentFilter string, las continue } sub.DashboardGroupId = &dashboardGroupId - if dashboard, ok := validatorDashboardConfig.DashboardsById[types.DashboardId(dashboardId)]; ok { - if dashboard.Name == "" { - dashboard.Name = fmt.Sprintf("Dashboard %d", dashboardId) + + dashboardConfigsToFetch = append(dashboardConfigsToFetch, types.DashboardId(dashboardId)) + } else { + if _, ok := subMap[sub.EventFilter]; !ok { + subMap[sub.EventFilter] = make([]*types.Subscription, 0) + } + subMap[sub.EventFilter] = append(subMap[sub.EventFilter], sub) + } + } + + if len(dashboardConfigsToFetch) > 0 { + log.Infof("fetching dashboard configurations for %d dashboards", len(dashboardConfigsToFetch)) + dashboardConfigRetrievalStartTs := time.Now() + type dashboardDefinitionRow struct { + DashboardId types.DashboardId `db:"dashboard_id"` + DashboardName string `db:"dashboard_name"` + UserId types.UserId `db:"user_id"` + GroupId types.DashboardGroupId `db:"group_id"` + GroupName string `db:"group_name"` + ValidatorIndex types.ValidatorIndex `db:"validator_index"` + WebhookTarget string `db:"webhook_target"` + WebhookFormat string `db:"webhook_format"` + } + var dashboardDefinitions []dashboardDefinitionRow + err = db.AlloyWriter.Select(&dashboardDefinitions, ` + SELECT + users_val_dashboards.id as dashboard_id, + users_val_dashboards.name as dashboard_name, + users_val_dashboards.user_id, + users_val_dashboards_groups.id as group_id, + users_val_dashboards_groups.name as group_name, + users_val_dashboards_validators.validator_index, + COALESCE(users_val_dashboards_groups.webhook_target, '') AS webhook_target, + COALESCE(users_val_dashboards_groups.webhook_format, '') AS webhook_format + FROM users_val_dashboards + LEFT JOIN users_val_dashboards_groups ON users_val_dashboards_groups.dashboard_id = users_val_dashboards.id + LEFT JOIN users_val_dashboards_validators ON users_val_dashboards_validators.dashboard_id = users_val_dashboards_groups.dashboard_id AND users_val_dashboards_validators.group_id = users_val_dashboards_groups.id + WHERE users_val_dashboards_validators.validator_index IS NOT NULL AND users_val_dashboards.id = ANY($1) + `, pq.Array(dashboardConfigsToFetch)) + if err != nil { + return nil, fmt.Errorf("error getting dashboard definitions: %v", err) + } + log.Infof("retrieved %d dashboard definitions", len(dashboardDefinitions)) + + // Now initialize the validator dashboard configuration map + validatorDashboardConfig := &types.ValidatorDashboardConfig{ + DashboardsById: make(map[types.DashboardId]*types.ValidatorDashboard), + RocketpoolNodeByPubkey: make(map[string]string), + } + for _, row := range dashboardDefinitions { + if validatorDashboardConfig.DashboardsById[row.DashboardId] == nil { + validatorDashboardConfig.DashboardsById[row.DashboardId] = &types.ValidatorDashboard{ + Name: row.DashboardName, + Groups: make(map[types.DashboardGroupId]*types.ValidatorDashboardGroup), } - if group, ok := dashboard.Groups[types.DashboardGroupId(dashboardGroupId)]; ok { - if group.Name == "" { - group.Name = "default" - } + } + if validatorDashboardConfig.DashboardsById[row.DashboardId].Groups[row.GroupId] == nil { + validatorDashboardConfig.DashboardsById[row.DashboardId].Groups[row.GroupId] = &types.ValidatorDashboardGroup{ + Name: row.GroupName, + Validators: []uint64{}, + } + } + validatorDashboardConfig.DashboardsById[row.DashboardId].Groups[row.GroupId].Validators = append(validatorDashboardConfig.DashboardsById[row.DashboardId].Groups[row.GroupId].Validators, uint64(row.ValidatorIndex)) + } + + log.Infof("retrieving dashboard definitions took: %v", time.Since(dashboardConfigRetrievalStartTs)) - uniqueRPLNodes := make(map[string]struct{}) + // Now collect the mapping of rocketpool node addresses to validator pubkeys + // This is needed for the rocketpool notifications + type rocketpoolNodeRow struct { + Pubkey []byte `db:"pubkey"` + NodeAddress []byte `db:"node_address"` + } + + var rocketpoolNodes []rocketpoolNodeRow + err = db.AlloyWriter.Select(&rocketpoolNodes, ` + SELECT + pubkey, + node_address + FROM rocketpool_minipools;`) + if err != nil { + return nil, fmt.Errorf("error getting rocketpool node addresses: %v", err) + } + + for _, row := range rocketpoolNodes { + validatorDashboardConfig.RocketpoolNodeByPubkey[hex.EncodeToString(row.Pubkey)] = hex.EncodeToString(row.NodeAddress) + } - for _, validatorIndex := range group.Validators { - validatorEventFilterRaw, err := GetPubkeyForIndex(validatorIndex) - if err != nil { - log.Error(err, "error retrieving pubkey for validator", 0, map[string]interface{}{"validator": validatorIndex}) - continue + //log.Infof("retrieved %d rocketpool node addresses", len(rocketpoolNodes)) + + for _, sub := range subs { + if strings.HasPrefix(sub.EventFilter, "vdb:") { + //log.Infof("hydrating subscription for dashboard %d and group %d for user %d", *sub.DashboardId, *sub.DashboardGroupId, *sub.UserID) + if dashboard, ok := validatorDashboardConfig.DashboardsById[types.DashboardId(*sub.DashboardId)]; ok { + if dashboard.Name == "" { + dashboard.Name = fmt.Sprintf("Dashboard %d", *sub.DashboardId) + } + if group, ok := dashboard.Groups[types.DashboardGroupId(*sub.DashboardGroupId)]; ok { + if group.Name == "" { + group.Name = "default" } - validatorEventFilter := hex.EncodeToString(validatorEventFilterRaw) - - if eventName == types.RocketpoolCollateralMaxReachedEventName || eventName == types.RocketpoolCollateralMinReachedEventName { - // Those two RPL notifications are not tied to a specific validator but to a node address, create a subscription for each - // node in the group - nodeAddress, ok := validatorDashboardConfig.RocketpoolNodeByPubkey[validatorEventFilter] - if !ok { - // Validator is not a rocketpool minipool + + uniqueRPLNodes := make(map[string]struct{}) + + for _, validatorIndex := range group.Validators { + validatorEventFilterRaw, err := GetPubkeyForIndex(validatorIndex) + if err != nil { + log.Error(err, "error retrieving pubkey for validator", 0, map[string]interface{}{"validator": validatorIndex}) continue } - if _, ok := uniqueRPLNodes[nodeAddress]; !ok { - if _, ok := subMap[nodeAddress]; !ok { - subMap[nodeAddress] = make([]types.Subscription, 0) + validatorEventFilter := hex.EncodeToString(validatorEventFilterRaw) + + if eventName == types.RocketpoolCollateralMaxReachedEventName || eventName == types.RocketpoolCollateralMinReachedEventName { + // Those two RPL notifications are not tied to a specific validator but to a node address, create a subscription for each + // node in the group + nodeAddress, ok := validatorDashboardConfig.RocketpoolNodeByPubkey[validatorEventFilter] + if !ok { + // Validator is not a rocketpool minipool + continue } - hydratedSub := types.Subscription{ + if _, ok := uniqueRPLNodes[nodeAddress]; !ok { + if _, ok := subMap[nodeAddress]; !ok { + subMap[nodeAddress] = make([]*types.Subscription, 0) + } + hydratedSub := &types.Subscription{ + ID: sub.ID, + UserID: sub.UserID, + EventName: sub.EventName, + EventFilter: nodeAddress, + LastSent: sub.LastSent, + LastEpoch: sub.LastEpoch, + CreatedTime: sub.CreatedTime, + CreatedEpoch: sub.CreatedEpoch, + EventThreshold: sub.EventThreshold, + DashboardId: sub.DashboardId, + DashboardName: dashboard.Name, + DashboardGroupId: sub.DashboardGroupId, + DashboardGroupName: group.Name, + } + subMap[nodeAddress] = append(subMap[nodeAddress], hydratedSub) + //log.Infof("hydrated subscription for validator %v of dashboard %d and group %d for user %d", hydratedSub.EventFilter, *hydratedSub.DashboardId, *hydratedSub.DashboardGroupId, *hydratedSub.UserID) + } + uniqueRPLNodes[nodeAddress] = struct{}{} + } else { + if _, ok := subMap[validatorEventFilter]; !ok { + subMap[validatorEventFilter] = make([]*types.Subscription, 0) + } + hydratedSub := &types.Subscription{ ID: sub.ID, UserID: sub.UserID, EventName: sub.EventName, - EventFilter: nodeAddress, + EventFilter: validatorEventFilter, LastSent: sub.LastSent, LastEpoch: sub.LastEpoch, CreatedTime: sub.CreatedTime, @@ -146,41 +267,15 @@ func GetSubsForEventFilter(eventName types.EventName, lastSentFilter string, las DashboardGroupId: sub.DashboardGroupId, DashboardGroupName: group.Name, } - subMap[nodeAddress] = append(subMap[nodeAddress], hydratedSub) - } - uniqueRPLNodes[nodeAddress] = struct{}{} - } else { - if _, ok := subMap[validatorEventFilter]; !ok { - subMap[validatorEventFilter] = make([]types.Subscription, 0) + subMap[validatorEventFilter] = append(subMap[validatorEventFilter], hydratedSub) + //log.Infof("hydrated subscription for validator %v of dashboard %d and group %d for user %d", hydratedSub.EventFilter, *hydratedSub.DashboardId, *hydratedSub.DashboardGroupId, *hydratedSub.UserID) } - hydratedSub := types.Subscription{ - ID: sub.ID, - UserID: sub.UserID, - EventName: sub.EventName, - EventFilter: validatorEventFilter, - LastSent: sub.LastSent, - LastEpoch: sub.LastEpoch, - CreatedTime: sub.CreatedTime, - CreatedEpoch: sub.CreatedEpoch, - EventThreshold: sub.EventThreshold, - DashboardId: sub.DashboardId, - DashboardName: dashboard.Name, - DashboardGroupId: sub.DashboardGroupId, - DashboardGroupName: group.Name, - } - subMap[validatorEventFilter] = append(subMap[validatorEventFilter], hydratedSub) } - - //log.Infof("hydrated subscription for validator %v of dashboard %d and group %d for user %d", hydratedSub.EventFilter, *hydratedSub.DashboardId, *hydratedSub.DashboardGroupId, *hydratedSub.UserID) } } } - } else { - if _, ok := subMap[sub.EventFilter]; !ok { - subMap[sub.EventFilter] = make([]types.Subscription, 0) - } - subMap[sub.EventFilter] = append(subMap[sub.EventFilter], sub) } + //log.Infof("hydrated %d subscriptions for event %s", len(subMap), eventName) } return subMap, nil diff --git a/backend/pkg/notification/types.go b/backend/pkg/notification/types.go index 9028ffa7e..a7ad5e1a3 100644 --- a/backend/pkg/notification/types.go +++ b/backend/pkg/notification/types.go @@ -135,6 +135,41 @@ func (n *ValidatorProposalNotification) GetLegacyTitle() string { return "-" } +type ValidatorUpcomingProposalNotification struct { + types.NotificationBaseImpl + + ValidatorIndex uint64 + Slot uint64 +} + +func (n *ValidatorUpcomingProposalNotification) GetEntitiyId() string { + return fmt.Sprintf("%d", n.ValidatorIndex) +} + +func (n *ValidatorUpcomingProposalNotification) GetInfo(format types.NotificationFormat) string { + vali := formatValidatorLink(format, n.ValidatorIndex) + slot := formatSlotLink(format, n.Slot) + dashboardAndGroupInfo := formatDashboardAndGroupLink(format, n) + return fmt.Sprintf(`New scheduled block proposal at slot %s for Validator %s%s.`, slot, vali, dashboardAndGroupInfo) +} + +func (n *ValidatorUpcomingProposalNotification) GetLegacyInfo() string { + var generalPart, suffix string + vali := strconv.FormatUint(n.ValidatorIndex, 10) + slot := strconv.FormatUint(n.Slot, 10) + generalPart = fmt.Sprintf(`New scheduled block proposal at slot %s for Validator %s.`, slot, vali) + + return generalPart + suffix +} + +func (n *ValidatorUpcomingProposalNotification) GetTitle() string { + return n.GetLegacyTitle() +} + +func (n *ValidatorUpcomingProposalNotification) GetLegacyTitle() string { + return "Upcoming Block Proposal" +} + type ValidatorIsOfflineNotification struct { types.NotificationBaseImpl From 52db45f80c3c6ff54a71c87a104c19454cf52e09 Mon Sep 17 00:00:00 2001 From: remoterami <142154971+remoterami@users.noreply.github.com> Date: Wed, 16 Oct 2024 13:33:43 +0200 Subject: [PATCH 006/124] scheduled blocks retrieval (WIP) --- backend/pkg/api/data_access/vdb_blocks.go | 213 +++++++++++++--------- 1 file changed, 126 insertions(+), 87 deletions(-) diff --git a/backend/pkg/api/data_access/vdb_blocks.go b/backend/pkg/api/data_access/vdb_blocks.go index ca881c0fb..4ab744b00 100644 --- a/backend/pkg/api/data_access/vdb_blocks.go +++ b/backend/pkg/api/data_access/vdb_blocks.go @@ -9,6 +9,7 @@ import ( "strings" "time" + "github.com/doug-martin/goqu/v9" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/gobitfly/beaconchain/pkg/api/enums" t "github.com/gobitfly/beaconchain/pkg/api/types" @@ -17,13 +18,38 @@ import ( "github.com/gobitfly/beaconchain/pkg/commons/log" "github.com/gobitfly/beaconchain/pkg/commons/types" "github.com/gobitfly/beaconchain/pkg/commons/utils" + "github.com/lib/pq" "github.com/shopspring/decimal" ) +type table string + +// Stringer interface +func (t table) String() string { + return string(t) +} + +//func (t table) C(column string) exp.IdentifierExpression { +// return goqu.I(string(t) + "." + column) +//} + +func (t table) C(column string) string { + return string(t) + "." + column +} + func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, dashboardId t.VDBId, cursor string, colSort t.Sort[enums.VDBBlocksColumn], search string, limit uint64, protocolModes t.VDBProtocolModes) ([]t.VDBBlocksTableRow, *t.Paging, error) { // @DATA-ACCESS incorporate protocolModes + + // ------------------------------------- + // Setup var err error var currentCursor t.BlocksCursor + validatorMapping, err := d.services.GetCurrentValidatorMapping() + if err != nil { + return nil, nil, err + } + validators := table("validators") + groups := table("goups") // TODO @LuccaBitfly move validation to handler? if cursor != "" { @@ -32,83 +58,127 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das } } - // regexes taken from api handler common.go searchPubkey := regexp.MustCompile(`^0x[0-9a-fA-F]{96}$`).MatchString(search) searchGroup := regexp.MustCompile(`^[a-zA-Z0-9_\-.\ ]+$`).MatchString(search) searchIndex := regexp.MustCompile(`^[0-9]+$`).MatchString(search) - validatorMap := make(map[t.VDBValidator]bool) - params := []interface{}{} - filteredValidatorsQuery := "" - validatorMapping, err := d.services.GetCurrentValidatorMapping() - if err != nil { - return nil, nil, err + // ------------------------------------- + // Goqu Query: Determine validators filtered by search + type validatorGroup struct { + Validator t.VDBValidator `db:"validator_index"` + Group uint64 `db:"group_id"` } - - // determine validators of interest first + var filteredValidators []validatorGroup + validatorsDs := goqu.Dialect("postgres"). + From( + goqu.T("users_val_dashboards_validators").As(validators), + ). + Select( + validators.C("validator_index"), + ) if dashboardId.Validators == nil { - // could also optimize this for the average and/or the whale case; will go with some middle-ground, needs testing - // (query validators twice: once without search applied (fast) to pre-filter scheduled proposals (which are sent to db, want to minimize), - // again for blocks query with search applied to not having to send potentially huge validator-list) - startTime := time.Now() - valis, err := d.getDashboardValidators(ctx, dashboardId, nil) - log.Debugf("=== getting validators took %s", time.Since(startTime)) - if err != nil { - return nil, nil, err - } - for _, v := range valis { - validatorMap[v] = true - } + validatorsDs = validatorsDs. + Select( + // TODO mustn't be here, can be done further down + validators.C("group_id"), + ). + Where(goqu.Ex{validators.C("dashboard_id"): dashboardId.Id}) - // create a subquery to get the (potentially filtered) validators and their groups for later - params = append(params, dashboardId.Id) - selectStr := `SELECT validator_index, group_id ` - from := `FROM users_val_dashboards_validators validators ` - where := `WHERE validators.dashboard_id = $1` - extraConds := make([]string, 0, 3) + // apply search filters if searchIndex { - params = append(params, search) - extraConds = append(extraConds, fmt.Sprintf(`validator_index = $%d`, len(params))) + validatorsDs = validatorsDs.Where(goqu.Ex{validators.C("validator_index"): search}) } if searchGroup { - from += `INNER JOIN users_val_dashboards_groups groups ON validators.dashboard_id = groups.dashboard_id AND validators.group_id = groups.id ` - // escape the psql single character wildcard "_"; apply prefix-search - params = append(params, strings.Replace(search, "_", "\\_", -1)+"%") - extraConds = append(extraConds, fmt.Sprintf(`LOWER(name) LIKE LOWER($%d)`, len(params))) + validatorsDs = validatorsDs. + InnerJoin(goqu.T("users_val_dashboards_groups").As(groups), goqu.On( + goqu.Ex{validators.C("dashboard_id"): groups.C("dashboard_id")}, + goqu.Ex{validators.C("group_id"): groups.C("id")}, + )). + Where( + goqu.L("LOWER(?)", groups.C("name")).Like(strings.Replace(search, "_", "\\_", -1) + "%"), + ) } if searchPubkey { index, ok := validatorMapping.ValidatorIndices[search] - if !ok && len(extraConds) == 0 { - // don't even need to query + if !ok && !searchGroup && !searchIndex { + // searched pubkey doesn't exist, don't even need to query anything return make([]t.VDBBlocksTableRow, 0), &t.Paging{}, nil } - params = append(params, index) - extraConds = append(extraConds, fmt.Sprintf(`validator_index = $%d`, len(params))) - } - if len(extraConds) > 0 { - where += ` AND (` + strings.Join(extraConds, ` OR `) + `)` - } - filteredValidatorsQuery = selectStr + from + where + validatorsDs = validatorsDs. + Where(goqu.Ex{validators.C("validator_index"): index}) + } } else { - validators := make([]t.VDBValidator, 0, len(dashboardId.Validators)) for _, validator := range dashboardId.Validators { if searchIndex && fmt.Sprint(validator) != search || searchPubkey && validator != validatorMapping.ValidatorIndices[search] { continue } - validatorMap[validator] = true - validators = append(validators, validator) + filteredValidators = append(filteredValidators, validatorGroup{ + Validator: validator, + Group: t.DefaultGroupId, + }) if searchIndex || searchPubkey { break } } - if len(validators) == 0 { - return make([]t.VDBBlocksTableRow, 0), &t.Paging{}, nil + validatorsDs = validatorsDs. + Where(goqu.L( + validators.C("validator_index")+" = ANY(?)", pq.Array(filteredValidators)), + ) + } + + if dashboardId.Validators == nil { + validatorsQuery, validatorsArgs, err := validatorsDs.Prepared(true).ToSQL() + if err != nil { + return nil, nil, err + } + if err = d.alloyReader.SelectContext(ctx, &filteredValidators, validatorsQuery, validatorsArgs...); err != nil { + return nil, nil, err + } + } + if len(filteredValidators) == 0 { + return make([]t.VDBBlocksTableRow, 0), &t.Paging{}, nil + } + + // ------------------------------------- + // Gather scheduled blocks + // found in dutiesInfo; pass results to final query later and let db do the sorting etc + validatorSet := make(map[t.VDBValidator]bool) + for _, v := range filteredValidators { + validatorSet[v.Validator] = true + } + var scheduledProposers []t.VDBValidator + var scheduledEpochs []uint64 + var scheduledSlots []uint64 + // don't need if requested slots are in the past + latestSlot := cache.LatestSlot.Get() + onlyPrimarySort := colSort.Column == enums.VDBBlockSlot || colSort.Column == enums.VDBBlockBlock + if !onlyPrimarySort || !currentCursor.IsValid() || + currentCursor.Slot > latestSlot+1 && currentCursor.Reverse != colSort.Desc || + currentCursor.Slot < latestSlot+1 && currentCursor.Reverse == colSort.Desc { + dutiesInfo, err := d.services.GetCurrentDutiesInfo() + if err == nil { + for slot, vali := range dutiesInfo.PropAssignmentsForSlot { + // only gather scheduled slots + if _, ok := dutiesInfo.SlotStatus[slot]; ok { + continue + } + // only gather slots scheduled for our validators + if _, ok := validatorSet[vali]; !ok { + continue + } + scheduledProposers = append(scheduledProposers, dutiesInfo.PropAssignmentsForSlot[slot]) + scheduledEpochs = append(scheduledEpochs, slot/utils.Config.Chain.ClConfig.SlotsPerEpoch) + scheduledSlots = append(scheduledSlots, slot) + } + } else { + log.Debugf("duties info not available, skipping scheduled slots: %s", err) } - params = append(params, validators) } + // WIP + var proposals []struct { Proposer t.VDBValidator `db:"proposer"` Group uint64 `db:"group_id"` @@ -126,26 +196,26 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das } // handle sorting + params := make([]any, 0) where := `` orderBy := `ORDER BY ` sortOrder := ` ASC` if colSort.Desc { sortOrder = ` DESC` } - var val any + var offset any sortColName := `slot` switch colSort.Column { case enums.VDBBlockProposer: sortColName = `proposer` - val = currentCursor.Proposer + offset = currentCursor.Proposer case enums.VDBBlockStatus: sortColName = `status` - val = currentCursor.Status + offset = currentCursor.Status case enums.VDBBlockProposerReward: sortColName = `el_reward + cl_reward` - val = currentCursor.Reward + offset = currentCursor.Reward } - onlyPrimarySort := sortColName == `slot` if currentCursor.IsValid() { sign := ` > ` if colSort.Desc && !currentCursor.IsReverse() || !colSort.Desc && currentCursor.IsReverse() { @@ -163,7 +233,7 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das if onlyPrimarySort { where += `slot` + sign + fmt.Sprintf(`$%d`, len(params)) } else { - params = append(params, val) + params = append(params, offset) secSign := ` < ` if currentCursor.IsReverse() { secSign = ` > ` @@ -190,36 +260,6 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das orderBy += `, slot ` + secSort } - // Get scheduled blocks. They aren't written to blocks table, get from duties - // Will just pass scheduled proposals to query and let db do the sorting etc - var scheduledProposers []t.VDBValidator - var scheduledEpochs []uint64 - var scheduledSlots []uint64 - // don't need to query if requested slots are in the past - latestSlot := cache.LatestSlot.Get() - if !onlyPrimarySort || !currentCursor.IsValid() || - currentCursor.Slot > latestSlot+1 && currentCursor.Reverse != colSort.Desc || - currentCursor.Slot < latestSlot+1 && currentCursor.Reverse == colSort.Desc { - dutiesInfo, err := d.services.GetCurrentDutiesInfo() - if err == nil { - for slot, vali := range dutiesInfo.PropAssignmentsForSlot { - // only gather scheduled slots - if _, ok := dutiesInfo.SlotStatus[slot]; ok { - continue - } - // only gather slots scheduled for our validators - if _, ok := validatorMap[vali]; !ok { - continue - } - scheduledProposers = append(scheduledProposers, dutiesInfo.PropAssignmentsForSlot[slot]) - scheduledEpochs = append(scheduledEpochs, slot/utils.Config.Chain.ClConfig.SlotsPerEpoch) - scheduledSlots = append(scheduledSlots, slot) - } - } else { - log.Debugf("duties info not available, skipping scheduled slots: %s", err) - } - } - groupIdCol := "group_id" if dashboardId.Validators != nil { groupIdCol = fmt.Sprintf("%d AS %s", t.DefaultGroupId, groupIdCol) @@ -256,9 +296,8 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das } query += `) as u `*/ if dashboardId.Validators == nil { - cte += fmt.Sprintf(` - INNER JOIN (%s) validators ON validators.validator_index = proposer - `, filteredValidatorsQuery) + //cte += fmt.Sprintf(` + //INNER JOIN (%s) validators ON validators.validator_index = proposer`, filteredValidatorsQuery) } else { if len(where) == 0 { where += `WHERE ` From 4b0b36bdbeca985faa97b0bee0c31b9b5f39d910 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Thu, 17 Oct 2024 06:59:38 +0000 Subject: [PATCH 007/124] fix(notifications): adapt withdrawal notification title --- backend/pkg/commons/types/frontend.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/pkg/commons/types/frontend.go b/backend/pkg/commons/types/frontend.go index 64b71cb09..e1f3f25ae 100644 --- a/backend/pkg/commons/types/frontend.go +++ b/backend/pkg/commons/types/frontend.go @@ -204,7 +204,7 @@ var EventLabel map[EventName]string = map[EventName]string{ ValidatorGotSlashedEventName: "Validator slashed", ValidatorDidSlashEventName: "Validator has slashed", ValidatorIsOfflineEventName: "Validator online / offline", - ValidatorReceivedWithdrawalEventName: "Validator withdrawal initiated", + ValidatorReceivedWithdrawalEventName: "Withdrawal processed", NetworkLivenessIncreasedEventName: "The network is experiencing liveness issues", EthClientUpdateEventName: "An Ethereum client has a new update available", MonitoringMachineOfflineEventName: "Machine offline", From 605ce2f13767c0289fe0348666c048d32dc36f18 Mon Sep 17 00:00:00 2001 From: marcel-bitfly <174338434+marcel-bitfly@users.noreply.github.com> Date: Thu, 17 Oct 2024 09:14:23 +0200 Subject: [PATCH 008/124] chore(notifications): remove `rocketpool table` See: BEDS-597 --- frontend/.vscode/settings.json | 1 - .../NotificationsRocketPoolTable.vue | 225 ------------------ frontend/locales/en.json | 16 +- frontend/pages/notifications.vue | 13 - .../useNotificationsRocketpoolStore.ts | 75 ------ frontend/types/customFetch.ts | 5 - 6 files changed, 1 insertion(+), 334 deletions(-) delete mode 100644 frontend/components/notifications/NotificationsRocketPoolTable.vue delete mode 100644 frontend/stores/notifications/useNotificationsRocketpoolStore.ts diff --git a/frontend/.vscode/settings.json b/frontend/.vscode/settings.json index ce016a207..8ebee51da 100644 --- a/frontend/.vscode/settings.json +++ b/frontend/.vscode/settings.json @@ -12,7 +12,6 @@ "NotificationsManagmentMachines", "NotificationsNetworkTable", "NotificationsOverview", - "NotificationsRocketPoolTable", "a11y", "checkout", "ci", diff --git a/frontend/components/notifications/NotificationsRocketPoolTable.vue b/frontend/components/notifications/NotificationsRocketPoolTable.vue deleted file mode 100644 index 74e654823..000000000 --- a/frontend/components/notifications/NotificationsRocketPoolTable.vue +++ /dev/null @@ -1,225 +0,0 @@ - - - - - diff --git a/frontend/locales/en.json b/frontend/locales/en.json index 586d44e26..50beb6d4e 100644 --- a/frontend/locales/en.json +++ b/frontend/locales/en.json @@ -805,19 +805,6 @@ }, "push": "Push" }, - "rocketpool": { - "col": { - "node_address": "Node Address", - "notification": "Notification", - "rocketpool_subscription":"Rocketpool ({count} Subscriptions)" - }, - "event_types": { - "collateral_max": "Max collateral reached", - "collateral_min": "Min collateral reached", - "reward_round": "New reward round" - }, - "search_placeholder": "Node Address" - }, "subscriptions": { "accounts": { "erc20_token_transfers": { @@ -906,8 +893,7 @@ "dashboards": "Dashboards", "general": "General", "machines": "Machines", - "networks": "Networks", - "rocketpool": "Rocket Pool" + "networks": "Networks" }, "title": "Notifications" }, diff --git a/frontend/pages/notifications.vue b/frontend/pages/notifications.vue index 53179e311..0ec1050d3 100644 --- a/frontend/pages/notifications.vue +++ b/frontend/pages/notifications.vue @@ -19,7 +19,6 @@ const tabKey = { dashboards: 'dashboards', machines: 'machines', networks: 'networks', - rocketpool: 'rocketpool', } const tabs: HashTabs = [ { @@ -37,10 +36,6 @@ const tabs: HashTabs = [ key: tabKey.clients, title: $t('notifications.tabs.clients'), }, - { - key: tabKey.rocketpool, - title: $t('notifications.tabs.rocketpool'), - }, { icon: faNetworkWired, key: tabKey.networks, @@ -106,14 +101,6 @@ const openManageNotifications = () => { @open-dialog="openManageNotifications" /> - - - - - - - () > {{ proposal.index }} - - - - - - () > {{ upcomingProposal.index }} - - - - { has-unit :label="$t('notifications.subscriptions.validators.max_collateral_reached.label')" /> - - Date: Fri, 18 Oct 2024 14:18:35 +0200 Subject: [PATCH 049/124] refactor(NotificationsManagementSubscriptionDialog): rename `component` It is easier to work with components if their name reflect the file path in nuxt. --- frontend/.vscode/settings.json | 2 +- ...Dialog.vue => NotificationsManagementSubscriptionDialog.vue} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename frontend/components/notifications/management/{SubscriptionDialog.vue => NotificationsManagementSubscriptionDialog.vue} (100%) diff --git a/frontend/.vscode/settings.json b/frontend/.vscode/settings.json index 22971fa5c..4e60da420 100644 --- a/frontend/.vscode/settings.json +++ b/frontend/.vscode/settings.json @@ -11,10 +11,10 @@ "NotificationsDashboardDialogEntity", "NotificationsDashboardTable", "NotificationsManagementModalWebhook", + "NotificationsManagementSubscriptionDialog", "NotificationsManagmentMachines", "NotificationsNetworkTable", "NotificationsOverview", - "SubscriptionDialog", "a11y", "checkout", "ci", diff --git a/frontend/components/notifications/management/SubscriptionDialog.vue b/frontend/components/notifications/management/NotificationsManagementSubscriptionDialog.vue similarity index 100% rename from frontend/components/notifications/management/SubscriptionDialog.vue rename to frontend/components/notifications/management/NotificationsManagementSubscriptionDialog.vue From cf997c42bd18f72054a4cdd86765b30c93befbdb Mon Sep 17 00:00:00 2001 From: remoterami <142154971+remoterami@users.noreply.github.com> Date: Fri, 18 Oct 2024 14:44:09 +0200 Subject: [PATCH 050/124] union past and scheduled blocks, syntax fixed --- backend/pkg/api/data_access/general.go | 13 +- backend/pkg/api/data_access/vdb_blocks.go | 224 +++++++----------- .../api/enums/validator_dashboard_enums.go | 2 +- backend/pkg/api/types/data_access.go | 13 +- .../dashboard/table/DashboardTableBlocks.vue | 2 +- 5 files changed, 108 insertions(+), 146 deletions(-) diff --git a/backend/pkg/api/data_access/general.go b/backend/pkg/api/data_access/general.go index 3c94db8a1..26d051dc9 100644 --- a/backend/pkg/api/data_access/general.go +++ b/backend/pkg/api/data_access/general.go @@ -62,6 +62,9 @@ func applySortAndPagination(defaultColumns []types.SortColumn, primary types.Sor if primary.Offset == nil { queryOrderColumns[0].Offset = column.Offset } + if len(primary.Table) == 0 { + queryOrderColumns[0].Table = column.Table + } continue } queryOrderColumns = append(queryOrderColumns, column) @@ -74,9 +77,9 @@ func applySortAndPagination(defaultColumns []types.SortColumn, primary types.Sor if cursor.IsReverse() { column.Desc = !column.Desc } - colOrder := goqu.C(column.Column).Asc() + colOrder := column.Expr().Asc() if column.Desc { - colOrder = goqu.C(column.Column).Desc() + colOrder = column.Expr().Desc() } queryOrder = append(queryOrder, colOrder) } @@ -87,15 +90,15 @@ func applySortAndPagination(defaultColumns []types.SortColumn, primary types.Sor // reverse order to nest conditions for i := len(queryOrderColumns) - 1; i >= 0; i-- { column := queryOrderColumns[i] - colWhere := goqu.C(column.Column).Gt(column.Offset) + colWhere := column.Expr().Gt(column.Offset) if column.Desc { - colWhere = goqu.C(column.Column).Lt(column.Offset) + colWhere = column.Expr().Lt(column.Offset) } if queryWhere == nil { queryWhere = colWhere } else { - queryWhere = goqu.And(goqu.C(column.Column).Eq(column.Offset), queryWhere) + queryWhere = goqu.And(column.Expr().Eq(column.Offset), queryWhere) queryWhere = goqu.Or(colWhere, queryWhere) } } diff --git a/backend/pkg/api/data_access/vdb_blocks.go b/backend/pkg/api/data_access/vdb_blocks.go index 409099ebc..f45ec2ac1 100644 --- a/backend/pkg/api/data_access/vdb_blocks.go +++ b/backend/pkg/api/data_access/vdb_blocks.go @@ -22,21 +22,6 @@ import ( "github.com/shopspring/decimal" ) -type table string - -// Stringer interface -func (t table) String() string { - return string(t) -} - -//func (t table) C(column string) exp.IdentifierExpression { -// return goqu.I(string(t) + "." + column) -//} - -func (t table) C(column string) string { - return string(t) + "." + column -} - func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, dashboardId t.VDBId, cursor string, colSort t.Sort[enums.VDBBlocksColumn], search string, limit uint64, protocolModes t.VDBProtocolModes) ([]t.VDBBlocksTableRow, *t.Paging, error) { // @DATA-ACCESS incorporate protocolModes @@ -48,9 +33,9 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das if err != nil { return nil, nil, err } - validators := table("validators") - blocks := table("blocks") - groups := table("goups") + validators := goqu.T("users_val_dashboards_validators").As("validators") + blocks := goqu.T("blocks") + groups := goqu.T("goups") // TODO @LuccaBitfly move validation to handler? if cursor != "" { @@ -72,31 +57,24 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das var filteredValidators []validatorGroup validatorsDs := goqu.Dialect("postgres"). Select( - validators.C("validator_index"), + "validator_index", ) if dashboardId.Validators == nil { validatorsDs = validatorsDs. - From( - goqu.T("users_val_dashboards_validators").As(validators), - ). - /*Select( - // TODO mustn't be here, can be done further down - validators.C("group_id"), - ).*/ - Where(goqu.Ex{validators.C("dashboard_id"): dashboardId.Id}) - + From(validators). + Where(validators.Col("dashboard_id").Eq(dashboardId.Id)) // apply search filters if searchIndex { - validatorsDs = validatorsDs.Where(goqu.Ex{validators.C("validator_index"): search}) + validatorsDs = validatorsDs.Where(validators.Col("validator_index").Eq(search)) } if searchGroup { validatorsDs = validatorsDs. InnerJoin(goqu.T("users_val_dashboards_groups").As(groups), goqu.On( - goqu.Ex{validators.C("dashboard_id"): groups.C("dashboard_id")}, - goqu.Ex{validators.C("group_id"): groups.C("id")}, + validators.Col("group_id").Eq(groups.Col("id")), + validators.Col("dashboard_id").Eq(groups.Col("dashboard_id")), )). Where( - goqu.L("LOWER(?)", groups.C("name")).Like(strings.Replace(search, "_", "\\_", -1) + "%"), + goqu.L("LOWER(?)", groups.Col("name")).Like(strings.Replace(search, "_", "\\_", -1) + "%"), ) } if searchPubkey { @@ -107,7 +85,7 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das } validatorsDs = validatorsDs. - Where(goqu.Ex{validators.C("validator_index"): index}) + Where(validators.Col("validator_index").Eq(index)) } } else { for _, validator := range dashboardId.Validators { @@ -126,7 +104,7 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das validatorsDs = validatorsDs. From( goqu.L("unnest(?)", pq.Array(filteredValidators)).As("validator_index"), - ).As(string(validators)) + ).As("validators") // TODO ? } if dashboardId.Validators == nil { @@ -180,51 +158,37 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das // Sorting and pagination if cursor is present defaultColumns := []t.SortColumn{ - {Column: enums.VDBBlocksColumns.Slot.ToString(), Desc: true, Offset: currentCursor.Slot}, + {Column: enums.VDBBlocksColumns.Slot.ToString(), Table: blocks.GetTable(), Desc: true, Offset: currentCursor.Slot}, } var offset any + var table string switch colSort.Column { case enums.VDBBlocksColumns.Proposer: offset = currentCursor.Proposer case enums.VDBBlocksColumns.Block: offset = currentCursor.Block + table = blocks.GetTable() case enums.VDBBlocksColumns.Status: offset = fmt.Sprintf("%d", currentCursor.Status) // type of 'status' column is text for some reason case enums.VDBBlocksColumns.ProposerReward: offset = currentCursor.Reward } - order, directions := applySortAndPagination(defaultColumns, t.SortColumn{Column: colSort.Column.ToString(), Desc: colSort.Desc, Offset: offset}, currentCursor.GenericCursor) + order, directions := applySortAndPagination(defaultColumns, t.SortColumn{Column: colSort.Column.ToString(), Table: table, Desc: colSort.Desc, Offset: offset}, currentCursor.GenericCursor) validatorsDs = validatorsDs.Order(order...) if directions != nil { validatorsDs = validatorsDs.Where(directions) } - // group id - if dashboardId.Validators == nil { - validatorsDs = validatorsDs.Select( - validators.C("group_id"), - ) - } else { - validatorsDs = validatorsDs.Select( - goqu.L("?", t.DefaultGroupId).As("group_id"), - ) - } - validatorsDs = validatorsDs. - Select( - blocks.C("proposer"), - blocks.C("epoch"), - blocks.C("slot"), - blocks.C("status"), - blocks.C("exec_block_number"), - blocks.C("graffiti_text"), - ). + InnerJoin(blocks, goqu.On( + blocks.Col("proposer").Eq(validators.Col("validator_index")), + )). LeftJoin(goqu.T("consensus_payloads").As("cp"), goqu.On( - goqu.Ex{blocks.C("slot"): goqu.I("cp.slot")}, + blocks.Col("slot").Eq(goqu.I("cp.slot")), )). LeftJoin(goqu.T("execution_payloads").As("ep"), goqu.On( - goqu.Ex{blocks.C("exec_block_hash"): goqu.I("ep.block_hash")}, + blocks.Col("exec_block_hash").Eq(goqu.I("ep.block_hash")), )). LeftJoin( // relay bribe deduplication; select most likely (=max) relay bribe value for each block @@ -232,101 +196,85 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das From(goqu.T("relays_blocks")). Select( goqu.I("relays_blocks.exec_block_hash"), + goqu.I("relays_blocks.proposer_fee_recipient"), goqu.MAX(goqu.I("relays_blocks.value")).As("value")). - // needed? TODO test - // Where(goqu.L("relays_blocks.exec_block_hash = blocks.exec_block_hash")). - GroupBy("exec_block_hash")).As("rb"), + GroupBy( + "exec_block_hash", + "proposer_fee_recipient", + )).As("rb"), goqu.On( - goqu.Ex{"rb.exec_block_hash": blocks.C("exec_block_hash")}, + goqu.I("rb.exec_block_hash").Eq(blocks.Col("exec_block_hash")), ), ). - Select( - goqu.COALESCE(goqu.I("rb.proposer_fee_recipient"), blocks.C("exec_fee_recipient")).As("fee_recipient"), + SelectAppend( + blocks.Col("epoch"), + blocks.Col("slot"), + blocks.Col("status"), + blocks.Col("exec_block_number"), + blocks.Col("graffiti_text"), + goqu.COALESCE(goqu.I("rb.proposer_fee_recipient"), blocks.Col("exec_fee_recipient")).As("fee_recipient"), goqu.COALESCE(goqu.L("rb.value / 1e18"), goqu.I("ep.fee_recipient_reward")).As("el_reward"), goqu.L("cp.cl_attestations_reward / 1e9 + cp.cl_sync_aggregate_reward / 1e9 + cp.cl_slashing_inclusion_reward / 1e9").As("cl_reward"), - ) - - // union scheduled blocks if present - // WIP - - params := make([]any, 0) - selectFields, where, orderBy, groupIdCol, sortColName := "", "", "", "", "" - cte := fmt.Sprintf(`WITH past_blocks AS (SELECT - %s - FROM blocks - `, selectFields) + ). + Limit(uint(limit + 1)) - if dashboardId.Validators == nil { - //cte += fmt.Sprintf(` - //INNER JOIN (%s) validators ON validators.validator_index = proposer`, filteredValidatorsQuery) - } else { - if len(where) == 0 { - where += `WHERE ` - } else { - where += `AND ` - } - where += `proposer = ANY($1) ` + // Group id + groupId := validators.Col("group_id") + if dashboardId.Validators != nil { + groupId = goqu.V(t.DefaultGroupId).As("group_id").GetAs() } + validatorsDs = validatorsDs.SelectAppend(groupId) - params = append(params, limit+1) - limitStr := fmt.Sprintf(` - LIMIT $%d - `, len(params)) + /* + if dashboardId.Validators == nil { + validatorsDs = validatorsDs.Select( + validators.Col("group_id"), + ) + } else { + validatorsDs = validatorsDs.Select( + goqu.L("?", t.DefaultGroupId).As("group_id"), + ) + }*/ - from := `past_blocks ` - selectStr := `SELECT * FROM ` + // union scheduled blocks if present + // WIP - query := selectStr + from + where + orderBy + limitStr - // supply scheduled proposals, if any + finalDs := validatorsDs if len(scheduledProposers) > 0 { - // distinct to filter out duplicates in an edge case (if dutiesInfo didn't update yet after a block was proposed, but the blocks table was) + scheduledDs := goqu.Dialect("postgres"). + From( + goqu.L("unnest(?, ?, ?) AS prov(validator_index, epoch, slot)", pq.Array(scheduledProposers), pq.Array(scheduledEpochs), pq.Array(scheduledSlots)), + ). + Select( + goqu.C("validator_index"), + goqu.C("epoch"), + goqu.C("slot"), + goqu.V("0").As("status"), + goqu.V(nil).As("exec_block_number"), + goqu.V(nil).As("fee_recipient"), + goqu.V(nil).As("el_reward"), + goqu.V(nil).As("cl_reward"), + goqu.V(nil).As("graffiti_text"), + ). + As("scheduled_blocks") + + // distinct + block number ordering to filter out duplicates in an edge case (if dutiesInfo didn't update yet after a block was proposed, but the blocks table was) // might be possible to remove this once the TODO in service_slot_viz.go:startSlotVizDataService is resolved - params = append(params, scheduledProposers) - params = append(params, scheduledEpochs) - params = append(params, scheduledSlots) - cte += fmt.Sprintf(`, - scheduled_blocks as ( - SELECT - prov.proposer, - prov.epoch, - prov.slot, - %s, - '0'::text AS status, - NULL::int AS exec_block_number, - ''::bytea AS fee_recipient, - NULL::float AS el_reward, - NULL::float AS cl_reward, - ''::text AS graffiti_text - FROM unnest($%d::int[], $%d::int[], $%d::int[]) AS prov(proposer, epoch, slot) - `, groupIdCol, len(params)-2, len(params)-1, len(params)) - if dashboardId.Validators == nil { - // add group id - cte += fmt.Sprintf(`INNER JOIN users_val_dashboards_validators validators - ON validators.dashboard_id = $1 - AND validators.validator_index = ANY($%d::int[]) - `, len(params)-2) - } - cte += `) ` - distinct := "slot" + finalDs = validatorsDs. + Union(scheduledDs). + Where(directions). + Order(order...). + OrderAppend(goqu.C("exec_block_number").Desc().NullsLast()). + Limit(uint(limit + 1)). + Distinct(blocks.Col("slot")) if !onlyPrimarySort { - distinct = sortColName + ", " + distinct + finalDs = finalDs. + Distinct(blocks.Col("slot"), blocks.Col("exec_block_number")) } - // keep all ordering, sorting etc - selectStr = `SELECT DISTINCT ON (` + distinct + `) * FROM ` - // encapsulate past blocks query to ensure performance - from = `( - ( ` + query + ` ) - UNION ALL - SELECT * FROM scheduled_blocks - ) as combined - ` - // make sure the distinct clause filters out the correct duplicated row (e.g. block=nil) - orderBy += `, exec_block_number NULLS LAST` - query = selectStr + from + where + orderBy + limitStr } var proposals []struct { - Proposer t.VDBValidator `db:"proposer"` + Proposer t.VDBValidator `db:"validator_index"` Group uint64 `db:"group_id"` Epoch uint64 `db:"epoch"` Slot uint64 `db:"slot"` @@ -341,11 +289,11 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das Reward decimal.Decimal } startTime := time.Now() - _, _, err = validatorsDs.Prepared(true).ToSQL() + query, args, err := finalDs.Prepared(true).ToSQL() if err != nil { return nil, nil, err } - err = d.alloyReader.SelectContext(ctx, &proposals, cte+query, params...) + err = d.alloyReader.SelectContext(ctx, &proposals, query, args...) log.Debugf("=== getting past blocks took %s", time.Since(startTime)) if err != nil { return nil, nil, err @@ -389,11 +337,11 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das } graffiti := proposal.GraffitiText data[i].Graffiti = &graffiti + block := uint64(proposal.Block.Int64) + data[i].Block = &block if proposal.Status == 3 { continue } - block := uint64(proposal.Block.Int64) - data[i].Block = &block var reward t.ClElValue[decimal.Decimal] if proposal.ElReward.Valid { rewardRecp := t.Address{ diff --git a/backend/pkg/api/enums/validator_dashboard_enums.go b/backend/pkg/api/enums/validator_dashboard_enums.go index 2646244e8..228b224dc 100644 --- a/backend/pkg/api/enums/validator_dashboard_enums.go +++ b/backend/pkg/api/enums/validator_dashboard_enums.go @@ -163,7 +163,7 @@ func (c VDBBlocksColumn) ToString() string { case VDBBlockSlot: return "slot" case VDBBlockBlock: - return "block" + return "exec_block_number" case VDBBlockStatus: return "status" case VDBBlockProposerReward: diff --git a/backend/pkg/api/types/data_access.go b/backend/pkg/api/types/data_access.go index 3b1d77c16..70961c5f2 100644 --- a/backend/pkg/api/types/data_access.go +++ b/backend/pkg/api/types/data_access.go @@ -1,8 +1,11 @@ package types import ( + "database/sql" "time" + "github.com/doug-martin/goqu/v9" + "github.com/doug-martin/goqu/v9/exp" "github.com/gobitfly/beaconchain/pkg/api/enums" "github.com/gobitfly/beaconchain/pkg/consapi/types" "github.com/gobitfly/beaconchain/pkg/monitoring/constants" @@ -24,11 +27,19 @@ type Sort[T enums.Enum] struct { type SortColumn struct { Column string + Table string // optional Desc bool // represents value from cursor Offset any } +func (s SortColumn) Expr() exp.IdentifierExpression { + if s.Table != "" { + return goqu.T(s.Table).Col(s.Column) + } + return goqu.C(s.Column) +} + type VDBIdPrimary int type VDBIdPublic string type VDBIdValidatorSet []VDBValidator @@ -167,7 +178,7 @@ type BlocksCursor struct { Proposer uint64 Slot uint64 // same as Age - Block uint64 + Block sql.NullInt64 Status uint64 Reward decimal.Decimal } diff --git a/frontend/components/dashboard/table/DashboardTableBlocks.vue b/frontend/components/dashboard/table/DashboardTableBlocks.vue index daa3a6e2f..01227c95f 100644 --- a/frontend/components/dashboard/table/DashboardTableBlocks.vue +++ b/frontend/components/dashboard/table/DashboardTableBlocks.vue @@ -52,7 +52,7 @@ const loadData = (query?: TableQueryParams) => { if (!query) { query = { limit: pageSize.value, - sort: 'block:desc', + sort: 'slot:desc', } } setQuery(query, true, true) From 123d0c0f9ea2997fafeb6a9081866d0cbbc4ff0a Mon Sep 17 00:00:00 2001 From: Patrick Date: Fri, 18 Oct 2024 15:49:06 +0200 Subject: [PATCH 051/124] chore(exporter): improve logging when fetching relays-data (#989) see: BEDS-90 --- backend/pkg/exporter/modules/relays.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/pkg/exporter/modules/relays.go b/backend/pkg/exporter/modules/relays.go index 8c0049e98..1b2573c01 100644 --- a/backend/pkg/exporter/modules/relays.go +++ b/backend/pkg/exporter/modules/relays.go @@ -114,7 +114,7 @@ func fetchDeliveredPayloads(r types.Relay, offset uint64) ([]BidTrace, error) { if err != nil { log.Error(err, "error retrieving delivered payloads", 0, map[string]interface{}{"relay": r.ID}) - return nil, err + return nil, fmt.Errorf("error retrieving delivered payloads for cursor: %v, url: %v: %v", offset, url, err) } defer resp.Body.Close() @@ -122,7 +122,7 @@ func fetchDeliveredPayloads(r types.Relay, offset uint64) ([]BidTrace, error) { err = json.NewDecoder(resp.Body).Decode(&payloads) if err != nil { - return nil, err + return nil, fmt.Errorf("error decoding json for delivered payloads for cursor: %v, url: %v: %v", offset, url, err) } return payloads, nil @@ -176,7 +176,7 @@ func retrieveAndInsertPayloadsFromRelay(r types.Relay, low_bound uint64, high_bo for { resp, err := fetchDeliveredPayloads(r, offset) if err != nil { - return err + return fmt.Errorf("error calling fetchDeliveredPayloads with offset: %v for relay: %v", r.ID, err) } if resp == nil { From ea224ae571d41ab25811650656f66c1063a13ec2 Mon Sep 17 00:00:00 2001 From: Patrick Date: Fri, 18 Oct 2024 16:03:07 +0200 Subject: [PATCH 052/124] chore(exporter): improve logging when fetching relays-data (#990) BEDS-90 --- backend/pkg/exporter/modules/relays.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/backend/pkg/exporter/modules/relays.go b/backend/pkg/exporter/modules/relays.go index 1b2573c01..78d068f93 100644 --- a/backend/pkg/exporter/modules/relays.go +++ b/backend/pkg/exporter/modules/relays.go @@ -113,16 +113,15 @@ func fetchDeliveredPayloads(r types.Relay, offset uint64) ([]BidTrace, error) { resp, err := client.Get(url) if err != nil { - log.Error(err, "error retrieving delivered payloads", 0, map[string]interface{}{"relay": r.ID}) - return nil, fmt.Errorf("error retrieving delivered payloads for cursor: %v, url: %v: %v", offset, url, err) + log.Error(err, "error retrieving delivered payloads", 0, map[string]interface{}{"relay": r.ID, "offset": offset, "url": url}) + return nil, fmt.Errorf("error retrieving delivered payloads for relay: %v, offset: %v, url: %v: %w", r.ID, offset, url, err) } defer resp.Body.Close() err = json.NewDecoder(resp.Body).Decode(&payloads) - if err != nil { - return nil, fmt.Errorf("error decoding json for delivered payloads for cursor: %v, url: %v: %v", offset, url, err) + return nil, fmt.Errorf("error decoding json for delivered payloads for relay: %v, offset: %v, url: %v: %w", r.ID, offset, url, err) } return payloads, nil @@ -176,7 +175,7 @@ func retrieveAndInsertPayloadsFromRelay(r types.Relay, low_bound uint64, high_bo for { resp, err := fetchDeliveredPayloads(r, offset) if err != nil { - return fmt.Errorf("error calling fetchDeliveredPayloads with offset: %v for relay: %v", r.ID, err) + return fmt.Errorf("error calling fetchDeliveredPayloads with offset: %v for relay: %v: %w", offset, r.ID, err) } if resp == nil { From a899c3e36b2aaf7de4bfc8d47b70367cde850911 Mon Sep 17 00:00:00 2001 From: benji-bitfly Date: Fri, 18 Oct 2024 15:58:56 +0200 Subject: [PATCH 053/124] refactor: fix `translation` for `attestation missed` See: BEDS-607 --- frontend/locales/en.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/locales/en.json b/frontend/locales/en.json index da75288c4..66a5f0ec2 100644 --- a/frontend/locales/en.json +++ b/frontend/locales/en.json @@ -837,7 +837,7 @@ "label": "All events" }, "attestation_missed": { - "info": "We will trigger every epoch ({count} minute) during downtime. | We will trigger every epoch ({count} minutes) during downtime.", + "info": "We will trigger a notification every epoch ({count} minute) during downtime. | We will trigger a notification every epoch ({count} minutes) during downtime.", "label": "Attestations missed" }, "block_proposal": { From f253ce867085b41e7fc0be0c2eb146473f7c166f Mon Sep 17 00:00:00 2001 From: marcel-bitfly <174338434+marcel-bitfly@users.noreply.github.com> Date: Fri, 18 Oct 2024 11:49:05 +0200 Subject: [PATCH 054/124] chore: add `command` to `mock api responses` --- frontend/README.md | 5 ++++- frontend/package.json | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/frontend/README.md b/frontend/README.md index 190640f2a..7f1a5afb3 100644 --- a/frontend/README.md +++ b/frontend/README.md @@ -128,7 +128,10 @@ If your `user` was added to the `ADMIN` or `DEV` group by the `api team`, you ca `mocked data` from the `api` for certain `endpoints` by adding `?is_mocked=true` as a `query parameter`. -You can `turn on` mocked data `globally` for all `configured enpoints` by setting `NUXT_PUBLIC_IS_API_MOCKED=true`. +You can `turn on` mocked data `globally` for all `configured enpoints` +- by setting `NUXT_PUBLIC_IS_API_MOCKED=true` +in your [.env](.env) or +- running `npm run dev:mock:api` (See: [package.json](package.json)) ## Descision Record diff --git a/frontend/package.json b/frontend/package.json index ab4b347d7..dce992e57 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -53,6 +53,7 @@ "scripts": { "build": "nuxt build", "dev": "NODE_TLS_REJECT_UNAUTHORIZED=0 nuxt dev", + "dev:mock:api": "NUXT_IS_API_MOCKED=true npm run dev", "dev:mock:production": "NUXT_DEPLOYMENT_TYPE=production npm run dev", "dev:mock:staging": "NUXT_DEPLOYMENT_TYPE=staging npm run dev", "generate": "nuxt generate", From 6a354cc894fe2b1a55986dbe84d2c0f9ef420241 Mon Sep 17 00:00:00 2001 From: marcel-bitfly <174338434+marcel-bitfly@users.noreply.github.com> Date: Fri, 18 Oct 2024 16:56:17 +0200 Subject: [PATCH 055/124] refactor(notifications): disable `machines tab ui` when `user has no machines configured` See: BEDS-574 --- frontend/components/bc/BcSlider.vue | 1 - .../NotificationsManagementMachines.vue | 43 +++++++++++++------ 2 files changed, 30 insertions(+), 14 deletions(-) diff --git a/frontend/components/bc/BcSlider.vue b/frontend/components/bc/BcSlider.vue index b3dcaf9c0..d965a55df 100644 --- a/frontend/components/bc/BcSlider.vue +++ b/frontend/components/bc/BcSlider.vue @@ -15,7 +15,6 @@ defineProps<{ :step class="bc-slider" type="range" - v-bind="$attrs" > diff --git a/frontend/components/notifications/management/NotificationsManagementMachines.vue b/frontend/components/notifications/management/NotificationsManagementMachines.vue index 67841ce52..b966c91bc 100644 --- a/frontend/components/notifications/management/NotificationsManagementMachines.vue +++ b/frontend/components/notifications/management/NotificationsManagementMachines.vue @@ -21,6 +21,8 @@ watchDebounced(() => notificationsManagementStore.settings.general_settings, asy deep: true, maxWait: waitExtraLongForSliders, }) + +const hasMachines = computed(() => notificationsManagementStore.settings.has_machines) From 327a88e4c8b90275a8b72d355269d17281f83cc3 Mon Sep 17 00:00:00 2001 From: remoterami <142154971+remoterami@users.noreply.github.com> Date: Mon, 21 Oct 2024 13:47:15 +0200 Subject: [PATCH 063/124] extended sorting/paging features, fixed blocks query --- backend/pkg/api/data_access/general.go | 19 +- backend/pkg/api/data_access/notifications.go | 38 +-- backend/pkg/api/data_access/vdb_blocks.go | 282 +++++++++--------- backend/pkg/api/enums/notifications_enums.go | 50 ++-- .../api/enums/validator_dashboard_enums.go | 25 +- backend/pkg/api/types/data_access.go | 16 +- 6 files changed, 223 insertions(+), 207 deletions(-) diff --git a/backend/pkg/api/data_access/general.go b/backend/pkg/api/data_access/general.go index 26d051dc9..d629d52df 100644 --- a/backend/pkg/api/data_access/general.go +++ b/backend/pkg/api/data_access/general.go @@ -62,9 +62,6 @@ func applySortAndPagination(defaultColumns []types.SortColumn, primary types.Sor if primary.Offset == nil { queryOrderColumns[0].Offset = column.Offset } - if len(primary.Table) == 0 { - queryOrderColumns[0].Table = column.Table - } continue } queryOrderColumns = append(queryOrderColumns, column) @@ -77,9 +74,9 @@ func applySortAndPagination(defaultColumns []types.SortColumn, primary types.Sor if cursor.IsReverse() { column.Desc = !column.Desc } - colOrder := column.Expr().Asc() + colOrder := column.Column.Asc() if column.Desc { - colOrder = column.Expr().Desc() + colOrder = column.Column.Desc() } queryOrder = append(queryOrder, colOrder) } @@ -90,15 +87,21 @@ func applySortAndPagination(defaultColumns []types.SortColumn, primary types.Sor // reverse order to nest conditions for i := len(queryOrderColumns) - 1; i >= 0; i-- { column := queryOrderColumns[i] - colWhere := column.Expr().Gt(column.Offset) + var colWhere exp.Expression + + // current convention is the psql default (ASC: nulls last, DESC: nulls first) + colWhere = goqu.Or(column.Column.Gt(column.Offset), column.Column.IsNull()) if column.Desc { - colWhere = column.Expr().Lt(column.Offset) + colWhere = column.Column.Lt(column.Offset) + if column.Offset == nil { + colWhere = goqu.Or(colWhere, column.Column.IsNull()) + } } if queryWhere == nil { queryWhere = colWhere } else { - queryWhere = goqu.And(column.Expr().Eq(column.Offset), queryWhere) + queryWhere = goqu.And(column.Column.Eq(column.Offset), queryWhere) queryWhere = goqu.Or(colWhere, queryWhere) } } diff --git a/backend/pkg/api/data_access/notifications.go b/backend/pkg/api/data_access/notifications.go index 776308329..7351de0c6 100644 --- a/backend/pkg/api/data_access/notifications.go +++ b/backend/pkg/api/data_access/notifications.go @@ -332,14 +332,14 @@ func (d *DataAccessService) GetDashboardNotifications(ctx context.Context, userI // sorting defaultColumns := []t.SortColumn{ - {Column: enums.NotificationsDashboardsColumns.Timestamp.ToString(), Desc: true, Offset: currentCursor.Epoch}, - {Column: enums.NotificationsDashboardsColumns.DashboardName.ToString(), Desc: false, Offset: currentCursor.DashboardName}, - {Column: enums.NotificationsDashboardsColumns.DashboardId.ToString(), Desc: false, Offset: currentCursor.DashboardId}, - {Column: enums.NotificationsDashboardsColumns.GroupName.ToString(), Desc: false, Offset: currentCursor.GroupName}, - {Column: enums.NotificationsDashboardsColumns.GroupId.ToString(), Desc: false, Offset: currentCursor.GroupId}, - {Column: enums.NotificationsDashboardsColumns.ChainId.ToString(), Desc: true, Offset: currentCursor.ChainId}, - } - order, directions := applySortAndPagination(defaultColumns, t.SortColumn{Column: colSort.Column.ToString(), Desc: colSort.Desc}, currentCursor.GenericCursor) + {Column: enums.NotificationsDashboardsColumns.Timestamp.ToExpr(), Desc: true, Offset: currentCursor.Epoch}, + {Column: enums.NotificationsDashboardsColumns.DashboardName.ToExpr(), Desc: false, Offset: currentCursor.DashboardName}, + {Column: enums.NotificationsDashboardsColumns.DashboardId.ToExpr(), Desc: false, Offset: currentCursor.DashboardId}, + {Column: enums.NotificationsDashboardsColumns.GroupName.ToExpr(), Desc: false, Offset: currentCursor.GroupName}, + {Column: enums.NotificationsDashboardsColumns.GroupId.ToExpr(), Desc: false, Offset: currentCursor.GroupId}, + {Column: enums.NotificationsDashboardsColumns.ChainId.ToExpr(), Desc: true, Offset: currentCursor.ChainId}, + } + order, directions := applySortAndPagination(defaultColumns, t.SortColumn{Column: colSort.Column.ToExpr(), Desc: colSort.Desc}, currentCursor.GenericCursor) unionQuery = unionQuery.Order(order...) if directions != nil { unionQuery = unionQuery.Where(directions) @@ -659,9 +659,9 @@ func (d *DataAccessService) GetMachineNotifications(ctx context.Context, userId // Sorting and limiting if cursor is present defaultColumns := []t.SortColumn{ - {Column: enums.NotificationsMachinesColumns.Timestamp.ToString(), Desc: true, Offset: currentCursor.Epoch}, - {Column: enums.NotificationsMachinesColumns.MachineId.ToString(), Desc: false, Offset: currentCursor.MachineId}, - {Column: enums.NotificationsMachinesColumns.EventType.ToString(), Desc: false, Offset: currentCursor.EventType}, + {Column: enums.NotificationsMachinesColumns.Timestamp.ToExpr(), Desc: true, Offset: currentCursor.Epoch}, + {Column: enums.NotificationsMachinesColumns.MachineId.ToExpr(), Desc: false, Offset: currentCursor.MachineId}, + {Column: enums.NotificationsMachinesColumns.EventType.ToExpr(), Desc: false, Offset: currentCursor.EventType}, } var offset interface{} switch colSort.Column { @@ -671,7 +671,7 @@ func (d *DataAccessService) GetMachineNotifications(ctx context.Context, userId offset = currentCursor.EventThreshold } - order, directions := applySortAndPagination(defaultColumns, t.SortColumn{Column: colSort.Column.ToString(), Desc: colSort.Desc, Offset: offset}, currentCursor.GenericCursor) + order, directions := applySortAndPagination(defaultColumns, t.SortColumn{Column: colSort.Column.ToExpr(), Desc: colSort.Desc, Offset: offset}, currentCursor.GenericCursor) ds = ds.Order(order...) if directions != nil { ds = ds.Where(directions) @@ -780,10 +780,10 @@ func (d *DataAccessService) GetClientNotifications(ctx context.Context, userId u // Sorting and limiting if cursor is present // Rows can be uniquely identified by (epoch, client) defaultColumns := []t.SortColumn{ - {Column: enums.NotificationsClientsColumns.Timestamp.ToString(), Desc: true, Offset: currentCursor.Epoch}, - {Column: enums.NotificationsClientsColumns.ClientName.ToString(), Desc: false, Offset: currentCursor.Client}, + {Column: enums.NotificationsClientsColumns.Timestamp.ToExpr(), Desc: true, Offset: currentCursor.Epoch}, + {Column: enums.NotificationsClientsColumns.ClientName.ToExpr(), Desc: false, Offset: currentCursor.Client}, } - order, directions := applySortAndPagination(defaultColumns, t.SortColumn{Column: colSort.Column.ToString(), Desc: colSort.Desc}, currentCursor.GenericCursor) + order, directions := applySortAndPagination(defaultColumns, t.SortColumn{Column: colSort.Column.ToExpr(), Desc: colSort.Desc}, currentCursor.GenericCursor) ds = ds.Order(order...) if directions != nil { ds = ds.Where(directions) @@ -1071,11 +1071,11 @@ func (d *DataAccessService) GetNetworkNotifications(ctx context.Context, userId // Sorting and limiting if cursor is present // Rows can be uniquely identified by (epoch, network, event_type) defaultColumns := []t.SortColumn{ - {Column: enums.NotificationNetworksColumns.Timestamp.ToString(), Desc: true, Offset: currentCursor.Epoch}, - {Column: enums.NotificationNetworksColumns.Network.ToString(), Desc: false, Offset: currentCursor.Network}, - {Column: enums.NotificationNetworksColumns.EventType.ToString(), Desc: false, Offset: currentCursor.EventType}, + {Column: enums.NotificationNetworksColumns.Timestamp.ToExpr(), Desc: true, Offset: currentCursor.Epoch}, + {Column: enums.NotificationNetworksColumns.Network.ToExpr(), Desc: false, Offset: currentCursor.Network}, + {Column: enums.NotificationNetworksColumns.EventType.ToExpr(), Desc: false, Offset: currentCursor.EventType}, } - order, directions := applySortAndPagination(defaultColumns, t.SortColumn{Column: colSort.Column.ToString(), Desc: colSort.Desc}, currentCursor.GenericCursor) + order, directions := applySortAndPagination(defaultColumns, t.SortColumn{Column: colSort.Column.ToExpr(), Desc: colSort.Desc}, currentCursor.GenericCursor) ds = ds.Order(order...) if directions != nil { ds = ds.Where(directions) diff --git a/backend/pkg/api/data_access/vdb_blocks.go b/backend/pkg/api/data_access/vdb_blocks.go index f45ec2ac1..cb9e6ef86 100644 --- a/backend/pkg/api/data_access/vdb_blocks.go +++ b/backend/pkg/api/data_access/vdb_blocks.go @@ -33,9 +33,6 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das if err != nil { return nil, nil, err } - validators := goqu.T("users_val_dashboards_validators").As("validators") - blocks := goqu.T("blocks") - groups := goqu.T("goups") // TODO @LuccaBitfly move validation to handler? if cursor != "" { @@ -48,27 +45,34 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das searchGroup := regexp.MustCompile(`^[a-zA-Z0-9_\-.\ ]+$`).MatchString(search) searchIndex := regexp.MustCompile(`^[0-9]+$`).MatchString(search) - // ------------------------------------- - // Goqu Query: Determine validators filtered by search + validators := goqu.T("users_val_dashboards_validators").As("validators") + blocks := goqu.T("blocks") + groups := goqu.T("groups") + type validatorGroup struct { Validator t.VDBValidator `db:"validator_index"` Group uint64 `db:"group_id"` } + + // ------------------------------------- + // Goqu Query to determine validators filtered by search + var filteredValidatorsDs *goqu.SelectDataset var filteredValidators []validatorGroup - validatorsDs := goqu.Dialect("postgres"). + + filteredValidatorsDs = goqu.Dialect("postgres"). Select( "validator_index", ) if dashboardId.Validators == nil { - validatorsDs = validatorsDs. + filteredValidatorsDs = filteredValidatorsDs. From(validators). Where(validators.Col("dashboard_id").Eq(dashboardId.Id)) // apply search filters if searchIndex { - validatorsDs = validatorsDs.Where(validators.Col("validator_index").Eq(search)) + filteredValidatorsDs = filteredValidatorsDs.Where(validators.Col("validator_index").Eq(search)) } if searchGroup { - validatorsDs = validatorsDs. + filteredValidatorsDs = filteredValidatorsDs. InnerJoin(goqu.T("users_val_dashboards_groups").As(groups), goqu.On( validators.Col("group_id").Eq(groups.Col("id")), validators.Col("dashboard_id").Eq(groups.Col("dashboard_id")), @@ -84,7 +88,7 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das return make([]t.VDBBlocksTableRow, 0), &t.Paging{}, nil } - validatorsDs = validatorsDs. + filteredValidatorsDs = filteredValidatorsDs. Where(validators.Col("validator_index").Eq(index)) } } else { @@ -101,86 +105,18 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das break } } - validatorsDs = validatorsDs. + filteredValidatorsDs = filteredValidatorsDs. From( goqu.L("unnest(?)", pq.Array(filteredValidators)).As("validator_index"), ).As("validators") // TODO ? } - if dashboardId.Validators == nil { - validatorsQuery, validatorsArgs, err := validatorsDs.Prepared(true).ToSQL() - if err != nil { - return nil, nil, err - } - if err = d.alloyReader.SelectContext(ctx, &filteredValidators, validatorsQuery, validatorsArgs...); err != nil { - return nil, nil, err - } - } - if len(filteredValidators) == 0 { - return make([]t.VDBBlocksTableRow, 0), &t.Paging{}, nil - } - // ------------------------------------- - // Gather scheduled blocks - // found in dutiesInfo; pass results to final query later and let db do the sorting etc - validatorSet := make(map[t.VDBValidator]bool) - for _, v := range filteredValidators { - validatorSet[v.Validator] = true - } - var scheduledProposers []t.VDBValidator - var scheduledEpochs []uint64 - var scheduledSlots []uint64 - // don't need if requested slots are in the past - latestSlot := cache.LatestSlot.Get() - onlyPrimarySort := colSort.Column == enums.VDBBlockSlot || colSort.Column == enums.VDBBlockBlock - if !onlyPrimarySort || !currentCursor.IsValid() || - currentCursor.Slot > latestSlot+1 && currentCursor.Reverse != colSort.Desc || - currentCursor.Slot < latestSlot+1 && currentCursor.Reverse == colSort.Desc { - dutiesInfo, err := d.services.GetCurrentDutiesInfo() - if err == nil { - for slot, vali := range dutiesInfo.PropAssignmentsForSlot { - // only gather scheduled slots - if _, ok := dutiesInfo.SlotStatus[slot]; ok { - continue - } - // only gather slots scheduled for our validators - if _, ok := validatorSet[vali]; !ok { - continue - } - scheduledProposers = append(scheduledProposers, dutiesInfo.PropAssignmentsForSlot[slot]) - scheduledEpochs = append(scheduledEpochs, slot/utils.Config.Chain.ClConfig.SlotsPerEpoch) - scheduledSlots = append(scheduledSlots, slot) - } - } else { - log.Debugf("duties info not available, skipping scheduled slots: %s", err) - } - } - - // Sorting and pagination if cursor is present - defaultColumns := []t.SortColumn{ - {Column: enums.VDBBlocksColumns.Slot.ToString(), Table: blocks.GetTable(), Desc: true, Offset: currentCursor.Slot}, - } - var offset any - var table string - switch colSort.Column { - case enums.VDBBlocksColumns.Proposer: - offset = currentCursor.Proposer - case enums.VDBBlocksColumns.Block: - offset = currentCursor.Block - table = blocks.GetTable() - case enums.VDBBlocksColumns.Status: - offset = fmt.Sprintf("%d", currentCursor.Status) // type of 'status' column is text for some reason - case enums.VDBBlocksColumns.ProposerReward: - offset = currentCursor.Reward - } + // Constuct final query + var blocksDs *goqu.SelectDataset - order, directions := applySortAndPagination(defaultColumns, t.SortColumn{Column: colSort.Column.ToString(), Table: table, Desc: colSort.Desc, Offset: offset}, currentCursor.GenericCursor) - validatorsDs = validatorsDs.Order(order...) - if directions != nil { - validatorsDs = validatorsDs.Where(directions) - } - - validatorsDs = validatorsDs. + // 1. Tables + blocksDs = filteredValidatorsDs. InnerJoin(blocks, goqu.On( blocks.Col("proposer").Eq(validators.Col("validator_index")), )). @@ -205,7 +141,10 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das goqu.On( goqu.I("rb.exec_block_hash").Eq(blocks.Col("exec_block_hash")), ), - ). + ) + + // 2. Selects + blocksDs = blocksDs. SelectAppend( blocks.Col("epoch"), blocks.Col("slot"), @@ -215,64 +154,128 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das goqu.COALESCE(goqu.I("rb.proposer_fee_recipient"), blocks.Col("exec_fee_recipient")).As("fee_recipient"), goqu.COALESCE(goqu.L("rb.value / 1e18"), goqu.I("ep.fee_recipient_reward")).As("el_reward"), goqu.L("cp.cl_attestations_reward / 1e9 + cp.cl_sync_aggregate_reward / 1e9 + cp.cl_slashing_inclusion_reward / 1e9").As("cl_reward"), - ). - Limit(uint(limit + 1)) + ) - // Group id groupId := validators.Col("group_id") if dashboardId.Validators != nil { groupId = goqu.V(t.DefaultGroupId).As("group_id").GetAs() } - validatorsDs = validatorsDs.SelectAppend(groupId) + blocksDs = blocksDs.SelectAppend(groupId) - /* - if dashboardId.Validators == nil { - validatorsDs = validatorsDs.Select( - validators.Col("group_id"), - ) - } else { - validatorsDs = validatorsDs.Select( - goqu.L("?", t.DefaultGroupId).As("group_id"), - ) - }*/ + // 3. Sorting and pagination + defaultColumns := []t.SortColumn{ + {Column: enums.VDBBlocksColumns.Slot.ToExpr(), Desc: true, Offset: currentCursor.Slot}, + } + var offset any + switch colSort.Column { + case enums.VDBBlocksColumns.Proposer: + offset = currentCursor.Proposer + case enums.VDBBlocksColumns.Block: + offset = currentCursor.Block + if !currentCursor.Block.Valid { + offset = nil + } + case enums.VDBBlocksColumns.Status: + offset = fmt.Sprintf("%d", currentCursor.Status) // type of 'status' column is text for some reason + case enums.VDBBlocksColumns.ProposerReward: + offset = currentCursor.Reward + } - // union scheduled blocks if present - // WIP + order, directions := applySortAndPagination(defaultColumns, t.SortColumn{Column: colSort.Column.ToExpr(), Desc: colSort.Desc, Offset: offset}, currentCursor.GenericCursor) + blocksDs = goqu.From(blocksDs). // encapsulate so we can use selected fields + Order(order...) + if directions != nil { + blocksDs = blocksDs.Where(directions) + } - finalDs := validatorsDs - if len(scheduledProposers) > 0 { - scheduledDs := goqu.Dialect("postgres"). - From( - goqu.L("unnest(?, ?, ?) AS prov(validator_index, epoch, slot)", pq.Array(scheduledProposers), pq.Array(scheduledEpochs), pq.Array(scheduledSlots)), - ). - Select( - goqu.C("validator_index"), - goqu.C("epoch"), - goqu.C("slot"), - goqu.V("0").As("status"), - goqu.V(nil).As("exec_block_number"), - goqu.V(nil).As("fee_recipient"), - goqu.V(nil).As("el_reward"), - goqu.V(nil).As("cl_reward"), - goqu.V(nil).As("graffiti_text"), - ). - As("scheduled_blocks") + // 4. Limit + blocksDs = blocksDs.Limit(uint(limit + 1)) + + // 5. Gather and supply scheduled blocks to let db do the sorting etc + latestSlot := cache.LatestSlot.Get() + onlyPrimarySort := colSort.Column == enums.VDBBlockSlot + if !(onlyPrimarySort || colSort.Column == enums.VDBBlockBlock) || !currentCursor.IsValid() || + currentCursor.Slot > latestSlot+1 && currentCursor.Reverse != colSort.Desc || + currentCursor.Slot < latestSlot+1 && currentCursor.Reverse == colSort.Desc { + dutiesInfo, err := d.services.GetCurrentDutiesInfo() + if err == nil { + if dashboardId.Validators == nil { + // fetch filtered validators if not done yet + validatorsQuery, validatorsArgs, err := filteredValidatorsDs.Prepared(true).ToSQL() + if err != nil { + return nil, nil, err + } + if err = d.alloyReader.SelectContext(ctx, &filteredValidators, validatorsQuery, validatorsArgs...); err != nil { + return nil, nil, err + } + } + if len(filteredValidators) == 0 { + return make([]t.VDBBlocksTableRow, 0), &t.Paging{}, nil + } + + validatorSet := make(map[t.VDBValidator]bool) + for _, v := range filteredValidators { + validatorSet[v.Validator] = true + } + var scheduledProposers []t.VDBValidator + var scheduledEpochs []uint64 + var scheduledSlots []uint64 + // don't need if requested slots are in the past + for slot, vali := range dutiesInfo.PropAssignmentsForSlot { + // only gather scheduled slots + if _, ok := dutiesInfo.SlotStatus[slot]; ok { + continue + } + // only gather slots scheduled for our validators + if _, ok := validatorSet[vali]; !ok { + continue + } + scheduledProposers = append(scheduledProposers, dutiesInfo.PropAssignmentsForSlot[slot]) + scheduledEpochs = append(scheduledEpochs, slot/utils.Config.Chain.ClConfig.SlotsPerEpoch) + scheduledSlots = append(scheduledSlots, slot) + } - // distinct + block number ordering to filter out duplicates in an edge case (if dutiesInfo didn't update yet after a block was proposed, but the blocks table was) - // might be possible to remove this once the TODO in service_slot_viz.go:startSlotVizDataService is resolved - finalDs = validatorsDs. - Union(scheduledDs). - Where(directions). - Order(order...). - OrderAppend(goqu.C("exec_block_number").Desc().NullsLast()). - Limit(uint(limit + 1)). - Distinct(blocks.Col("slot")) - if !onlyPrimarySort { - finalDs = finalDs. - Distinct(blocks.Col("slot"), blocks.Col("exec_block_number")) + scheduledDs := goqu.Dialect("postgres"). + From( + goqu.L("unnest(?::int[], ?::int[], ?::int[]) AS prov(validator_index, epoch, slot)", pq.Array(scheduledProposers), pq.Array(scheduledEpochs), pq.Array(scheduledSlots)), + ). + Select( + goqu.C("validator_index"), + goqu.C("epoch"), + goqu.C("slot"), + goqu.V("0").As("status"), + goqu.V(nil).As("exec_block_number"), + goqu.V(nil).As("fee_recipient"), + goqu.V(nil).As("el_reward"), + goqu.V(nil).As("cl_reward"), + goqu.V(nil).As("graffiti_text"), + goqu.V(t.DefaultGroupId).As("group_id"), + ). + As("scheduled_blocks") + + // Supply to result query + // distinct + block number ordering to filter out duplicates in an edge case (if dutiesInfo didn't update yet after a block was proposed, but the blocks table was) + // might be possible to remove this once the TODO in service_slot_viz.go:startSlotVizDataService is resolved + blocksDs = goqu.Dialect("Postgres"). + From(blocksDs.Union(scheduledDs)). // wrap union to apply order + Order(order...). + OrderAppend(goqu.C("exec_block_number").Desc().NullsLast()). + Limit(uint(limit + 1)). + Distinct(enums.VDBBlocksColumns.Slot.ToExpr()) + if directions != nil { + blocksDs = blocksDs.Where(directions) + } + if !onlyPrimarySort { + blocksDs = blocksDs. + Distinct(colSort.Column.ToExpr(), enums.VDBBlocksColumns.Slot.ToExpr()) + } + } else { + log.Warnf("Error getting scheduled proposals, DutiesInfo not available in Redis: %s", err) } } + // ------------------------------------- + // Execute query var proposals []struct { Proposer t.VDBValidator `db:"validator_index"` Group uint64 `db:"group_id"` @@ -283,13 +286,13 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das FeeRecipient []byte `db:"fee_recipient"` ElReward decimal.NullDecimal `db:"el_reward"` ClReward decimal.NullDecimal `db:"cl_reward"` - GraffitiText string `db:"graffiti_text"` + GraffitiText sql.NullString `db:"graffiti_text"` // for cursor only Reward decimal.Decimal } startTime := time.Now() - query, args, err := finalDs.Prepared(true).ToSQL() + query, args, err := blocksDs.Prepared(true).ToSQL() if err != nil { return nil, nil, err } @@ -301,6 +304,9 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das if len(proposals) == 0 { return make([]t.VDBBlocksTableRow, 0), &t.Paging{}, nil } + + // ------------------------------------- + // Prepare result moreDataFlag := len(proposals) > int(limit) if moreDataFlag { proposals = proposals[:len(proposals)-1] @@ -335,10 +341,14 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das if proposal.Status == 0 || proposal.Status == 2 { continue } - graffiti := proposal.GraffitiText - data[i].Graffiti = &graffiti - block := uint64(proposal.Block.Int64) - data[i].Block = &block + if proposal.GraffitiText.Valid { + graffiti := proposal.GraffitiText.String + data[i].Graffiti = &graffiti + } + if proposal.Block.Valid { + block := uint64(proposal.Block.Int64) + data[i].Block = &block + } if proposal.Status == 3 { continue } diff --git a/backend/pkg/api/enums/notifications_enums.go b/backend/pkg/api/enums/notifications_enums.go index 1fb78529e..9af65cb1a 100644 --- a/backend/pkg/api/enums/notifications_enums.go +++ b/backend/pkg/api/enums/notifications_enums.go @@ -1,5 +1,7 @@ package enums +import "github.com/doug-martin/goqu/v9" + // ------------------------------------------------------------ // Notifications Dashboard Table Columns @@ -34,22 +36,22 @@ func (NotificationDashboardsColumn) NewFromString(s string) NotificationDashboar } // internal use, used to map to query column names -func (c NotificationDashboardsColumn) ToString() string { +func (c NotificationDashboardsColumn) ToExpr() OrderableSortable { switch c { case NotificationDashboardChainId: - return "chain_id" + return goqu.C("chain_id") case NotificationDashboardEpoch: - return "epoch" + return goqu.C("epoch") case NotificationDashboardDashboardName: - return "dashboard_name" + return goqu.C("dashboard_name") case NotificationDashboardDashboardId: - return "dashboard_id" + return goqu.C("dashboard_id") case NotificationDashboardGroupName: - return "group_name" + return goqu.C("group_name") case NotificationDashboardGroupId: - return "group_id" + return goqu.C("group_id") default: - return "" + return nil } } @@ -104,20 +106,20 @@ func (NotificationMachinesColumn) NewFromString(s string) NotificationMachinesCo } // internal use, used to map to query column names -func (c NotificationMachinesColumn) ToString() string { +func (c NotificationMachinesColumn) ToExpr() OrderableSortable { switch c { case NotificationMachineId: - return "machine_id" + return goqu.C("machine_id") case NotificationMachineName: - return "machine_name" + return goqu.C("machine_name") case NotificationMachineThreshold: - return "threshold" + return goqu.C("threshold") case NotificationMachineEventType: - return "event_type" + return goqu.C("event_type") case NotificationMachineTimestamp: - return "epoch" + return goqu.C("epoch") default: - return "" + return nil } } @@ -163,14 +165,14 @@ func (NotificationClientsColumn) NewFromString(s string) NotificationClientsColu } // internal use, used to map to query column names -func (c NotificationClientsColumn) ToString() string { +func (c NotificationClientsColumn) ToExpr() OrderableSortable { switch c { case NotificationClientName: - return "client_name" + return goqu.C("client_name") case NotificationClientTimestamp: - return "epoch" + return goqu.C("epoch") default: - return "" + return nil } } @@ -251,16 +253,16 @@ func (NotificationNetworksColumn) NewFromString(s string) NotificationNetworksCo } // internal use, used to map to query column names -func (c NotificationNetworksColumn) ToString() string { +func (c NotificationNetworksColumn) ToExpr() OrderableSortable { switch c { case NotificationNetworkTimestamp: - return "epoch" + return goqu.C("epoch") case NotificationNetworkNetwork: - return "network" + return goqu.C("network") case NotificationNetworkEventType: - return "event_type" + return goqu.C("event_type") default: - return "" + return nil } } diff --git a/backend/pkg/api/enums/validator_dashboard_enums.go b/backend/pkg/api/enums/validator_dashboard_enums.go index 228b224dc..c241ef98e 100644 --- a/backend/pkg/api/enums/validator_dashboard_enums.go +++ b/backend/pkg/api/enums/validator_dashboard_enums.go @@ -1,5 +1,10 @@ package enums +import ( + "github.com/doug-martin/goqu/v9" + "github.com/doug-martin/goqu/v9/exp" +) + // ---------------- // Validator Dashboard Summary Table @@ -156,20 +161,26 @@ func (VDBBlocksColumn) NewFromString(s string) VDBBlocksColumn { } } -func (c VDBBlocksColumn) ToString() string { +type OrderableSortable interface { + exp.Orderable + exp.Comparable + exp.Isable +} + +func (c VDBBlocksColumn) ToExpr() OrderableSortable { switch c { case VDBBlockProposer: - return "proposer" + return goqu.C("validator_index") case VDBBlockSlot: - return "slot" + return goqu.C("slot") case VDBBlockBlock: - return "exec_block_number" + return goqu.C("exec_block_number") case VDBBlockStatus: - return "status" + return goqu.C("status") case VDBBlockProposerReward: - return "reward" + return goqu.L("el_reward + cl_reward") default: - return "" + return nil } } diff --git a/backend/pkg/api/types/data_access.go b/backend/pkg/api/types/data_access.go index 70961c5f2..6bcf30472 100644 --- a/backend/pkg/api/types/data_access.go +++ b/backend/pkg/api/types/data_access.go @@ -4,8 +4,6 @@ import ( "database/sql" "time" - "github.com/doug-martin/goqu/v9" - "github.com/doug-martin/goqu/v9/exp" "github.com/gobitfly/beaconchain/pkg/api/enums" "github.com/gobitfly/beaconchain/pkg/consapi/types" "github.com/gobitfly/beaconchain/pkg/monitoring/constants" @@ -26,18 +24,10 @@ type Sort[T enums.Enum] struct { } type SortColumn struct { - Column string - Table string // optional + // defaults + Column enums.OrderableSortable Desc bool - // represents value from cursor - Offset any -} - -func (s SortColumn) Expr() exp.IdentifierExpression { - if s.Table != "" { - return goqu.T(s.Table).Col(s.Column) - } - return goqu.C(s.Column) + Offset any // nil to indicate null value } type VDBIdPrimary int From 244281ac81126ca9fbf63ce64a3d42a4fdc31556 Mon Sep 17 00:00:00 2001 From: marcel-bitfly <174338434+marcel-bitfly@users.noreply.github.com> Date: Mon, 21 Oct 2024 14:10:54 +0200 Subject: [PATCH 064/124] fix(NotificationsManagementNetwork): `participation rate` --- frontend/.vscode/settings.json | 3 ++- .../management/NotificationsManagementNetwork.vue | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/frontend/.vscode/settings.json b/frontend/.vscode/settings.json index 4e60da420..fa5575cd5 100644 --- a/frontend/.vscode/settings.json +++ b/frontend/.vscode/settings.json @@ -33,7 +33,8 @@ "qrCode", "useNotificationsOverviewStore", "useWindowSize", - "vscode" + "vscode", + "NotificationsManagementNetwork" ], "editor.codeActionsOnSave": { "source.fixAll.eslint": "always" diff --git a/frontend/components/notifications/management/NotificationsManagementNetwork.vue b/frontend/components/notifications/management/NotificationsManagementNetwork.vue index ddc6be294..f07e94187 100644 --- a/frontend/components/notifications/management/NotificationsManagementNetwork.vue +++ b/frontend/components/notifications/management/NotificationsManagementNetwork.vue @@ -30,9 +30,9 @@ watchDebounced([ ], async () => { if (!currentNetworkSettings.value) return currentNetworkSettings.value.is_gas_above_subscribed = hasGasAbove.value - currentNetworkSettings.value.gas_above_threshold = thresholdGasAbove.value currentNetworkSettings.value.is_gas_below_subscribed = hasGasBelow.value currentNetworkSettings.value.is_new_reward_round_subscribed = hasNewRewardRound.value + currentNetworkSettings.value.is_participation_rate_subscribed = hasParticipationRate.value currentNetworkSettings.value.gas_above_threshold = formatToWei(thresholdGasAbove.value, { from: 'gwei' }) currentNetworkSettings.value.gas_below_threshold = formatToWei(thresholdGasBelow.value, { from: 'gwei' }) From 3d4b61409466bfe9428f4b3299a373dc74ef9913 Mon Sep 17 00:00:00 2001 From: remoterami <142154971+remoterami@users.noreply.github.com> Date: Mon, 21 Oct 2024 14:11:46 +0200 Subject: [PATCH 065/124] goqu quirck workaround --- backend/pkg/api/data_access/vdb_blocks.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/backend/pkg/api/data_access/vdb_blocks.go b/backend/pkg/api/data_access/vdb_blocks.go index cb9e6ef86..00495e265 100644 --- a/backend/pkg/api/data_access/vdb_blocks.go +++ b/backend/pkg/api/data_access/vdb_blocks.go @@ -162,7 +162,10 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das } blocksDs = blocksDs.SelectAppend(groupId) - // 3. Sorting and pagination + // 3. Limit + blocksDs = blocksDs.Limit(uint(limit + 1)) + + // 4. Sorting and pagination defaultColumns := []t.SortColumn{ {Column: enums.VDBBlocksColumns.Slot.ToExpr(), Desc: true, Offset: currentCursor.Slot}, } @@ -188,9 +191,6 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das blocksDs = blocksDs.Where(directions) } - // 4. Limit - blocksDs = blocksDs.Limit(uint(limit + 1)) - // 5. Gather and supply scheduled blocks to let db do the sorting etc latestSlot := cache.LatestSlot.Get() onlyPrimarySort := colSort.Column == enums.VDBBlockSlot From 90603d09abf7c8577110c08f1acad29056ef5dbd Mon Sep 17 00:00:00 2001 From: remoterami <142154971+remoterami@users.noreply.github.com> Date: Mon, 21 Oct 2024 14:29:05 +0200 Subject: [PATCH 066/124] fixed column name --- backend/pkg/api/enums/notifications_enums.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/pkg/api/enums/notifications_enums.go b/backend/pkg/api/enums/notifications_enums.go index 1fb78529e..033141edd 100644 --- a/backend/pkg/api/enums/notifications_enums.go +++ b/backend/pkg/api/enums/notifications_enums.go @@ -166,7 +166,7 @@ func (NotificationClientsColumn) NewFromString(s string) NotificationClientsColu func (c NotificationClientsColumn) ToString() string { switch c { case NotificationClientName: - return "client_name" + return "client" case NotificationClientTimestamp: return "epoch" default: From 73f0f264a2bc9a14fde08798d38d1e7686ad99de Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Mon, 21 Oct 2024 12:31:07 +0000 Subject: [PATCH 067/124] fix(notifications): scale withdrawal amounts --- backend/pkg/api/data_access/notifications.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/pkg/api/data_access/notifications.go b/backend/pkg/api/data_access/notifications.go index 2bf6d619f..b0819ac73 100644 --- a/backend/pkg/api/data_access/notifications.go +++ b/backend/pkg/api/data_access/notifications.go @@ -591,7 +591,7 @@ func (d *DataAccessService) GetValidatorDashboardNotificationDetails(ctx context addressMapping[hexutil.Encode(curNotification.Address)] = &addr notificationDetails.Withdrawal = append(notificationDetails.Withdrawal, t.NotificationEventWithdrawal{ Index: curNotification.ValidatorIndex, - Amount: decimal.NewFromUint64(curNotification.Amount), + Amount: decimal.NewFromUint64(curNotification.Amount).Mul(decimal.NewFromFloat(params.GWei)), // Amounts have to be in WEI Address: addr, }) case types.NetworkLivenessIncreasedEventName, From 7ba5d3ec86e74e5ebdfbb92e513e386b364971d3 Mon Sep 17 00:00:00 2001 From: Stefan Pletka <124689083+Eisei24@users.noreply.github.com> Date: Mon, 21 Oct 2024 14:51:13 +0200 Subject: [PATCH 068/124] Adjusted the event name for ethereum/mainnet chainId --- backend/pkg/api/data_access/dummy.go | 15 +++++++++------ backend/pkg/api/data_access/networks.go | 15 +++++++++------ backend/pkg/api/data_access/notifications.go | 8 ++++---- backend/pkg/api/types/data_access.go | 5 +++-- 4 files changed, 25 insertions(+), 18 deletions(-) diff --git a/backend/pkg/api/data_access/dummy.go b/backend/pkg/api/data_access/dummy.go index 7482328c0..24472343a 100644 --- a/backend/pkg/api/data_access/dummy.go +++ b/backend/pkg/api/data_access/dummy.go @@ -395,16 +395,19 @@ func (d *DummyService) GetValidatorDashboardRocketPoolMinipools(ctx context.Cont func (d *DummyService) GetAllNetworks() ([]t.NetworkInfo, error) { return []types.NetworkInfo{ { - ChainId: 1, - Name: "ethereum", + ChainId: 1, + Name: "ethereum", + NotificationsName: "mainnet", }, { - ChainId: 100, - Name: "gnosis", + ChainId: 100, + Name: "gnosis", + NotificationsName: "gnosis", }, { - ChainId: 17000, - Name: "holesky", + ChainId: 17000, + Name: "holesky", + NotificationsName: "holesky", }, }, nil } diff --git a/backend/pkg/api/data_access/networks.go b/backend/pkg/api/data_access/networks.go index df95fb008..30c0669ec 100644 --- a/backend/pkg/api/data_access/networks.go +++ b/backend/pkg/api/data_access/networks.go @@ -12,16 +12,19 @@ func (d *DataAccessService) GetAllNetworks() ([]types.NetworkInfo, error) { return []types.NetworkInfo{ { - ChainId: 1, - Name: "ethereum", + ChainId: 1, + Name: "ethereum", + NotificationsName: "mainnet", }, { - ChainId: 100, - Name: "gnosis", + ChainId: 100, + Name: "gnosis", + NotificationsName: "gnosis", }, { - ChainId: 17000, - Name: "holesky", + ChainId: 17000, + Name: "holesky", + NotificationsName: "holesky", }, }, nil } diff --git a/backend/pkg/api/data_access/notifications.go b/backend/pkg/api/data_access/notifications.go index b0819ac73..a99a289be 100644 --- a/backend/pkg/api/data_access/notifications.go +++ b/backend/pkg/api/data_access/notifications.go @@ -223,7 +223,7 @@ func (d *DataAccessService) GetNotificationOverview(ctx context.Context, userId if len(whereNetwork) > 0 { whereNetwork += " OR " } - whereNetwork += "event_name like '" + network.Name + ":rocketpool_%' OR event_name like '" + network.Name + ":network_%'" + whereNetwork += "event_name like '" + network.NotificationsName + ":rocketpool_%' OR event_name like '" + network.NotificationsName + ":network_%'" } query := goqu.Dialect("postgres"). @@ -1231,7 +1231,7 @@ func (d *DataAccessService) GetNotificationSettings(ctx context.Context, userId } networksSettings := make(map[string]*t.NotificationNetwork, len(networks)) for _, network := range networks { - networksSettings[network.Name] = &t.NotificationNetwork{ + networksSettings[network.NotificationsName] = &t.NotificationNetwork{ ChainId: network.ChainId, Settings: t.NotificationSettingsNetwork{ GasAboveThreshold: decimal.NewFromFloat(GasAboveThresholdDefault).Mul(decimal.NewFromInt(params.GWei)), @@ -1566,7 +1566,7 @@ func (d *DataAccessService) UpdateNotificationSettingsNetworks(ctx context.Conte networkName := "" for _, network := range networks { if network.ChainId == chainId { - networkName = network.Name + networkName = network.NotificationsName break } } @@ -2083,7 +2083,7 @@ func (d *DataAccessService) UpdateNotificationSettingsValidatorDashboard(ctx con networkName := "" for _, network := range networks { if network.ChainId == chainId { - networkName = network.Name + networkName = network.NotificationsName break } } diff --git a/backend/pkg/api/types/data_access.go b/backend/pkg/api/types/data_access.go index 229677a9b..66e4b5fb8 100644 --- a/backend/pkg/api/types/data_access.go +++ b/backend/pkg/api/types/data_access.go @@ -186,8 +186,9 @@ type NotificationsDashboardsCursor struct { } type NetworkInfo struct { - ChainId uint64 - Name string + ChainId uint64 + Name string + NotificationsName string } type ClientInfo struct { From 7d40e41145f05201e781aabbe11954675e06155f Mon Sep 17 00:00:00 2001 From: marcel-bitfly <174338434+marcel-bitfly@users.noreply.github.com> Date: Mon, 21 Oct 2024 15:48:24 +0200 Subject: [PATCH 069/124] refactor(NotificationsTableEmpty): rename `component` Component is also used for other tables. --- frontend/.vscode/settings.json | 5 +++-- .../components/notifications/NotificationsClientsTable.vue | 2 +- .../notifications/NotificationsDashboardsTable.vue | 2 +- .../components/notifications/NotificationsMachinesTable.vue | 2 +- .../components/notifications/NotificationsNetworkTable.vue | 2 +- ...{DashboardsTableEmpty.vue => NotificationsTableEmpty.vue} | 4 +++- 6 files changed, 10 insertions(+), 7 deletions(-) rename frontend/components/notifications/{DashboardsTableEmpty.vue => NotificationsTableEmpty.vue} (96%) diff --git a/frontend/.vscode/settings.json b/frontend/.vscode/settings.json index fa5575cd5..6c8cfe14e 100644 --- a/frontend/.vscode/settings.json +++ b/frontend/.vscode/settings.json @@ -11,10 +11,12 @@ "NotificationsDashboardDialogEntity", "NotificationsDashboardTable", "NotificationsManagementModalWebhook", + "NotificationsManagementNetwork", "NotificationsManagementSubscriptionDialog", "NotificationsManagmentMachines", "NotificationsNetworkTable", "NotificationsOverview", + "NotificationsTableEmpty", "a11y", "checkout", "ci", @@ -33,8 +35,7 @@ "qrCode", "useNotificationsOverviewStore", "useWindowSize", - "vscode", - "NotificationsManagementNetwork" + "vscode" ], "editor.codeActionsOnSave": { "source.fixAll.eslint": "always" diff --git a/frontend/components/notifications/NotificationsClientsTable.vue b/frontend/components/notifications/NotificationsClientsTable.vue index 19c1f716f..d24d3fae7 100644 --- a/frontend/components/notifications/NotificationsClientsTable.vue +++ b/frontend/components/notifications/NotificationsClientsTable.vue @@ -89,7 +89,7 @@ const { overview } = useNotificationsDashboardOverviewStore() From 34df50119dd157f49c53b3275ef4889ae94c5d8c Mon Sep 17 00:00:00 2001 From: Patrick Date: Tue, 22 Oct 2024 10:24:15 +0200 Subject: [PATCH 072/124] fix(ci): only run converted-types-check if types changed (#1006) BEDS-90 --- .github/workflows/backend-converted-types-check.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/backend-converted-types-check.yml b/.github/workflows/backend-converted-types-check.yml index c6f09fdac..a59593a02 100644 --- a/.github/workflows/backend-converted-types-check.yml +++ b/.github/workflows/backend-converted-types-check.yml @@ -2,14 +2,14 @@ name: Backend-Converted-Types-Check on: push: paths: - - 'backend/**' + - 'backend/pkg/api/types/**' - 'frontend/types/api/**' branches: - main - staging pull_request: paths: - - 'backend/**' + - 'backend/pkg/api/types/**' - 'frontend/types/api/**' branches: - '*' @@ -41,6 +41,7 @@ jobs: newHash=$(find ../frontend/types/api -type f -print0 | sort -z | xargs -0 sha1sum | sha256sum | head -c 64) if [ "$currHash" != "$newHash" ]; then echo "frontend-types have changed, please commit the changes" + git diff --stat exit 1 fi From 306bfd56e88dd5316f0cceb3a63e565c5ea91ebf Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Tue, 22 Oct 2024 08:35:24 +0000 Subject: [PATCH 073/124] feat(notifications): add upcoming block proposal notification collection --- backend/pkg/api/data_access/notifications.go | 1 + ...ead_notification_status_tracking_table.sql | 16 ++ backend/pkg/notification/collection.go | 203 ++++++++++-------- backend/pkg/notification/db.go | 7 +- 4 files changed, 137 insertions(+), 90 deletions(-) create mode 100644 backend/pkg/commons/db/migrations/postgres/20241022072552_head_notification_status_tracking_table.sql diff --git a/backend/pkg/api/data_access/notifications.go b/backend/pkg/api/data_access/notifications.go index 781414571..f9756df96 100644 --- a/backend/pkg/api/data_access/notifications.go +++ b/backend/pkg/api/data_access/notifications.go @@ -63,6 +63,7 @@ func (*DataAccessService) registerNotificationInterfaceTypes() { var once sync.Once once.Do(func() { gob.Register(&n.ValidatorProposalNotification{}) + gob.Register(&n.ValidatorUpcomingProposalNotification{}) gob.Register(&n.ValidatorAttestationNotification{}) gob.Register(&n.ValidatorIsOfflineNotification{}) gob.Register(&n.ValidatorGotSlashedNotification{}) diff --git a/backend/pkg/commons/db/migrations/postgres/20241022072552_head_notification_status_tracking_table.sql b/backend/pkg/commons/db/migrations/postgres/20241022072552_head_notification_status_tracking_table.sql new file mode 100644 index 000000000..637704a10 --- /dev/null +++ b/backend/pkg/commons/db/migrations/postgres/20241022072552_head_notification_status_tracking_table.sql @@ -0,0 +1,16 @@ +-- +goose Up +-- +goose StatementBegin +SELECT 'creating epochs_notified_head table'; +CREATE TABLE IF NOT EXISTS epochs_notified_head ( + epoch INTEGER NOT NULL, + event_name VARCHAR(255) NOT NULL, + senton TIMESTAMP WITHOUT TIME ZONE NOT NULL, + PRIMARY KEY (epoch, event_name) +); +-- +goose StatementEnd + +-- +goose Down +-- +goose StatementBegin +SELECT 'dropping epochs_notified_head table'; +DROP TABLE IF EXISTS epochs_notified_head; +-- +goose StatementEnd diff --git a/backend/pkg/notification/collection.go b/backend/pkg/notification/collection.go index a50c21326..720d78227 100644 --- a/backend/pkg/notification/collection.go +++ b/backend/pkg/notification/collection.go @@ -46,6 +46,7 @@ func notificationCollector() { var once sync.Once once.Do(func() { gob.Register(&ValidatorProposalNotification{}) + gob.Register(&ValidatorUpcomingProposalNotification{}) gob.Register(&ValidatorAttestationNotification{}) gob.Register(&ValidatorIsOfflineNotification{}) gob.Register(&ValidatorGotSlashedNotification{}) @@ -58,6 +59,56 @@ func notificationCollector() { gob.Register(&SyncCommitteeSoonNotification{}) }) + go func() { + log.Infof("starting head notification collector") + mc, err := modules.GetModuleContext() + if err != nil { + log.Fatal(err, "error getting module context", 0) + } + + for ; ; time.Sleep(time.Second * 30) { + // get the head epoch + head, err := mc.ConsClient.GetChainHead() + if err != nil { + log.Error(err, "error getting chain head", 0) + continue + } + + headEpoch := head.HeadEpoch + + var lastNotifiedEpoch uint64 + err = db.WriterDb.Get(&lastNotifiedEpoch, "SELECT COUNT(*) FROM epochs_notified_head WHERE epoch = $1 AND event_name = $2", headEpoch, types.ValidatorUpcomingProposalEventName) + + if err != nil { + log.Error(err, fmt.Sprintf("error checking if upcoming block proposal notifications for epoch %v have already been collected", headEpoch), 0) + continue + } + + if lastNotifiedEpoch > 0 { + log.Warnf("head epoch notifications for epoch %v have already been collected", headEpoch) + continue + } + + notifications, err := collectHeadNotifications(mc, headEpoch) + if err != nil { + log.Error(err, "error collecting head notifications", 0) + } + + _, err = db.WriterDb.Exec("INSERT INTO epochs_notified_head (epoch, event_name, senton) VALUES ($1, $2, NOW())", headEpoch, types.ValidatorUpcomingProposalEventName) + if err != nil { + log.Error(err, "error marking head notification status for epoch in db", 0) + continue + } + + if len(notifications) > 0 { + err = queueNotifications(headEpoch, notifications) + if err != nil { + log.Error(err, "error queuing head notifications", 0) + } + } + } + }() + for { latestFinalizedEpoch := cache.LatestFinalizedEpoch.Get() @@ -158,6 +209,73 @@ func notificationCollector() { } } +func collectHeadNotifications(mc modules.ModuleContext, headEpoch uint64) (types.NotificationsPerUserId, error) { + notificationsByUserID := types.NotificationsPerUserId{} + start := time.Now() + err := collectUpcomingBlockProposalNotifications(notificationsByUserID, mc, headEpoch) + if err != nil { + metrics.Errors.WithLabelValues("notifications_collect_upcoming_block_proposal").Inc() + return nil, fmt.Errorf("error collecting upcoming block proposal notifications: %v", err) + } + log.Infof("collecting upcoming block proposal notifications took: %v", time.Since(start)) + + return notificationsByUserID, nil +} + +func collectUpcomingBlockProposalNotifications(notificationsByUserID types.NotificationsPerUserId, mc modules.ModuleContext, headEpoch uint64) (err error) { + nextEpoch := headEpoch + 1 + log.Infof("collecting upcoming block proposal notifications for epoch %v (head epoch is %d)", nextEpoch, headEpoch) + + if utils.EpochToTime(nextEpoch).Before(time.Now()) { + log.Error(fmt.Errorf("error upcoming block proposal notifications for epoch %v are already in the past", nextEpoch), "", 0) + return nil + } + + assignments, err := mc.CL.GetPropoalAssignments(nextEpoch) + if err != nil { + return fmt.Errorf("error getting proposal assignments: %w", err) + } + + subs, err := GetSubsForEventFilter(types.ValidatorUpcomingProposalEventName, "", nil, nil) + if err != nil { + return fmt.Errorf("error getting subscriptions for upcoming block proposal notifications: %w", err) + } + + log.Infof("retrieved %d subscriptions for upcoming block proposal notifications", len(subs)) + if len(subs) == 0 { + return nil + } + + for _, assignment := range assignments.Data { + log.Infof("upcoming block proposal for validator %d in slot %d", assignment.ValidatorIndex, assignment.Slot) + for _, sub := range subs[hex.EncodeToString(assignment.Pubkey)] { + if sub.UserID == nil || sub.ID == nil { + return fmt.Errorf("error expected userId and subId to be defined but got user: %v, sub: %v", sub.UserID, sub.ID) + } + + log.Infof("creating %v notification for validator %v in epoch %v (dashboard: %v)", sub.EventName, assignment.ValidatorIndex, nextEpoch, sub.DashboardId != nil) + n := &ValidatorUpcomingProposalNotification{ + NotificationBaseImpl: types.NotificationBaseImpl{ + SubscriptionID: *sub.ID, + UserID: *sub.UserID, + Epoch: nextEpoch, + EventName: sub.EventName, + EventFilter: hex.EncodeToString(assignment.Pubkey), + DashboardId: sub.DashboardId, + DashboardName: sub.DashboardName, + DashboardGroupId: sub.DashboardGroupId, + DashboardGroupName: sub.DashboardGroupName, + }, + ValidatorIndex: assignment.ValidatorIndex, + Slot: uint64(assignment.Slot), + } + notificationsByUserID.AddNotification(n) + metrics.NotificationsCollected.WithLabelValues(string(n.GetEventName())).Inc() + } + } + return nil +} + func collectNotifications(epoch uint64) (types.NotificationsPerUserId, error) { notificationsByUserID := types.NotificationsPerUserId{} start := time.Now() @@ -189,13 +307,6 @@ func collectNotifications(epoch uint64) (types.NotificationsPerUserId, error) { // The following functions will collect the notifications and add them to the // notificationsByUserID map. The notifications will be queued and sent later // by the notification sender process - err = collectUpcomingBlockProposalNotifications(notificationsByUserID) - if err != nil { - metrics.Errors.WithLabelValues("notifications_collect_upcoming_block_proposal").Inc() - return nil, fmt.Errorf("error collecting upcoming block proposal notifications: %v", err) - } - log.Infof("collecting attestation & offline notifications took: %v", time.Since(start)) - err = collectAttestationAndOfflineValidatorNotifications(notificationsByUserID, epoch) if err != nil { metrics.Errors.WithLabelValues("notifications_collect_missed_attestation").Inc() @@ -348,84 +459,6 @@ func collectUserDbNotifications(epoch uint64) (types.NotificationsPerUserId, err return notificationsByUserID, nil } -func collectUpcomingBlockProposalNotifications(notificationsByUserID types.NotificationsPerUserId) (err error) { - mc, err := modules.GetModuleContext() - if err != nil { - return fmt.Errorf("error getting module context: %w", err) - } - - // get the head epoch - head, err := mc.ConsClient.GetChainHead() - if err != nil { - return fmt.Errorf("error getting chain head: %w", err) - } - - headEpoch := head.HeadEpoch - nextEpoch := headEpoch + 1 - - var lastNotifiedEpoch uint64 - err = db.WriterDb.Get(&lastNotifiedEpoch, "SELECT COUNT(*) FROM epochs_notified_head WHERE epoch = $1 AND event_name = $2", nextEpoch, types.ValidatorUpcomingProposalEventName) - - if err != nil { - return fmt.Errorf("error checking if upcoming block proposal notifications for epoch %v have already been collected: %w", nextEpoch, err) - } - - if lastNotifiedEpoch > 0 { - log.Error(fmt.Errorf("upcoming block proposal notifications for epoch %v have already been collected", nextEpoch), "", 0) - return nil - } - - // todo: make sure not to collect notifications for the same epoch twice - assignments, err := mc.CL.GetPropoalAssignments(nextEpoch) - if err != nil { - return fmt.Errorf("error getting proposal assignments: %w", err) - } - - subs, err := GetSubsForEventFilter(types.ValidatorUpcomingProposalEventName, "", nil, nil) - if err != nil { - return fmt.Errorf("error getting subscriptions for upcoming block proposal notifications: %w", err) - } - - log.Infof("retrieved %d subscriptions for upcoming block proposal notifications", len(subs)) - if len(subs) == 0 { - return nil - } - - for _, assignment := range assignments.Data { - log.Infof("upcoming block proposal for validator %d in slot %d", assignment.ValidatorIndex, assignment.Slot) - for _, sub := range subs[hex.EncodeToString(assignment.Pubkey)] { - if sub.UserID == nil || sub.ID == nil { - return fmt.Errorf("error expected userId and subId to be defined but got user: %v, sub: %v", sub.UserID, sub.ID) - } - - log.Infof("creating %v notification for validator %v in epoch %v (dashboard: %v)", sub.EventName, assignment.ValidatorIndex, nextEpoch, sub.DashboardId != nil) - n := &ValidatorUpcomingProposalNotification{ - NotificationBaseImpl: types.NotificationBaseImpl{ - SubscriptionID: *sub.ID, - UserID: *sub.UserID, - Epoch: nextEpoch, - EventName: sub.EventName, - EventFilter: hex.EncodeToString(assignment.Pubkey), - DashboardId: sub.DashboardId, - DashboardName: sub.DashboardName, - DashboardGroupId: sub.DashboardGroupId, - DashboardGroupName: sub.DashboardGroupName, - }, - ValidatorIndex: assignment.ValidatorIndex, - Slot: uint64(assignment.Slot), - } - notificationsByUserID.AddNotification(n) - metrics.NotificationsCollected.WithLabelValues(string(n.GetEventName())).Inc() - } - } - - _, err = db.WriterDb.Exec("INSERT INTO epochs_notified_head (epoch, event_name) VALUES ($1, $2)", nextEpoch, types.ValidatorUpcomingProposalEventName) - if err != nil { - return fmt.Errorf("error marking notification status for epoch %v in db: %w", nextEpoch, err) - } - return nil -} - func collectBlockProposalNotifications(notificationsByUserID types.NotificationsPerUserId, status uint64, eventName types.EventName, epoch uint64) error { type dbResult struct { Proposer uint64 `db:"proposer"` diff --git a/backend/pkg/notification/db.go b/backend/pkg/notification/db.go index 4642d8373..aff13d5a0 100644 --- a/backend/pkg/notification/db.go +++ b/backend/pkg/notification/db.go @@ -67,14 +67,11 @@ func GetSubsForEventFilter(eventName types.EventName, lastSentFilter string, las ds = ds.Where(goqu.L("(event_filter = ANY(?))", pq.StringArray(eventFilters))) } - query, args, err := ds.Prepared(false).ToSQL() + query, args, err := ds.Prepared(true).ToSQL() if err != nil { return nil, err } - log.Info(query) - log.Info(args) - subMap := make(map[string][]*types.Subscription, 0) err = db.FrontendWriterDB.Select(&subs, query, args...) if err != nil { @@ -116,7 +113,7 @@ func GetSubsForEventFilter(eventName types.EventName, lastSentFilter string, las } if len(dashboardConfigsToFetch) > 0 { - log.Infof("fetching dashboard configurations for %d dashboards", len(dashboardConfigsToFetch)) + log.Infof("fetching dashboard configurations for %d dashboards (%v)", len(dashboardConfigsToFetch), dashboardConfigsToFetch) dashboardConfigRetrievalStartTs := time.Now() type dashboardDefinitionRow struct { DashboardId types.DashboardId `db:"dashboard_id"` From 513a79c0a2505aa8544cd0dbf6aa6cf3a18b40b1 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Tue, 22 Oct 2024 08:52:43 +0000 Subject: [PATCH 074/124] fix(notification): add labels for upcoming block proposal notifications --- backend/pkg/commons/types/frontend.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/backend/pkg/commons/types/frontend.go b/backend/pkg/commons/types/frontend.go index 0ef4bde69..b45f1b06d 100644 --- a/backend/pkg/commons/types/frontend.go +++ b/backend/pkg/commons/types/frontend.go @@ -120,6 +120,7 @@ const ( ) var EventSortOrder = []EventName{ + ValidatorUpcomingProposalEventName, ValidatorGotSlashedEventName, ValidatorDidSlashEventName, ValidatorMissedProposalEventName, @@ -173,6 +174,7 @@ var MachineEventsMap = map[EventName]struct{}{ } var LegacyEventLabel map[EventName]string = map[EventName]string{ + ValidatorUpcomingProposalEventName: "Your validator(s) will soon propose a block", ValidatorMissedProposalEventName: "Your validator(s) missed a proposal", ValidatorExecutedProposalEventName: "Your validator(s) submitted a proposal", ValidatorMissedAttestationEventName: "Your validator(s) missed an attestation", @@ -196,6 +198,7 @@ var LegacyEventLabel map[EventName]string = map[EventName]string{ } var EventLabel map[EventName]string = map[EventName]string{ + ValidatorUpcomingProposalEventName: "Upcoming block proposal", ValidatorMissedProposalEventName: "Block proposal missed", ValidatorExecutedProposalEventName: "Block proposal submitted", ValidatorMissedAttestationEventName: "Attestation missed", From 537fc235fffd9c51b864033855bcc6b1ec1afa11 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Tue, 22 Oct 2024 08:53:51 +0000 Subject: [PATCH 075/124] fix(notifications): add online notification type back to registry --- backend/pkg/api/data_access/notifications.go | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/pkg/api/data_access/notifications.go b/backend/pkg/api/data_access/notifications.go index ec2c253e8..1d3427307 100644 --- a/backend/pkg/api/data_access/notifications.go +++ b/backend/pkg/api/data_access/notifications.go @@ -65,6 +65,7 @@ func (*DataAccessService) registerNotificationInterfaceTypes() { gob.Register(&n.ValidatorUpcomingProposalNotification{}) gob.Register(&n.ValidatorAttestationNotification{}) gob.Register(&n.ValidatorIsOfflineNotification{}) + gob.Register(&n.ValidatorIsOnlineNotification{}) gob.Register(&n.ValidatorGotSlashedNotification{}) gob.Register(&n.ValidatorWithdrawalNotification{}) gob.Register(&n.NetworkNotification{}) From 340ad12cc8c963fd922a0cae8e0285d3a368acac Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Tue, 22 Oct 2024 09:14:23 +0000 Subject: [PATCH 076/124] fix(notifications): properly handle upcoming proposals in data access --- backend/pkg/api/data_access/notifications.go | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/backend/pkg/api/data_access/notifications.go b/backend/pkg/api/data_access/notifications.go index 1d3427307..01d54b860 100644 --- a/backend/pkg/api/data_access/notifications.go +++ b/backend/pkg/api/data_access/notifications.go @@ -530,6 +530,15 @@ func (d *DataAccessService) GetValidatorDashboardNotificationDetails(ctx context continue } notificationDetails.AttestationMissed = append(notificationDetails.AttestationMissed, t.IndexEpoch{Index: curNotification.ValidatorIndex, Epoch: curNotification.Epoch}) + case types.ValidatorUpcomingProposalEventName: + curNotification, ok := notification.(*n.ValidatorUpcomingProposalNotification) + if !ok { + return nil, fmt.Errorf("failed to cast notification to ValidatorUpcomingProposalNotification") + } + if searchEnabled && !searchIndexSet[curNotification.ValidatorIndex] { + continue + } + notificationDetails.UpcomingProposals = append(notificationDetails.UpcomingProposals, t.IndexSlots{Index: curNotification.ValidatorIndex, Slots: []uint64{curNotification.Slot}}) case types.ValidatorGotSlashedEventName: curNotification, ok := notification.(*n.ValidatorGotSlashedNotification) if !ok { From 355f1fb3aa637751c47e82eb924210fec4849ae9 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Tue, 22 Oct 2024 10:16:31 +0000 Subject: [PATCH 077/124] feat(notifications): implement group efficiency notifications --- backend/pkg/api/data_access/vdb_helpers.go | 24 +--- backend/pkg/api/data_access/vdb_summary.go | 6 +- backend/pkg/commons/utils/efficiency.go | 25 ++++ backend/pkg/notification/collection.go | 160 +++++++++++++++++++++ backend/pkg/notification/types.go | 78 ++++------ 5 files changed, 218 insertions(+), 75 deletions(-) create mode 100644 backend/pkg/commons/utils/efficiency.go diff --git a/backend/pkg/api/data_access/vdb_helpers.go b/backend/pkg/api/data_access/vdb_helpers.go index 12bfe2c46..30fd72f61 100644 --- a/backend/pkg/api/data_access/vdb_helpers.go +++ b/backend/pkg/api/data_access/vdb_helpers.go @@ -41,28 +41,6 @@ func (d DataAccessService) getDashboardValidators(ctx context.Context, dashboard return dashboardId.Validators, nil } -func (d DataAccessService) calculateTotalEfficiency(attestationEff, proposalEff, syncEff sql.NullFloat64) float64 { - efficiency := float64(0) - - if !attestationEff.Valid && !proposalEff.Valid && !syncEff.Valid { - efficiency = 0 - } else if attestationEff.Valid && !proposalEff.Valid && !syncEff.Valid { - efficiency = attestationEff.Float64 * 100.0 - } else if attestationEff.Valid && proposalEff.Valid && !syncEff.Valid { - efficiency = ((56.0 / 64.0 * attestationEff.Float64) + (8.0 / 64.0 * proposalEff.Float64)) * 100.0 - } else if attestationEff.Valid && !proposalEff.Valid && syncEff.Valid { - efficiency = ((62.0 / 64.0 * attestationEff.Float64) + (2.0 / 64.0 * syncEff.Float64)) * 100.0 - } else { - efficiency = (((54.0 / 64.0) * attestationEff.Float64) + ((8.0 / 64.0) * proposalEff.Float64) + ((2.0 / 64.0) * syncEff.Float64)) * 100.0 - } - - if efficiency < 0 { - efficiency = 0 - } - - return efficiency -} - func (d DataAccessService) calculateChartEfficiency(efficiencyType enums.VDBSummaryChartEfficiencyType, row *t.VDBValidatorSummaryChartRow) (float64, error) { efficiency := float64(0) switch efficiencyType { @@ -81,7 +59,7 @@ func (d DataAccessService) calculateChartEfficiency(efficiencyType enums.VDBSumm syncEfficiency.Valid = true } - efficiency = d.calculateTotalEfficiency(attestationEfficiency, proposerEfficiency, syncEfficiency) + efficiency = utils.CalculateTotalEfficiency(attestationEfficiency, proposerEfficiency, syncEfficiency) case enums.VDBSummaryChartAttestation: if row.AttestationIdealReward > 0 { efficiency = (row.AttestationReward / row.AttestationIdealReward) * 100 diff --git a/backend/pkg/api/data_access/vdb_summary.go b/backend/pkg/api/data_access/vdb_summary.go index 7f51b5bed..0c83b0ddc 100644 --- a/backend/pkg/api/data_access/vdb_summary.go +++ b/backend/pkg/api/data_access/vdb_summary.go @@ -90,7 +90,7 @@ func (d *DataAccessService) GetValidatorDashboardSummary(ctx context.Context, da if err != nil { return nil, nil, err } - averageNetworkEfficiency := d.calculateTotalEfficiency( + averageNetworkEfficiency := utils.CalculateTotalEfficiency( efficiency.AttestationEfficiency[period], efficiency.ProposalEfficiency[period], efficiency.SyncEfficiency[period]) // ------------------------------------------------------------------------------------------------------------------ @@ -366,7 +366,7 @@ func (d *DataAccessService) GetValidatorDashboardSummary(ctx context.Context, da syncEfficiency.Float64 = float64(queryEntry.SyncExecuted) / float64(queryEntry.SyncScheduled) syncEfficiency.Valid = true } - resultEntry.Efficiency = d.calculateTotalEfficiency(attestationEfficiency, proposerEfficiency, syncEfficiency) + resultEntry.Efficiency = utils.CalculateTotalEfficiency(attestationEfficiency, proposerEfficiency, syncEfficiency) // Add the duties info to the total total.AttestationReward = total.AttestationReward.Add(queryEntry.AttestationReward) @@ -486,7 +486,7 @@ func (d *DataAccessService) GetValidatorDashboardSummary(ctx context.Context, da totalSyncEfficiency.Float64 = float64(total.SyncExecuted) / float64(total.SyncScheduled) totalSyncEfficiency.Valid = true } - totalEntry.Efficiency = d.calculateTotalEfficiency(totalAttestationEfficiency, totalProposerEfficiency, totalSyncEfficiency) + totalEntry.Efficiency = utils.CalculateTotalEfficiency(totalAttestationEfficiency, totalProposerEfficiency, totalSyncEfficiency) result = append([]t.VDBSummaryTableRow{totalEntry}, result...) } diff --git a/backend/pkg/commons/utils/efficiency.go b/backend/pkg/commons/utils/efficiency.go new file mode 100644 index 000000000..5bb7cd57c --- /dev/null +++ b/backend/pkg/commons/utils/efficiency.go @@ -0,0 +1,25 @@ +package utils + +import "database/sql" + +func CalculateTotalEfficiency(attestationEff, proposalEff, syncEff sql.NullFloat64) float64 { + efficiency := float64(0) + + if !attestationEff.Valid && !proposalEff.Valid && !syncEff.Valid { + efficiency = 0 + } else if attestationEff.Valid && !proposalEff.Valid && !syncEff.Valid { + efficiency = attestationEff.Float64 * 100.0 + } else if attestationEff.Valid && proposalEff.Valid && !syncEff.Valid { + efficiency = ((56.0 / 64.0 * attestationEff.Float64) + (8.0 / 64.0 * proposalEff.Float64)) * 100.0 + } else if attestationEff.Valid && !proposalEff.Valid && syncEff.Valid { + efficiency = ((62.0 / 64.0 * attestationEff.Float64) + (2.0 / 64.0 * syncEff.Float64)) * 100.0 + } else { + efficiency = (((54.0 / 64.0) * attestationEff.Float64) + ((8.0 / 64.0) * proposalEff.Float64) + ((2.0 / 64.0) * syncEff.Float64)) * 100.0 + } + + if efficiency < 0 { + efficiency = 0 + } + + return efficiency +} diff --git a/backend/pkg/notification/collection.go b/backend/pkg/notification/collection.go index 7633e786a..5bbb3322a 100644 --- a/backend/pkg/notification/collection.go +++ b/backend/pkg/notification/collection.go @@ -12,6 +12,7 @@ import ( "time" gcp_bigtable "cloud.google.com/go/bigtable" + "github.com/doug-martin/goqu/v9" "github.com/ethereum/go-ethereum/common" "github.com/gobitfly/beaconchain/pkg/commons/cache" "github.com/gobitfly/beaconchain/pkg/commons/db" @@ -24,6 +25,7 @@ import ( "github.com/gobitfly/beaconchain/pkg/exporter/modules" "github.com/lib/pq" "github.com/rocket-pool/rocketpool-go/utils/eth" + "github.com/shopspring/decimal" ) func InitNotificationCollector(pubkeyCachePath string) { @@ -308,6 +310,13 @@ func collectNotifications(epoch uint64) (types.NotificationsPerUserId, error) { // The following functions will collect the notifications and add them to the // notificationsByUserID map. The notifications will be queued and sent later // by the notification sender process + err = collectGroupEfficiencyNotifications(notificationsByUserID, epoch) + if err != nil { + metrics.Errors.WithLabelValues("notifications_collect_group_efficiency").Inc() + return nil, fmt.Errorf("error collecting validator_group_efficiency notifications: %v", err) + } + log.Infof("collecting attestation & offline notifications took: %v", time.Since(start)) + err = collectAttestationAndOfflineValidatorNotifications(notificationsByUserID, epoch) if err != nil { metrics.Errors.WithLabelValues("notifications_collect_missed_attestation").Inc() @@ -460,6 +469,157 @@ func collectUserDbNotifications(epoch uint64) (types.NotificationsPerUserId, err return notificationsByUserID, nil } +func collectGroupEfficiencyNotifications(notificationsByUserID types.NotificationsPerUserId, epoch uint64) error { + subMap, err := GetSubsForEventFilter("group_efficiency", "", nil, nil) + if err != nil { + return fmt.Errorf("error getting subscriptions for (missed) block proposals %w", err) + } + + // create a lookup map for the dashboard & groups + type groupDetails struct { + Validators []types.ValidatorIndex + Subscription *types.Subscription + } + dashboardMap := make(map[types.UserId]map[types.DashboardId]map[types.DashboardGroupId]*groupDetails) + + for _, subs := range subMap { + for _, sub := range subs { + if sub.DashboardId == nil || sub.DashboardGroupId == nil { + continue + } + userId := *sub.UserID + dashboardId := types.DashboardId(*sub.DashboardId) + groupId := types.DashboardGroupId(*sub.DashboardGroupId) + if _, ok := dashboardMap[userId]; !ok { + dashboardMap[userId] = make(map[types.DashboardId]map[types.DashboardGroupId]*groupDetails) + } + if _, ok := dashboardMap[userId][dashboardId]; !ok { + dashboardMap[userId][dashboardId] = make(map[types.DashboardGroupId]*groupDetails) + } + if _, ok := dashboardMap[userId][dashboardId][groupId]; !ok { + dashboardMap[userId][dashboardId][groupId] = &groupDetails{ + Validators: []types.ValidatorIndex{}, + } + } + if sub.EventFilter != "" { + pubkeyDecoded, err := hex.DecodeString(sub.EventFilter) + if err != nil { + return fmt.Errorf("error decoding pubkey %v: %w", sub.EventFilter, err) + } + validatorIndex, err := GetIndexForPubkey(pubkeyDecoded) + if err != nil { + return fmt.Errorf("error getting validator index for pubkey %v: %w", sub.EventFilter, err) + } + dashboardMap[userId][dashboardId][groupId].Validators = append(dashboardMap[*sub.UserID][dashboardId][groupId].Validators, types.ValidatorIndex(validatorIndex)) + } + dashboardMap[userId][dashboardId][groupId].Subscription = sub + } + } + + type dbResult struct { + ValidatorIndex uint64 `db:"validator_index"` + AttestationReward decimal.Decimal `db:"attestations_reward"` + AttestationIdealReward decimal.Decimal `db:"attestations_ideal_reward"` + BlocksProposed uint64 `db:"blocks_proposed"` + BlocksScheduled uint64 `db:"blocks_scheduled"` + SyncExecuted uint64 `db:"sync_executed"` + SyncScheduled uint64 `db:"sync_scheduled"` + } + + var queryResult []*dbResult + clickhouseTable := "validator_dashboard_data_epoch" + // retrieve efficiency data for the epoch + ds := goqu.Dialect("postgres"). + From(goqu.L(fmt.Sprintf(`%s AS r FINAL`, clickhouseTable))). + Select( + goqu.L("validator_index"), + goqu.L("COALESCE(r.attestations_reward, 0) AS attestations_reward"), + goqu.L("COALESCE(r.attestations_ideal_reward, 0) AS attestations_ideal_reward"), + goqu.L("COALESCE(r.blocks_proposed, 0) AS blocks_proposed"), + goqu.L("COALESCE(r.blocks_scheduled, 0) AS blocks_scheduled"), + goqu.L("COALESCE(r.sync_executed, 0) AS sync_executed"), + goqu.L("COALESCE(r.sync_scheduled, 0) AS sync_scheduled")). + Where(goqu.L("r.epoch = ?", epoch)) + query, args, err := ds.Prepared(true).ToSQL() + if err != nil { + return fmt.Errorf("error preparing query: %v", err) + } + + err = db.ClickHouseReader.Select(&queryResult, query, args...) + if err != nil { + return fmt.Errorf("error retrieving data from table %s: %v", clickhouseTable, err) + } + + efficiencyMap := make(map[types.ValidatorIndex]*dbResult) + for _, row := range queryResult { + efficiencyMap[types.ValidatorIndex(row.ValidatorIndex)] = row + } + + for userId, dashboards := range dashboardMap { + for dashboardId, groups := range dashboards { + for groupId, groupDetails := range groups { + + attestationReward := decimal.Decimal{} + attestationIdealReward := decimal.Decimal{} + blocksProposed := uint64(0) + blocksScheduled := uint64(0) + syncExecuted := uint64(0) + syncScheduled := uint64(0) + + for _, validatorIndex := range groupDetails.Validators { + if row, ok := efficiencyMap[validatorIndex]; ok { + attestationReward = attestationReward.Add(row.AttestationReward) + attestationIdealReward = attestationIdealReward.Add(row.AttestationIdealReward) + blocksProposed += row.BlocksProposed + blocksScheduled += row.BlocksScheduled + syncExecuted += row.SyncExecuted + syncScheduled += row.SyncScheduled + } + } + + var attestationEfficiency, proposerEfficiency, syncEfficiency sql.NullFloat64 + + if !attestationIdealReward.IsZero() { + attestationEfficiency.Float64 = attestationReward.Div(attestationIdealReward).InexactFloat64() + attestationEfficiency.Valid = true + } + if blocksScheduled > 0 { + proposerEfficiency.Float64 = float64(blocksProposed) / float64(blocksScheduled) + proposerEfficiency.Valid = true + } + if syncScheduled > 0 { + syncEfficiency.Float64 = float64(syncExecuted) / float64(syncScheduled) + syncEfficiency.Valid = true + } + + efficiency := utils.CalculateTotalEfficiency(attestationEfficiency, proposerEfficiency, syncEfficiency) + + if efficiency < groupDetails.Subscription.EventThreshold { + log.Infof("creating group efficiency notification for user %v, dashboard %v, group %v in epoch %v", userId, dashboardId, groupId, epoch) + n := &ValidatorGroupEfficiencyNotification{ + NotificationBaseImpl: types.NotificationBaseImpl{ + SubscriptionID: *groupDetails.Subscription.ID, + UserID: *groupDetails.Subscription.UserID, + Epoch: epoch, + EventName: groupDetails.Subscription.EventName, + EventFilter: "-", + DashboardId: groupDetails.Subscription.DashboardId, + DashboardName: groupDetails.Subscription.DashboardName, + DashboardGroupId: groupDetails.Subscription.DashboardGroupId, + DashboardGroupName: groupDetails.Subscription.DashboardGroupName, + }, + Threshold: groupDetails.Subscription.EventThreshold, + Efficiency: efficiency, + } + notificationsByUserID.AddNotification(n) + metrics.NotificationsCollected.WithLabelValues(string(n.GetEventName())).Inc() + } + } + } + } + + return nil +} func collectBlockProposalNotifications(notificationsByUserID types.NotificationsPerUserId, status uint64, eventName types.EventName, epoch uint64) error { type dbResult struct { Proposer uint64 `db:"proposer"` diff --git a/backend/pkg/notification/types.go b/backend/pkg/notification/types.go index b3dcc8377..baed697c2 100644 --- a/backend/pkg/notification/types.go +++ b/backend/pkg/notification/types.go @@ -230,55 +230,35 @@ func (n *ValidatorIsOnlineNotification) GetLegacyTitle() string { return "Validator Back Online" } -// type validatorGroupIsOfflineNotification struct { -// types.NotificationBaseImpl - -// IsOffline bool -// } - -// func (n *validatorGroupIsOfflineNotification) GetEntitiyId() string { -// return fmt.Sprintf("%s - %s", n.GetDashboardName(), n.GetDashboardGroupName()) -// } - -// // Overwrite specific methods -// func (n *validatorGroupIsOfflineNotification) GetInfo(format types.NotificationFormat) string { -// epoch := "" -// if n.IsOffline { -// epoch = formatEpochLink(format, n.LatestState) -// } else { -// epoch = formatEpochLink(format, n.Epoch) -// } - -// if n.IsOffline { -// return fmt.Sprintf(`Group %s is offline since epoch %s.`, n.DashboardGroupName, epoch) -// } else { -// return fmt.Sprintf(`Group %s is back online since epoch %v.`, n.DashboardGroupName, epoch) -// } -// } - -// func (n *validatorGroupIsOfflineNotification) GetTitle() string { -// if n.IsOffline { -// return "Group is offline" -// } else { -// return "Group is back online" -// } -// } - -// func (n *validatorGroupIsOfflineNotification) GetLegacyInfo() string { -// if n.IsOffline { -// return fmt.Sprintf(`Group %s is offline since epoch %s.`, n.DashboardGroupName, n.LatestState) -// } else { -// return fmt.Sprintf(`Group %s is back online since epoch %v.`, n.DashboardGroupName, n.Epoch) -// } -// } - -// func (n *validatorGroupIsOfflineNotification) GetLegacyTitle() string { -// if n.IsOffline { -// return "Group is offline" -// } else { -// return "Group is back online" -// } -// } +type ValidatorGroupEfficiencyNotification struct { + types.NotificationBaseImpl + + Threshold float64 + Efficiency float64 +} + +func (n *ValidatorGroupEfficiencyNotification) GetEntitiyId() string { + return fmt.Sprintf("%s - %s", n.GetDashboardName(), n.GetDashboardGroupName()) +} + +// Overwrite specific methods +func (n *ValidatorGroupEfficiencyNotification) GetInfo(format types.NotificationFormat) string { + dashboardAndGroupInfo := formatDashboardAndGroupLink(format, n) + epoch := formatEpochLink(format, n.Epoch) + return fmt.Sprintf(`%s%s efficiency of %.2f is below the threhold of %.2f in epoch %s.`, dashboardAndGroupInfo, n.Efficiency, n.Threshold epoch) +} + +func (n *ValidatorGroupEfficiencyNotification) GetTitle() string { + return "Low group efficiency" +} + +func (n *ValidatorGroupEfficiencyNotification) GetLegacyInfo() string { + return n.GetInfo(types.NotifciationFormatText) +} + +func (n *ValidatorGroupEfficiencyNotification) GetLegacyTitle() string { + return n.GetTitle() +} type ValidatorAttestationNotification struct { types.NotificationBaseImpl From 691035595647e550ea8b2ca26b9aeb355814bb0e Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Tue, 22 Oct 2024 10:30:22 +0000 Subject: [PATCH 078/124] feat(notifications): add clickhouse db connection --- backend/cmd/notification_collector/main.go | 27 ++++++++ backend/pkg/notification/collection.go | 73 +++++++++++----------- backend/pkg/notification/types.go | 2 +- 3 files changed, 66 insertions(+), 36 deletions(-) diff --git a/backend/cmd/notification_collector/main.go b/backend/cmd/notification_collector/main.go index d6556eaa7..feeecf98b 100644 --- a/backend/cmd/notification_collector/main.go +++ b/backend/cmd/notification_collector/main.go @@ -150,6 +150,31 @@ func Run() { }, "pgx", "postgres") }() + wg.Add(1) + go func() { + defer wg.Done() + // clickhouse + db.ClickHouseWriter, db.ClickHouseReader = db.MustInitDB(&types.DatabaseConfig{ + Username: cfg.ClickHouse.WriterDatabase.Username, + Password: cfg.ClickHouse.WriterDatabase.Password, + Name: cfg.ClickHouse.WriterDatabase.Name, + Host: cfg.ClickHouse.WriterDatabase.Host, + Port: cfg.ClickHouse.WriterDatabase.Port, + MaxOpenConns: cfg.ClickHouse.WriterDatabase.MaxOpenConns, + SSL: true, + MaxIdleConns: cfg.ClickHouse.WriterDatabase.MaxIdleConns, + }, &types.DatabaseConfig{ + Username: cfg.ClickHouse.ReaderDatabase.Username, + Password: cfg.ClickHouse.ReaderDatabase.Password, + Name: cfg.ClickHouse.ReaderDatabase.Name, + Host: cfg.ClickHouse.ReaderDatabase.Host, + Port: cfg.ClickHouse.ReaderDatabase.Port, + MaxOpenConns: cfg.ClickHouse.ReaderDatabase.MaxOpenConns, + SSL: true, + MaxIdleConns: cfg.ClickHouse.ReaderDatabase.MaxIdleConns, + }, "clickhouse", "clickhouse") + }() + wg.Add(1) go func() { defer wg.Done() @@ -184,6 +209,8 @@ func Run() { defer db.FrontendWriterDB.Close() defer db.AlloyReader.Close() defer db.AlloyWriter.Close() + defer db.ClickHouseReader.Close() + defer db.ClickHouseWriter.Close() defer db.BigtableClient.Close() log.Infof("database connection established") diff --git a/backend/pkg/notification/collection.go b/backend/pkg/notification/collection.go index 5bbb3322a..145b45a1b 100644 --- a/backend/pkg/notification/collection.go +++ b/backend/pkg/notification/collection.go @@ -470,6 +470,44 @@ func collectUserDbNotifications(epoch uint64) (types.NotificationsPerUserId, err } func collectGroupEfficiencyNotifications(notificationsByUserID types.NotificationsPerUserId, epoch uint64) error { + type dbResult struct { + ValidatorIndex uint64 `db:"validator_index"` + AttestationReward decimal.Decimal `db:"attestations_reward"` + AttestationIdealReward decimal.Decimal `db:"attestations_ideal_reward"` + BlocksProposed uint64 `db:"blocks_proposed"` + BlocksScheduled uint64 `db:"blocks_scheduled"` + SyncExecuted uint64 `db:"sync_executed"` + SyncScheduled uint64 `db:"sync_scheduled"` + } + + var queryResult []*dbResult + clickhouseTable := "validator_dashboard_data_epoch" + // retrieve efficiency data for the epoch + ds := goqu.Dialect("postgres"). + From(goqu.L(fmt.Sprintf(`%s AS r FINAL`, clickhouseTable))). + Select( + goqu.L("validator_index"), + goqu.L("COALESCE(r.attestations_reward, 0) AS attestations_reward"), + goqu.L("COALESCE(r.attestations_ideal_reward, 0) AS attestations_ideal_reward"), + goqu.L("COALESCE(r.blocks_proposed, 0) AS blocks_proposed"), + goqu.L("COALESCE(r.blocks_scheduled, 0) AS blocks_scheduled"), + goqu.L("COALESCE(r.sync_executed, 0) AS sync_executed"), + goqu.L("COALESCE(r.sync_scheduled, 0) AS sync_scheduled")). + Where(goqu.L("r.epoch = ?", epoch)) + query, args, err := ds.Prepared(true).ToSQL() + if err != nil { + return fmt.Errorf("error preparing query: %v", err) + } + + err = db.ClickHouseReader.Select(&queryResult, query, args...) + if err != nil { + return fmt.Errorf("error retrieving data from table %s: %v", clickhouseTable, err) + } + + if len(queryResult) == 0 { + return fmt.Errorf("no efficiency data found for epoch %v", epoch) + } + subMap, err := GetSubsForEventFilter("group_efficiency", "", nil, nil) if err != nil { return fmt.Errorf("error getting subscriptions for (missed) block proposals %w", err) @@ -516,40 +554,6 @@ func collectGroupEfficiencyNotifications(notificationsByUserID types.Notificatio } } - type dbResult struct { - ValidatorIndex uint64 `db:"validator_index"` - AttestationReward decimal.Decimal `db:"attestations_reward"` - AttestationIdealReward decimal.Decimal `db:"attestations_ideal_reward"` - BlocksProposed uint64 `db:"blocks_proposed"` - BlocksScheduled uint64 `db:"blocks_scheduled"` - SyncExecuted uint64 `db:"sync_executed"` - SyncScheduled uint64 `db:"sync_scheduled"` - } - - var queryResult []*dbResult - clickhouseTable := "validator_dashboard_data_epoch" - // retrieve efficiency data for the epoch - ds := goqu.Dialect("postgres"). - From(goqu.L(fmt.Sprintf(`%s AS r FINAL`, clickhouseTable))). - Select( - goqu.L("validator_index"), - goqu.L("COALESCE(r.attestations_reward, 0) AS attestations_reward"), - goqu.L("COALESCE(r.attestations_ideal_reward, 0) AS attestations_ideal_reward"), - goqu.L("COALESCE(r.blocks_proposed, 0) AS blocks_proposed"), - goqu.L("COALESCE(r.blocks_scheduled, 0) AS blocks_scheduled"), - goqu.L("COALESCE(r.sync_executed, 0) AS sync_executed"), - goqu.L("COALESCE(r.sync_scheduled, 0) AS sync_scheduled")). - Where(goqu.L("r.epoch = ?", epoch)) - query, args, err := ds.Prepared(true).ToSQL() - if err != nil { - return fmt.Errorf("error preparing query: %v", err) - } - - err = db.ClickHouseReader.Select(&queryResult, query, args...) - if err != nil { - return fmt.Errorf("error retrieving data from table %s: %v", clickhouseTable, err) - } - efficiencyMap := make(map[types.ValidatorIndex]*dbResult) for _, row := range queryResult { efficiencyMap[types.ValidatorIndex(row.ValidatorIndex)] = row @@ -558,7 +562,6 @@ func collectGroupEfficiencyNotifications(notificationsByUserID types.Notificatio for userId, dashboards := range dashboardMap { for dashboardId, groups := range dashboards { for groupId, groupDetails := range groups { - attestationReward := decimal.Decimal{} attestationIdealReward := decimal.Decimal{} blocksProposed := uint64(0) diff --git a/backend/pkg/notification/types.go b/backend/pkg/notification/types.go index baed697c2..d7b146649 100644 --- a/backend/pkg/notification/types.go +++ b/backend/pkg/notification/types.go @@ -245,7 +245,7 @@ func (n *ValidatorGroupEfficiencyNotification) GetEntitiyId() string { func (n *ValidatorGroupEfficiencyNotification) GetInfo(format types.NotificationFormat) string { dashboardAndGroupInfo := formatDashboardAndGroupLink(format, n) epoch := formatEpochLink(format, n.Epoch) - return fmt.Sprintf(`%s%s efficiency of %.2f is below the threhold of %.2f in epoch %s.`, dashboardAndGroupInfo, n.Efficiency, n.Threshold epoch) + return fmt.Sprintf(`%s%s efficiency of %.2f is below the threhold of %.2f in epoch %s.`, dashboardAndGroupInfo, n.Efficiency, n.Threshold, epoch) } func (n *ValidatorGroupEfficiencyNotification) GetTitle() string { From b2a1c718f1eccdad8aad46ef4e9a3ba40ca91659 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Tue, 22 Oct 2024 10:34:40 +0000 Subject: [PATCH 079/124] feat(notifications): add efficiency threshold types --- backend/pkg/api/data_access/notifications.go | 30 +++++++++----------- backend/pkg/api/data_access/vdb_summary.go | 2 +- backend/pkg/commons/types/frontend.go | 9 ++++-- backend/pkg/notification/collection.go | 2 +- 4 files changed, 21 insertions(+), 22 deletions(-) diff --git a/backend/pkg/api/data_access/notifications.go b/backend/pkg/api/data_access/notifications.go index 01d54b860..f37948ac0 100644 --- a/backend/pkg/api/data_access/notifications.go +++ b/backend/pkg/api/data_access/notifications.go @@ -568,19 +568,19 @@ func (d *DataAccessService) GetValidatorDashboardNotificationDetails(ctx context continue } notificationDetails.ValidatorBackOnline = append(notificationDetails.ValidatorBackOnline, t.NotificationEventValidatorBackOnline{Index: curNotification.ValidatorIndex, EpochCount: curNotification.Epoch}) - case types.ValidatorGroupIsOfflineEventName: - // TODO type / collection not present yet, skipping - /*curNotification, ok := not.(*notification.validatorGroupIsOfflineNotification) - if !ok { - return nil, fmt.Errorf("failed to cast notification to validatorGroupIsOfflineNotification") - } - if curNotification.Status == 0 { - notificationDetails.GroupOffline = ... - notificationDetails.GroupOfflineReminder = ... - } else { - notificationDetails.GroupBackOnline = ... - } - */ + // case types.ValidatorGroupIsOfflineEventName: + // TODO type / collection not present yet, skipping + /*curNotification, ok := not.(*notification.validatorGroupIsOfflineNotification) + if !ok { + return nil, fmt.Errorf("failed to cast notification to validatorGroupIsOfflineNotification") + } + if curNotification.Status == 0 { + notificationDetails.GroupOffline = ... + notificationDetails.GroupOfflineReminder = ... + } else { + notificationDetails.GroupBackOnline = ... + } + */ case types.ValidatorReceivedWithdrawalEventName: curNotification, ok := notification.(*n.ValidatorWithdrawalNotification) if !ok { @@ -1878,9 +1878,6 @@ func (d *DataAccessService) GetNotificationSettingsDashboards(ctx context.Contex switch eventName { case types.ValidatorIsOfflineEventName: settings.IsValidatorOfflineSubscribed = true - case types.GroupIsOfflineEventName: - settings.IsGroupOfflineSubscribed = true - settings.GroupOfflineThreshold = event.Threshold case types.ValidatorMissedAttestationEventName: settings.IsAttestationsMissedSubscribed = true case types.ValidatorMissedProposalEventName, types.ValidatorExecutedProposalEventName: @@ -2110,7 +2107,6 @@ func (d *DataAccessService) UpdateNotificationSettingsValidatorDashboard(ctx con eventFilter := fmt.Sprintf("%s:%d:%d", ValidatorDashboardEventPrefix, dashboardId, groupId) d.AddOrRemoveEvent(&eventsToInsert, &eventsToDelete, settings.IsValidatorOfflineSubscribed, userId, types.ValidatorIsOfflineEventName, networkName, eventFilter, epoch, 0) - d.AddOrRemoveEvent(&eventsToInsert, &eventsToDelete, settings.IsGroupOfflineSubscribed, userId, types.GroupIsOfflineEventName, networkName, eventFilter, epoch, settings.GroupOfflineThreshold) d.AddOrRemoveEvent(&eventsToInsert, &eventsToDelete, settings.IsAttestationsMissedSubscribed, userId, types.ValidatorMissedAttestationEventName, networkName, eventFilter, epoch, 0) d.AddOrRemoveEvent(&eventsToInsert, &eventsToDelete, settings.IsUpcomingBlockProposalSubscribed, userId, types.ValidatorUpcomingProposalEventName, networkName, eventFilter, epoch, 0) d.AddOrRemoveEvent(&eventsToInsert, &eventsToDelete, settings.IsSyncSubscribed, userId, types.SyncCommitteeSoon, networkName, eventFilter, epoch, 0) diff --git a/backend/pkg/api/data_access/vdb_summary.go b/backend/pkg/api/data_access/vdb_summary.go index 0c83b0ddc..f4216ec7b 100644 --- a/backend/pkg/api/data_access/vdb_summary.go +++ b/backend/pkg/api/data_access/vdb_summary.go @@ -1021,7 +1021,7 @@ func (d *DataAccessService) GetValidatorDashboardSummaryChart(ctx context.Contex if err != nil { return nil, err } - averageNetworkEfficiency := d.calculateTotalEfficiency( + averageNetworkEfficiency := utils.CalculateTotalEfficiency( efficiency.AttestationEfficiency[enums.Last24h], efficiency.ProposalEfficiency[enums.Last24h], efficiency.SyncEfficiency[enums.Last24h]) for ts := range tsMap { diff --git a/backend/pkg/commons/types/frontend.go b/backend/pkg/commons/types/frontend.go index b45f1b06d..9bb2b4e8a 100644 --- a/backend/pkg/commons/types/frontend.go +++ b/backend/pkg/commons/types/frontend.go @@ -68,8 +68,8 @@ const ( ValidatorMissedProposalEventName EventName = "validator_proposal_missed" ValidatorExecutedProposalEventName EventName = "validator_proposal_submitted" - ValidatorDidSlashEventName EventName = "validator_did_slash" - ValidatorGroupIsOfflineEventName EventName = "validator_group_is_offline" + ValidatorDidSlashEventName EventName = "validator_did_slash" + ValidatorGroupEfficiencyEventName EventName = "validator_group_efficiency" ValidatorReceivedDepositEventName EventName = "validator_received_deposit" NetworkSlashingEventName EventName = "network_slashing" @@ -86,7 +86,6 @@ const ( // Validator dashboard events ValidatorIsOfflineEventName EventName = "validator_is_offline" ValidatorIsOnlineEventName EventName = "validator_is_online" - GroupIsOfflineEventName EventName = "group_is_offline" ValidatorMissedAttestationEventName EventName = "validator_attestation_missed" ValidatorProposalEventName EventName = "validator_proposal" ValidatorUpcomingProposalEventName EventName = "validator_proposal_upcoming" @@ -132,6 +131,7 @@ var EventSortOrder = []EventName{ SyncCommitteeSoonEventName, ValidatorIsOfflineEventName, ValidatorIsOnlineEventName, + ValidatorGroupEfficiencyEventName, ValidatorReceivedWithdrawalEventName, NetworkLivenessIncreasedEventName, EthClientUpdateEventName, @@ -175,6 +175,7 @@ var MachineEventsMap = map[EventName]struct{}{ var LegacyEventLabel map[EventName]string = map[EventName]string{ ValidatorUpcomingProposalEventName: "Your validator(s) will soon propose a block", + ValidatorGroupEfficiencyEventName: "Your validator group efficiency is low", ValidatorMissedProposalEventName: "Your validator(s) missed a proposal", ValidatorExecutedProposalEventName: "Your validator(s) submitted a proposal", ValidatorMissedAttestationEventName: "Your validator(s) missed an attestation", @@ -199,6 +200,7 @@ var LegacyEventLabel map[EventName]string = map[EventName]string{ var EventLabel map[EventName]string = map[EventName]string{ ValidatorUpcomingProposalEventName: "Upcoming block proposal", + ValidatorGroupEfficiencyEventName: "Low validator group efficiency", ValidatorMissedProposalEventName: "Block proposal missed", ValidatorExecutedProposalEventName: "Block proposal submitted", ValidatorMissedAttestationEventName: "Attestation missed", @@ -233,6 +235,7 @@ func IsMachineNotification(event EventName) bool { var EventNames = []EventName{ ValidatorExecutedProposalEventName, + ValidatorGroupEfficiencyEventName, ValidatorMissedProposalEventName, ValidatorMissedAttestationEventName, ValidatorGotSlashedEventName, diff --git a/backend/pkg/notification/collection.go b/backend/pkg/notification/collection.go index 145b45a1b..d5f4b3b8a 100644 --- a/backend/pkg/notification/collection.go +++ b/backend/pkg/notification/collection.go @@ -508,7 +508,7 @@ func collectGroupEfficiencyNotifications(notificationsByUserID types.Notificatio return fmt.Errorf("no efficiency data found for epoch %v", epoch) } - subMap, err := GetSubsForEventFilter("group_efficiency", "", nil, nil) + subMap, err := GetSubsForEventFilter(types.ValidatorGroupEfficiencyEventName, "", nil, nil) if err != nil { return fmt.Errorf("error getting subscriptions for (missed) block proposals %w", err) } From bb1ba290ae7551fd98b78943c02d3e59654fa035 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Tue, 22 Oct 2024 10:35:33 +0000 Subject: [PATCH 080/124] fix(notifications): resolve compiler errors --- backend/pkg/api/data_access/mobile.go | 4 ++-- backend/pkg/api/data_access/vdb_management.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/backend/pkg/api/data_access/mobile.go b/backend/pkg/api/data_access/mobile.go index 4d6165b20..dc526746b 100644 --- a/backend/pkg/api/data_access/mobile.go +++ b/backend/pkg/api/data_access/mobile.go @@ -171,7 +171,7 @@ func (d *DataAccessService) GetValidatorDashboardMobileWidget(ctx context.Contex if err != nil { return nil, fmt.Errorf("error retrieving validator dashboard overview data: %w", err) } - data.NetworkEfficiency = d.calculateTotalEfficiency( + data.NetworkEfficiency = utils.CalculateTotalEfficiency( efficiency.AttestationEfficiency[enums.AllTime], efficiency.ProposalEfficiency[enums.AllTime], efficiency.SyncEfficiency[enums.AllTime]) // Validator status @@ -327,7 +327,7 @@ func (d *DataAccessService) GetValidatorDashboardMobileWidget(ctx context.Contex syncEfficiency.Float64 = float64(queryResult.SyncExecuted) / float64(queryResult.SyncScheduled) syncEfficiency.Valid = true } - *efficiency = d.calculateTotalEfficiency(attestationEfficiency, proposerEfficiency, syncEfficiency) + *efficiency = utils.CalculateTotalEfficiency(attestationEfficiency, proposerEfficiency, syncEfficiency) return nil }) diff --git a/backend/pkg/api/data_access/vdb_management.go b/backend/pkg/api/data_access/vdb_management.go index cbf0a4b64..d152d6cac 100644 --- a/backend/pkg/api/data_access/vdb_management.go +++ b/backend/pkg/api/data_access/vdb_management.go @@ -520,7 +520,7 @@ func (d *DataAccessService) GetValidatorDashboardOverview(ctx context.Context, d syncEfficiency.Float64 = float64(queryResult.SyncExecuted) / float64(queryResult.SyncScheduled) syncEfficiency.Valid = true } - *efficiency = d.calculateTotalEfficiency(attestationEfficiency, proposerEfficiency, syncEfficiency) + *efficiency = utils.CalculateTotalEfficiency(attestationEfficiency, proposerEfficiency, syncEfficiency) return nil }) From 85d7259da5ffab4cb9799e78389a41d167aa60fd Mon Sep 17 00:00:00 2001 From: Stefan Pletka <124689083+Eisei24@users.noreply.github.com> Date: Tue, 22 Oct 2024 12:40:12 +0200 Subject: [PATCH 081/124] Adjusted discord webhook handling --- backend/pkg/api/data_access/notifications.go | 66 +++++++++++++------- 1 file changed, 42 insertions(+), 24 deletions(-) diff --git a/backend/pkg/api/data_access/notifications.go b/backend/pkg/api/data_access/notifications.go index 01d54b860..b84296825 100644 --- a/backend/pkg/api/data_access/notifications.go +++ b/backend/pkg/api/data_access/notifications.go @@ -81,8 +81,6 @@ const ( ValidatorDashboardEventPrefix string = "vdb" AccountDashboardEventPrefix string = "adb" - DiscordWebhookFormat string = "discord" - GroupOfflineThresholdDefault float64 = 0.1 MaxCollateralThresholdDefault float64 = 1.0 MinCollateralThresholdDefault float64 = 0.2 @@ -1765,14 +1763,14 @@ func (d *DataAccessService) GetNotificationSettingsDashboards(ctx context.Contex // ------------------------------------- // Get the validator dashboards valDashboards := []struct { - DashboardId uint64 `db:"dashboard_id"` - DashboardName string `db:"dashboard_name"` - GroupId uint64 `db:"group_id"` - GroupName string `db:"group_name"` - Network uint64 `db:"network"` - WebhookUrl sql.NullString `db:"webhook_target"` - IsWebhookDiscordEnabled sql.NullBool `db:"discord_webhook"` - IsRealTimeModeEnabled sql.NullBool `db:"realtime_notifications"` + DashboardId uint64 `db:"dashboard_id"` + DashboardName string `db:"dashboard_name"` + GroupId uint64 `db:"group_id"` + GroupName string `db:"group_name"` + Network uint64 `db:"network"` + WebhookUrl sql.NullString `db:"webhook_target"` + WebhookFormat sql.NullString `db:"webhook_format"` + IsRealTimeModeEnabled sql.NullBool `db:"realtime_notifications"` }{} wg.Go(func() error { err := d.alloyReader.SelectContext(ctx, &valDashboards, ` @@ -1783,11 +1781,11 @@ func (d *DataAccessService) GetNotificationSettingsDashboards(ctx context.Contex g.name AS group_name, d.network, g.webhook_target, - (g.webhook_format = $1) AS discord_webhook, + g.webhook_format, g.realtime_notifications FROM users_val_dashboards d INNER JOIN users_val_dashboards_groups g ON d.id = g.dashboard_id - WHERE d.user_id = $2`, DiscordWebhookFormat, userId) + WHERE d.user_id = $1`, userId) if err != nil { return fmt.Errorf(`error retrieving data for validator dashboard notifications: %w`, err) } @@ -1803,7 +1801,7 @@ func (d *DataAccessService) GetNotificationSettingsDashboards(ctx context.Contex GroupId uint64 `db:"group_id"` GroupName string `db:"group_name"` WebhookUrl sql.NullString `db:"webhook_target"` - IsWebhookDiscordEnabled sql.NullBool `db:"discord_webhook"` + WebhookFormat sql.NullString `db:"webhook_format"` IsIgnoreSpamTransactionsEnabled bool `db:"ignore_spam_transactions"` SubscribedChainIds []uint64 `db:"subscribed_chain_ids"` }{} @@ -1816,12 +1814,12 @@ func (d *DataAccessService) GetNotificationSettingsDashboards(ctx context.Contex // g.id AS group_id, // g.name AS group_name, // g.webhook_target, - // (g.webhook_format = $1) AS discord_webhook, + // g.webhook_format, // g.ignore_spam_transactions, // g.subscribed_chain_ids // FROM users_acc_dashboards d // INNER JOIN users_acc_dashboards_groups g ON d.id = g.dashboard_id - // WHERE d.user_id = $2`, DiscordWebhookFormat, userId) + // WHERE d.user_id = $1`, userId) // if err != nil { // return fmt.Errorf(`error retrieving data for validator dashboard notifications: %w`, err) // } @@ -1944,7 +1942,8 @@ func (d *DataAccessService) GetNotificationSettingsDashboards(ctx context.Contex // Set the settings if valSettings, ok := resultMap[key].Settings.(*t.NotificationSettingsValidatorDashboard); ok { valSettings.WebhookUrl = valDashboard.WebhookUrl.String - valSettings.IsWebhookDiscordEnabled = valDashboard.IsWebhookDiscordEnabled.Bool + valSettings.IsWebhookDiscordEnabled = valDashboard.WebhookFormat.Valid && + types.NotificationChannel(valDashboard.WebhookFormat.String) == types.WebhookDiscordNotificationChannel valSettings.IsRealTimeModeEnabled = valDashboard.IsRealTimeModeEnabled.Bool } } @@ -1972,7 +1971,8 @@ func (d *DataAccessService) GetNotificationSettingsDashboards(ctx context.Contex // Set the settings if accSettings, ok := resultMap[key].Settings.(*t.NotificationSettingsAccountDashboard); ok { accSettings.WebhookUrl = accDashboard.WebhookUrl.String - accSettings.IsWebhookDiscordEnabled = accDashboard.IsWebhookDiscordEnabled.Bool + accSettings.IsWebhookDiscordEnabled = accDashboard.WebhookFormat.Valid && + types.NotificationChannel(accDashboard.WebhookFormat.String) == types.WebhookDiscordNotificationChannel accSettings.IsIgnoreSpamTransactionsEnabled = accDashboard.IsIgnoreSpamTransactionsEnabled accSettings.SubscribedChainIds = accDashboard.SubscribedChainIds } @@ -2167,13 +2167,22 @@ func (d *DataAccessService) UpdateNotificationSettingsValidatorDashboard(ctx con } // Set non-event settings + var webhookFormat sql.NullString + if settings.WebhookUrl != "" { + webhookFormat.String = string(types.WebhookNotificationChannel) + webhookFormat.Valid = true + if settings.IsWebhookDiscordEnabled { + webhookFormat.String = string(types.WebhookDiscordNotificationChannel) + } + } + _, err = d.alloyWriter.ExecContext(ctx, ` UPDATE users_val_dashboards_groups SET webhook_target = NULLIF($1, ''), - webhook_format = CASE WHEN $2 THEN $3 ELSE NULL END, - realtime_notifications = CASE WHEN $4 THEN TRUE ELSE NULL END - WHERE dashboard_id = $5 AND id = $6`, settings.WebhookUrl, settings.IsWebhookDiscordEnabled, DiscordWebhookFormat, settings.IsRealTimeModeEnabled, dashboardId, groupId) + webhook_format = $2, + realtime_notifications = CASE WHEN $3 THEN TRUE ELSE NULL END + WHERE dashboard_id = $4 AND id = $5`, settings.WebhookUrl, webhookFormat, settings.IsRealTimeModeEnabled, dashboardId, groupId) if err != nil { return err } @@ -2247,14 +2256,23 @@ func (d *DataAccessService) UpdateNotificationSettingsAccountDashboard(ctx conte // } // // Set non-event settings + // var webhookFormat sql.NullString + // if settings.WebhookUrl != "" { + // webhookFormat.String = string(types.WebhookNotificationChannel) + // webhookFormat.Valid = true + // if settings.IsWebhookDiscordEnabled { + // webhookFormat.String = string(types.WebhookDiscordNotificationChannel) + // } + // } + // _, err = d.alloyWriter.ExecContext(ctx, ` // UPDATE users_acc_dashboards_groups // SET // webhook_target = NULLIF($1, ''), - // webhook_format = CASE WHEN $2 THEN $3 ELSE NULL END, - // ignore_spam_transactions = $4, - // subscribed_chain_ids = $5 - // WHERE dashboard_id = $6 AND id = $7`, settings.WebhookUrl, settings.IsWebhookDiscordEnabled, DiscordWebhookFormat, settings.IsIgnoreSpamTransactionsEnabled, settings.SubscribedChainIds, dashboardId, groupId) + // webhook_format = $2, + // ignore_spam_transactions = $3, + // subscribed_chain_ids = $4 + // WHERE dashboard_id = $5 AND id = $6`, settings.WebhookUrl, webhookFormat, settings.IsIgnoreSpamTransactionsEnabled, settings.SubscribedChainIds, dashboardId, groupId) // if err != nil { // return err // } From cfcc538c68d04733b7ea78acb3205c56e2856486 Mon Sep 17 00:00:00 2001 From: Stefan Pletka <124689083+Eisei24@users.noreply.github.com> Date: Tue, 22 Oct 2024 12:59:43 +0200 Subject: [PATCH 082/124] Fixed pointer type check --- backend/pkg/api/data_access/notifications.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/backend/pkg/api/data_access/notifications.go b/backend/pkg/api/data_access/notifications.go index b84296825..a2dd67fb0 100644 --- a/backend/pkg/api/data_access/notifications.go +++ b/backend/pkg/api/data_access/notifications.go @@ -1940,11 +1940,13 @@ func (d *DataAccessService) GetNotificationSettingsDashboards(ctx context.Contex resultMap[key].ChainIds = []uint64{valDashboard.Network} // Set the settings - if valSettings, ok := resultMap[key].Settings.(*t.NotificationSettingsValidatorDashboard); ok { + if valSettings, ok := resultMap[key].Settings.(t.NotificationSettingsValidatorDashboard); ok { valSettings.WebhookUrl = valDashboard.WebhookUrl.String valSettings.IsWebhookDiscordEnabled = valDashboard.WebhookFormat.Valid && types.NotificationChannel(valDashboard.WebhookFormat.String) == types.WebhookDiscordNotificationChannel valSettings.IsRealTimeModeEnabled = valDashboard.IsRealTimeModeEnabled.Bool + + resultMap[key].Settings = valSettings } } @@ -1969,12 +1971,14 @@ func (d *DataAccessService) GetNotificationSettingsDashboards(ctx context.Contex resultMap[key].ChainIds = accDashboard.SubscribedChainIds // Set the settings - if accSettings, ok := resultMap[key].Settings.(*t.NotificationSettingsAccountDashboard); ok { + if accSettings, ok := resultMap[key].Settings.(t.NotificationSettingsAccountDashboard); ok { accSettings.WebhookUrl = accDashboard.WebhookUrl.String accSettings.IsWebhookDiscordEnabled = accDashboard.WebhookFormat.Valid && types.NotificationChannel(accDashboard.WebhookFormat.String) == types.WebhookDiscordNotificationChannel accSettings.IsIgnoreSpamTransactionsEnabled = accDashboard.IsIgnoreSpamTransactionsEnabled accSettings.SubscribedChainIds = accDashboard.SubscribedChainIds + + resultMap[key].Settings = accSettings } } From 737991827897c88e2d6d0dbd3ffeb399059569cb Mon Sep 17 00:00:00 2001 From: Stefan Pletka <124689083+Eisei24@users.noreply.github.com> Date: Tue, 22 Oct 2024 13:44:29 +0200 Subject: [PATCH 083/124] Changed device identifier to id --- backend/pkg/api/data_access/dummy.go | 4 +-- backend/pkg/api/data_access/notifications.go | 30 ++++++++++---------- backend/pkg/api/handlers/public.go | 4 +-- backend/pkg/api/types/notifications.go | 2 +- frontend/types/api/notifications.ts | 2 +- 5 files changed, 21 insertions(+), 21 deletions(-) diff --git a/backend/pkg/api/data_access/dummy.go b/backend/pkg/api/data_access/dummy.go index 24472343a..cf123aec6 100644 --- a/backend/pkg/api/data_access/dummy.go +++ b/backend/pkg/api/data_access/dummy.go @@ -576,10 +576,10 @@ func (d *DummyService) UpdateNotificationSettingsGeneral(ctx context.Context, us func (d *DummyService) UpdateNotificationSettingsNetworks(ctx context.Context, userId uint64, chainId uint64, settings t.NotificationSettingsNetwork) error { return nil } -func (d *DummyService) UpdateNotificationSettingsPairedDevice(ctx context.Context, userId uint64, pairedDeviceId string, name string, IsNotificationsEnabled bool) error { +func (d *DummyService) UpdateNotificationSettingsPairedDevice(ctx context.Context, userId uint64, pairedDeviceId uint64, name string, IsNotificationsEnabled bool) error { return nil } -func (d *DummyService) DeleteNotificationSettingsPairedDevice(ctx context.Context, userId uint64, pairedDeviceId string) error { +func (d *DummyService) DeleteNotificationSettingsPairedDevice(ctx context.Context, userId uint64, pairedDeviceId uint64) error { return nil } diff --git a/backend/pkg/api/data_access/notifications.go b/backend/pkg/api/data_access/notifications.go index 01d54b860..106b4b61f 100644 --- a/backend/pkg/api/data_access/notifications.go +++ b/backend/pkg/api/data_access/notifications.go @@ -50,8 +50,8 @@ type NotificationsRepository interface { GetNotificationSettingsDefaultValues(ctx context.Context) (*t.NotificationSettingsDefaultValues, error) UpdateNotificationSettingsGeneral(ctx context.Context, userId uint64, settings t.NotificationSettingsGeneral) error UpdateNotificationSettingsNetworks(ctx context.Context, userId uint64, chainId uint64, settings t.NotificationSettingsNetwork) error - UpdateNotificationSettingsPairedDevice(ctx context.Context, userId uint64, pairedDeviceId string, name string, IsNotificationsEnabled bool) error - DeleteNotificationSettingsPairedDevice(ctx context.Context, userId uint64, pairedDeviceId string) error + UpdateNotificationSettingsPairedDevice(ctx context.Context, userId uint64, pairedDeviceId uint64, name string, IsNotificationsEnabled bool) error + DeleteNotificationSettingsPairedDevice(ctx context.Context, userId uint64, pairedDeviceId uint64) error UpdateNotificationSettingsClients(ctx context.Context, userId uint64, clientId uint64, IsSubscribed bool) (*t.NotificationSettingsClient, error) GetNotificationSettingsDashboards(ctx context.Context, userId uint64, cursor string, colSort t.Sort[enums.NotificationSettingsDashboardColumn], search string, limit uint64) ([]t.NotificationSettingsDashboardsTableRow, *t.Paging, error) UpdateNotificationSettingsValidatorDashboard(ctx context.Context, userId uint64, dashboardId t.VDBIdPrimary, groupId uint64, settings t.NotificationSettingsValidatorDashboard) error @@ -1325,20 +1325,20 @@ func (d *DataAccessService) GetNotificationSettings(ctx context.Context, userId // ------------------------------------- // Get the paired devices pairedDevices := []struct { - DeviceIdentifier sql.NullString `db:"device_identifier"` - CreatedTs time.Time `db:"created_ts"` - DeviceName string `db:"device_name"` - NotifyEnabled bool `db:"notify_enabled"` + DeviceId uint64 `db:"id"` + CreatedTs time.Time `db:"created_ts"` + DeviceName string `db:"device_name"` + NotifyEnabled bool `db:"notify_enabled"` }{} wg.Go(func() error { err := d.userReader.SelectContext(ctx, &pairedDevices, ` SELECT - device_identifier, + id, created_ts, device_name, COALESCE(notify_enabled, false) AS notify_enabled FROM users_devices - WHERE user_id = $1 AND device_identifier IS NOT NULL`, userId) + WHERE user_id = $1`, userId) if err != nil { return fmt.Errorf(`error retrieving data for notifications paired devices: %w`, err) } @@ -1429,7 +1429,7 @@ func (d *DataAccessService) GetNotificationSettings(ctx context.Context, userId for _, device := range pairedDevices { result.PairedDevices = append(result.PairedDevices, t.NotificationPairedDevice{ - Id: device.DeviceIdentifier.String, + Id: device.DeviceId, PairedTimestamp: device.CreatedTs.Unix(), Name: device.DeviceName, IsNotificationsEnabled: device.NotifyEnabled, @@ -1642,13 +1642,13 @@ func (d *DataAccessService) UpdateNotificationSettingsNetworks(ctx context.Conte } return nil } -func (d *DataAccessService) UpdateNotificationSettingsPairedDevice(ctx context.Context, userId uint64, pairedDeviceId string, name string, IsNotificationsEnabled bool) error { +func (d *DataAccessService) UpdateNotificationSettingsPairedDevice(ctx context.Context, userId uint64, pairedDeviceId uint64, name string, IsNotificationsEnabled bool) error { result, err := d.userWriter.ExecContext(ctx, ` UPDATE users_devices SET device_name = $1, notify_enabled = $2 - WHERE user_id = $3 AND device_identifier = $4`, + WHERE user_id = $3 AND id = $4`, name, IsNotificationsEnabled, userId, pairedDeviceId) if err != nil { return err @@ -1660,14 +1660,14 @@ func (d *DataAccessService) UpdateNotificationSettingsPairedDevice(ctx context.C return err } if rowsAffected == 0 { - return fmt.Errorf("device with id %s to update notification settings not found", pairedDeviceId) + return fmt.Errorf("device with id %v to update notification settings not found", pairedDeviceId) } return nil } -func (d *DataAccessService) DeleteNotificationSettingsPairedDevice(ctx context.Context, userId uint64, pairedDeviceId string) error { +func (d *DataAccessService) DeleteNotificationSettingsPairedDevice(ctx context.Context, userId uint64, pairedDeviceId uint64) error { result, err := d.userWriter.ExecContext(ctx, ` DELETE FROM users_devices - WHERE user_id = $1 AND device_identifier = $2`, + WHERE user_id = $1 AND id = $2`, userId, pairedDeviceId) if err != nil { return err @@ -1679,7 +1679,7 @@ func (d *DataAccessService) DeleteNotificationSettingsPairedDevice(ctx context.C return err } if rowsAffected == 0 { - return fmt.Errorf("device with id %s to delete not found", pairedDeviceId) + return fmt.Errorf("device with id %v to delete not found", pairedDeviceId) } return nil } diff --git a/backend/pkg/api/handlers/public.go b/backend/pkg/api/handlers/public.go index 98d1b8b55..f3792d7e5 100644 --- a/backend/pkg/api/handlers/public.go +++ b/backend/pkg/api/handlers/public.go @@ -2345,7 +2345,7 @@ func (h *HandlerService) PublicPutUserNotificationSettingsPairedDevices(w http.R return } // TODO use a better way to validate the paired device id - pairedDeviceId := v.checkRegex(reNonEmpty, mux.Vars(r)["paired_device_id"], "paired_device_id") + pairedDeviceId := v.checkUint(mux.Vars(r)["paired_device_id"], "paired_device_id") name := v.checkNameNotEmpty(req.Name) if v.hasErrors() { handleErr(w, r, v) @@ -2386,7 +2386,7 @@ func (h *HandlerService) PublicDeleteUserNotificationSettingsPairedDevices(w htt return } // TODO use a better way to validate the paired device id - pairedDeviceId := v.checkRegex(reNonEmpty, mux.Vars(r)["paired_device_id"], "paired_device_id") + pairedDeviceId := v.checkUint(mux.Vars(r)["paired_device_id"], "paired_device_id") if v.hasErrors() { handleErr(w, r, v) return diff --git a/backend/pkg/api/types/notifications.go b/backend/pkg/api/types/notifications.go index 36ca1b1b6..e2baeb32c 100644 --- a/backend/pkg/api/types/notifications.go +++ b/backend/pkg/api/types/notifications.go @@ -160,7 +160,7 @@ type NotificationNetwork struct { type InternalPutUserNotificationSettingsNetworksResponse ApiDataResponse[NotificationNetwork] type NotificationPairedDevice struct { - Id string `json:"id"` + Id uint64 `json:"id"` PairedTimestamp int64 `json:"paired_timestamp"` Name string `json:"name,omitempty"` IsNotificationsEnabled bool `json:"is_notifications_enabled"` diff --git a/frontend/types/api/notifications.ts b/frontend/types/api/notifications.ts index 2dc1c96ee..7abe5c7f9 100644 --- a/frontend/types/api/notifications.ts +++ b/frontend/types/api/notifications.ts @@ -152,7 +152,7 @@ export interface NotificationNetwork { } export type InternalPutUserNotificationSettingsNetworksResponse = ApiDataResponse; export interface NotificationPairedDevice { - id: string; + id: number /* uint64 */; paired_timestamp: number /* int64 */; name?: string; is_notifications_enabled: boolean; From 707cfcba6d1f49a7aca1717e719ae7d243038fec Mon Sep 17 00:00:00 2001 From: Stefan Pletka <124689083+Eisei24@users.noreply.github.com> Date: Tue, 22 Oct 2024 14:13:11 +0200 Subject: [PATCH 084/124] Fix linter issue --- backend/pkg/api/handlers/input_validation.go | 1 - 1 file changed, 1 deletion(-) diff --git a/backend/pkg/api/handlers/input_validation.go b/backend/pkg/api/handlers/input_validation.go index 16478449a..eca791aa8 100644 --- a/backend/pkg/api/handlers/input_validation.go +++ b/backend/pkg/api/handlers/input_validation.go @@ -33,7 +33,6 @@ var ( reEthereumAddress = regexp.MustCompile(`^(0x)?[0-9a-fA-F]{40}$`) reWithdrawalCredential = regexp.MustCompile(`^(0x0[01])?[0-9a-fA-F]{62}$`) reEnsName = regexp.MustCompile(`^.+\.eth$`) - reNonEmpty = regexp.MustCompile(`^\s*\S.*$`) reGraffiti = regexp.MustCompile(`^.{2,}$`) // at least 2 characters, so that queries won't time out reCursor = regexp.MustCompile(`^[A-Za-z0-9-_]+$`) // has to be base64 reEmail = regexp.MustCompile("^[a-zA-Z0-9.!#$%&'*+/=?^_`{|}~-]+@[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?(?:\\.[a-zA-Z0-9](?:[a-zA-Z0-9-]{0,61}[a-zA-Z0-9])?)*$") From 18aecaa98e7e595e04e4ce9eee10e69cfd00bc10 Mon Sep 17 00:00:00 2001 From: Stefan Pletka <124689083+Eisei24@users.noreply.github.com> Date: Tue, 22 Oct 2024 14:42:13 +0200 Subject: [PATCH 085/124] refactor(notifications): Use number id instead of string device_identifier as unique device identifier --- .../NotificationsManagementPairedDeviceModalContent.vue | 4 ++-- .../management/NotificationsManagementPairedDevicesModal.vue | 2 +- .../stores/notifications/useNotificationsManagementStore.ts | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/frontend/components/notifications/management/NotificationsManagementPairedDeviceModalContent.vue b/frontend/components/notifications/management/NotificationsManagementPairedDeviceModalContent.vue index 4aa91fc2d..3118dc856 100644 --- a/frontend/components/notifications/management/NotificationsManagementPairedDeviceModalContent.vue +++ b/frontend/components/notifications/management/NotificationsManagementPairedDeviceModalContent.vue @@ -10,12 +10,12 @@ const props = defineProps<{ }>() const emit = defineEmits<{ - (e: 'remove-device', id: string): void, + (e: 'remove-device', id: number): void, (e: 'toggle-notifications', { id, value, }: { - id: string, + id: number, value: boolean, }): void, }>() diff --git a/frontend/components/notifications/management/NotificationsManagementPairedDevicesModal.vue b/frontend/components/notifications/management/NotificationsManagementPairedDevicesModal.vue index d4ed85e8a..c4fea645a 100644 --- a/frontend/components/notifications/management/NotificationsManagementPairedDevicesModal.vue +++ b/frontend/components/notifications/management/NotificationsManagementPairedDevicesModal.vue @@ -11,7 +11,7 @@ const handleToggleNotifications = ({ id, value, }: { - id: string, + id: number, value: boolean, }) => { notificationsManagementStore.setNotificationForPairedDevice({ diff --git a/frontend/stores/notifications/useNotificationsManagementStore.ts b/frontend/stores/notifications/useNotificationsManagementStore.ts index f27f09553..776caa87e 100644 --- a/frontend/stores/notifications/useNotificationsManagementStore.ts +++ b/frontend/stores/notifications/useNotificationsManagementStore.ts @@ -44,7 +44,7 @@ export const useNotificationsManagementStore = defineStore('notifications-manage ) } - const removeDevice = async (id: string) => { + const removeDevice = async (id: number) => { await fetch( API_PATH.NOTIFICATIONS_MANAGEMENT_PAIRED_DEVICES_DELETE, {}, @@ -61,7 +61,7 @@ export const useNotificationsManagementStore = defineStore('notifications-manage id, value, }: { - id: string, + id: number, value: boolean, }) => { await fetch( From a99d2b2a2b50eba3555cbfb6ca825788eda7cc8f Mon Sep 17 00:00:00 2001 From: Lucca Dukic <109136188+LuccaBitfly@users.noreply.github.com> Date: Tue, 22 Oct 2024 14:57:14 +0200 Subject: [PATCH 086/124] fix: implement boundaries for all mocked float values (between 0 and 1) See: BEDS-444 --- backend/pkg/api/data_access/dummy.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/backend/pkg/api/data_access/dummy.go b/backend/pkg/api/data_access/dummy.go index 24472343a..7fe7967c4 100644 --- a/backend/pkg/api/data_access/dummy.go +++ b/backend/pkg/api/data_access/dummy.go @@ -10,9 +10,9 @@ import ( "time" "github.com/go-faker/faker/v4" + "github.com/go-faker/faker/v4/pkg/interfaces" "github.com/go-faker/faker/v4/pkg/options" "github.com/gobitfly/beaconchain/pkg/api/enums" - "github.com/gobitfly/beaconchain/pkg/api/types" t "github.com/gobitfly/beaconchain/pkg/api/types" commontypes "github.com/gobitfly/beaconchain/pkg/commons/types" "github.com/gobitfly/beaconchain/pkg/userservice" @@ -55,7 +55,7 @@ func randomEthDecimal() decimal.Decimal { // must pass a pointer to the data func commonFakeData(a interface{}) error { // TODO fake decimal.Decimal - return faker.FakeData(a, options.WithRandomMapAndSliceMaxSize(5)) + return faker.FakeData(a, options.WithRandomMapAndSliceMaxSize(5), options.WithRandomFloatBoundaries(interfaces.RandomFloatBoundary{Start: 0, End: 1})) } func (d *DummyService) StartDataAccessServices() { @@ -393,7 +393,7 @@ func (d *DummyService) GetValidatorDashboardRocketPoolMinipools(ctx context.Cont } func (d *DummyService) GetAllNetworks() ([]t.NetworkInfo, error) { - return []types.NetworkInfo{ + return []t.NetworkInfo{ { ChainId: 1, Name: "ethereum", @@ -412,8 +412,8 @@ func (d *DummyService) GetAllNetworks() ([]t.NetworkInfo, error) { }, nil } -func (d *DummyService) GetAllClients() ([]types.ClientInfo, error) { - return []types.ClientInfo{ +func (d *DummyService) GetAllClients() ([]t.ClientInfo, error) { + return []t.ClientInfo{ // execution_layer { Id: 0, @@ -756,8 +756,8 @@ func (d *DummyService) GetValidatorDashboardMobileWidget(ctx context.Context, da return getDummyStruct[t.MobileWidgetData]() } -func (d *DummyService) GetUserMachineMetrics(ctx context.Context, userID uint64, limit int, offset int) (*types.MachineMetricsData, error) { - data, err := getDummyStruct[types.MachineMetricsData]() +func (d *DummyService) GetUserMachineMetrics(ctx context.Context, userID uint64, limit int, offset int) (*t.MachineMetricsData, error) { + data, err := getDummyStruct[t.MachineMetricsData]() if err != nil { return nil, err } From 312c0038fe64f3862d90208e8eb0275b111184fb Mon Sep 17 00:00:00 2001 From: Lucca Dukic <109136188+LuccaBitfly@users.noreply.github.com> Date: Tue, 22 Oct 2024 14:57:28 +0200 Subject: [PATCH 087/124] feat: add sliding expiration for sessions, extend session length to 1 year See: BEDS-444 --- backend/pkg/api/auth.go | 18 +++++++++++++++++- backend/pkg/api/router.go | 2 +- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/backend/pkg/api/auth.go b/backend/pkg/api/auth.go index dc8a62814..698c15eeb 100644 --- a/backend/pkg/api/auth.go +++ b/backend/pkg/api/auth.go @@ -13,6 +13,9 @@ import ( "github.com/gorilla/csrf" ) +var day time.Duration = time.Hour * 24 +var sessionDuration time.Duration = day * 365 + func newSessionManager(cfg *types.Config) *scs.SessionManager { // TODO: replace redis with user db down the line (or replace sessions with oauth2) pool := &redis.Pool{ @@ -23,7 +26,7 @@ func newSessionManager(cfg *types.Config) *scs.SessionManager { } scs := scs.New() - scs.Lifetime = time.Hour * 24 * 7 + scs.Lifetime = sessionDuration scs.Cookie.Name = "session_id" scs.Cookie.HttpOnly = true scs.Cookie.Persist = true @@ -42,6 +45,19 @@ func newSessionManager(cfg *types.Config) *scs.SessionManager { return scs } +// returns a middleware that extends the session expiration if the session is older than 1 day +func getSlidingSessionExpirationMiddleware(scs *scs.SessionManager) func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + deadline := scs.Deadline(r.Context()) // unauthenticated requests have deadline set to now+sessionDuration + if time.Until(deadline) < sessionDuration-day { + scs.SetDeadline(r.Context(), time.Now().Add(sessionDuration).UTC()) // setting to utc because library also does that internally + } + next.ServeHTTP(w, r) + }) + } +} + // returns goriila/csrf middleware with the given config settings func getCsrfProtectionMiddleware(cfg *types.Config) func(http.Handler) http.Handler { csrfBytes, err := hex.DecodeString(cfg.Frontend.CsrfAuthKey) diff --git a/backend/pkg/api/router.go b/backend/pkg/api/router.go index 946fca724..7e4f3699d 100644 --- a/backend/pkg/api/router.go +++ b/backend/pkg/api/router.go @@ -28,7 +28,7 @@ func NewApiRouter(dataAccessor dataaccess.DataAccessor, dummy dataaccess.DataAcc legacyRouter := apiRouter.PathPrefix("/v1").Subrouter() internalRouter := apiRouter.PathPrefix("/i").Subrouter() sessionManager := newSessionManager(cfg) - internalRouter.Use(sessionManager.LoadAndSave) + internalRouter.Use(sessionManager.LoadAndSave, getSlidingSessionExpirationMiddleware(sessionManager)) if !(cfg.Frontend.CsrfInsecure || cfg.Frontend.Debug) { internalRouter.Use(getCsrfProtectionMiddleware(cfg), csrfInjecterMiddleware) From c9dd69806e744444acc2b1bccddec9fbd5150f1f Mon Sep 17 00:00:00 2001 From: Lucca Dukic <109136188+LuccaBitfly@users.noreply.github.com> Date: Tue, 22 Oct 2024 09:38:46 +0200 Subject: [PATCH 088/124] feat: replace group_offline with group_efficiency_below event See: BEDS-847 --- backend/pkg/api/data_access/notifications.go | 22 ++-- backend/pkg/api/handlers/public.go | 12 +- backend/pkg/api/types/data_access.go | 2 +- backend/pkg/api/types/notifications.go | 12 +- backend/pkg/api/types/user.go | 38 +++---- backend/pkg/commons/db/user.go | 110 +++++++++---------- backend/pkg/commons/types/frontend.go | 3 +- frontend/types/api/notifications.ts | 10 +- 8 files changed, 103 insertions(+), 106 deletions(-) diff --git a/backend/pkg/api/data_access/notifications.go b/backend/pkg/api/data_access/notifications.go index 106b4b61f..76baf3091 100644 --- a/backend/pkg/api/data_access/notifications.go +++ b/backend/pkg/api/data_access/notifications.go @@ -83,7 +83,7 @@ const ( DiscordWebhookFormat string = "discord" - GroupOfflineThresholdDefault float64 = 0.1 + GroupEfficiencyBelowThresholdDefault float64 = 0.95 MaxCollateralThresholdDefault float64 = 1.0 MinCollateralThresholdDefault float64 = 0.2 ERC20TokenTransfersValueThresholdDefault float64 = 0.1 @@ -1456,7 +1456,7 @@ func (d *DataAccessService) GetNotificationSettings(ctx context.Context, userId func (d *DataAccessService) GetNotificationSettingsDefaultValues(ctx context.Context) (*t.NotificationSettingsDefaultValues, error) { return &t.NotificationSettingsDefaultValues{ - GroupOfflineThreshold: GroupOfflineThresholdDefault, + GroupEfficiencyBelowThreshold: GroupEfficiencyBelowThresholdDefault, MaxCollateralThreshold: MaxCollateralThresholdDefault, MinCollateralThreshold: MinCollateralThresholdDefault, ERC20TokenTransfersValueThreshold: ERC20TokenTransfersValueThresholdDefault, @@ -1859,9 +1859,9 @@ func (d *DataAccessService) GetNotificationSettingsDashboards(ctx context.Contex if dashboardType == ValidatorDashboardEventPrefix { resultMap[event.Filter] = &t.NotificationSettingsDashboardsTableRow{ Settings: t.NotificationSettingsValidatorDashboard{ - GroupOfflineThreshold: GroupOfflineThresholdDefault, - MaxCollateralThreshold: MaxCollateralThresholdDefault, - MinCollateralThreshold: MinCollateralThresholdDefault, + GroupEfficiencyBelowThreshold: GroupEfficiencyBelowThresholdDefault, + MaxCollateralThreshold: MaxCollateralThresholdDefault, + MinCollateralThreshold: MinCollateralThresholdDefault, }, } } else if dashboardType == AccountDashboardEventPrefix { @@ -1879,8 +1879,8 @@ func (d *DataAccessService) GetNotificationSettingsDashboards(ctx context.Contex case types.ValidatorIsOfflineEventName: settings.IsValidatorOfflineSubscribed = true case types.GroupIsOfflineEventName: - settings.IsGroupOfflineSubscribed = true - settings.GroupOfflineThreshold = event.Threshold + settings.IsGroupEfficiencyBelowSubscribed = true + settings.GroupEfficiencyBelowThreshold = event.Threshold case types.ValidatorMissedAttestationEventName: settings.IsAttestationsMissedSubscribed = true case types.ValidatorMissedProposalEventName, types.ValidatorExecutedProposalEventName: @@ -1926,9 +1926,9 @@ func (d *DataAccessService) GetNotificationSettingsDashboards(ctx context.Contex if _, ok := resultMap[key]; !ok { resultMap[key] = &t.NotificationSettingsDashboardsTableRow{ Settings: t.NotificationSettingsValidatorDashboard{ - GroupOfflineThreshold: GroupOfflineThresholdDefault, - MaxCollateralThreshold: MaxCollateralThresholdDefault, - MinCollateralThreshold: MinCollateralThresholdDefault, + GroupEfficiencyBelowThreshold: GroupEfficiencyBelowThresholdDefault, + MaxCollateralThreshold: MaxCollateralThresholdDefault, + MinCollateralThreshold: MinCollateralThresholdDefault, }, } } @@ -2110,7 +2110,7 @@ func (d *DataAccessService) UpdateNotificationSettingsValidatorDashboard(ctx con eventFilter := fmt.Sprintf("%s:%d:%d", ValidatorDashboardEventPrefix, dashboardId, groupId) d.AddOrRemoveEvent(&eventsToInsert, &eventsToDelete, settings.IsValidatorOfflineSubscribed, userId, types.ValidatorIsOfflineEventName, networkName, eventFilter, epoch, 0) - d.AddOrRemoveEvent(&eventsToInsert, &eventsToDelete, settings.IsGroupOfflineSubscribed, userId, types.GroupIsOfflineEventName, networkName, eventFilter, epoch, settings.GroupOfflineThreshold) + d.AddOrRemoveEvent(&eventsToInsert, &eventsToDelete, settings.IsGroupEfficiencyBelowSubscribed, userId, types.ValidatorGroupEfficiencyEventName, networkName, eventFilter, epoch, settings.GroupEfficiencyBelowThreshold) d.AddOrRemoveEvent(&eventsToInsert, &eventsToDelete, settings.IsAttestationsMissedSubscribed, userId, types.ValidatorMissedAttestationEventName, networkName, eventFilter, epoch, 0) d.AddOrRemoveEvent(&eventsToInsert, &eventsToDelete, settings.IsUpcomingBlockProposalSubscribed, userId, types.ValidatorUpcomingProposalEventName, networkName, eventFilter, epoch, 0) d.AddOrRemoveEvent(&eventsToInsert, &eventsToDelete, settings.IsSyncSubscribed, userId, types.SyncCommitteeSoon, networkName, eventFilter, epoch, 0) diff --git a/backend/pkg/api/handlers/public.go b/backend/pkg/api/handlers/public.go index f3792d7e5..13ae20af1 100644 --- a/backend/pkg/api/handlers/public.go +++ b/backend/pkg/api/handlers/public.go @@ -2495,9 +2495,9 @@ func (h *HandlerService) PublicGetUserNotificationSettingsDashboards(w http.Resp handleErr(w, r, errors.New("invalid settings type")) return } - if !userInfo.PremiumPerks.NotificationsValidatorDashboardGroupOffline && settings.IsGroupOfflineSubscribed { - settings.IsGroupOfflineSubscribed = false - settings.GroupOfflineThreshold = defaultSettings.GroupOfflineThreshold + if !userInfo.PremiumPerks.NotificationsValidatorDashboardGroupEfficiency && settings.IsGroupEfficiencyBelowSubscribed { + settings.IsGroupEfficiencyBelowSubscribed = false + settings.GroupEfficiencyBelowThreshold = defaultSettings.GroupEfficiencyBelowThreshold } if !userInfo.PremiumPerks.NotificationsValidatorDashboardRealTimeMode && settings.IsRealTimeModeEnabled { settings.IsRealTimeModeEnabled = false @@ -2537,7 +2537,7 @@ func (h *HandlerService) PublicPutUserNotificationSettingsValidatorDashboard(w h handleErr(w, r, err) return } - checkMinMax(&v, req.GroupOfflineThreshold, 0, 1, "group_offline_threshold") + checkMinMax(&v, req.GroupEfficiencyBelowThreshold, 0, 1, "group_offline_threshold") vars := mux.Vars(r) dashboardId := v.checkPrimaryDashboardId(vars["dashboard_id"]) groupId := v.checkExistingGroupId(vars["group_id"]) @@ -2553,8 +2553,8 @@ func (h *HandlerService) PublicPutUserNotificationSettingsValidatorDashboard(w h handleErr(w, r, err) return } - if !userInfo.PremiumPerks.NotificationsValidatorDashboardGroupOffline && req.IsGroupOfflineSubscribed { - returnForbidden(w, r, errors.New("user does not have premium perks to subscribe group offline")) + if !userInfo.PremiumPerks.NotificationsValidatorDashboardGroupEfficiency && req.IsGroupEfficiencyBelowSubscribed { + returnForbidden(w, r, errors.New("user does not have premium perks to subscribe group efficiency event")) return } if !userInfo.PremiumPerks.NotificationsValidatorDashboardRealTimeMode && req.IsRealTimeModeEnabled { diff --git a/backend/pkg/api/types/data_access.go b/backend/pkg/api/types/data_access.go index 66e4b5fb8..6a2859f6f 100644 --- a/backend/pkg/api/types/data_access.go +++ b/backend/pkg/api/types/data_access.go @@ -309,7 +309,7 @@ type MobileAppBundleStats struct { // Notification structs type NotificationSettingsDefaultValues struct { - GroupOfflineThreshold float64 + GroupEfficiencyBelowThreshold float64 MaxCollateralThreshold float64 MinCollateralThreshold float64 ERC20TokenTransfersValueThreshold float64 diff --git a/backend/pkg/api/types/notifications.go b/backend/pkg/api/types/notifications.go index e2baeb32c..21cb7cd6b 100644 --- a/backend/pkg/api/types/notifications.go +++ b/backend/pkg/api/types/notifications.go @@ -40,7 +40,7 @@ type NotificationDashboardsTableRow struct { GroupId uint64 `db:"group_id" json:"group_id"` GroupName string `db:"group_name" json:"group_name"` EntityCount uint64 `db:"entity_count" json:"entity_count"` - EventTypes pq.StringArray `db:"event_types" json:"event_types" tstype:"('validator_online' | 'validator_offline' | 'group_online' | 'group_offline' | 'attestation_missed' | 'proposal_success' | 'proposal_missed' | 'proposal_upcoming' | 'max_collateral' | 'min_collateral' | 'sync' | 'withdrawal' | 'validator_got_slashed' | 'validator_has_slashed' | 'incoming_tx' | 'outgoing_tx' | 'transfer_erc20' | 'transfer_erc721' | 'transfer_erc1155')[]" faker:"slice_len=2, oneof: validator_online, validator_offline, group_online, group_offline, attestation_missed, proposal_success, proposal_missed, proposal_upcoming, max_collateral, min_collateral, sync, withdrawal, validator_got_slashed, validator_has_slashed, incoming_tx, outgoing_tx, transfer_erc20, transfer_erc721, transfer_erc1155"` + EventTypes pq.StringArray `db:"event_types" json:"event_types" tstype:"('validator_online' | 'validator_offline' | 'group_efficiency_below' | 'attestation_missed' | 'proposal_success' | 'proposal_missed' | 'proposal_upcoming' | 'max_collateral' | 'min_collateral' | 'sync' | 'withdrawal' | 'validator_got_slashed' | 'validator_has_slashed' | 'incoming_tx' | 'outgoing_tx' | 'transfer_erc20' | 'transfer_erc721' | 'transfer_erc1155')[]" faker:"slice_len=2, oneof: validator_online, validator_offline, group_efficiency_below, attestation_missed, proposal_success, proposal_missed, proposal_upcoming, max_collateral, min_collateral, sync, withdrawal, validator_got_slashed, validator_has_slashed, incoming_tx, outgoing_tx, transfer_erc20, transfer_erc721, transfer_erc1155"` } type InternalGetUserNotificationDashboardsResponse ApiPagingResponse[NotificationDashboardsTableRow] @@ -62,8 +62,8 @@ type NotificationEventWithdrawal struct { type NotificationValidatorDashboardDetail struct { DashboardName string `db:"dashboard_name" json:"dashboard_name"` GroupName string `db:"group_name" json:"group_name"` - ValidatorOffline []uint64 `json:"validator_offline"` // validator indices - GroupOffline bool `json:"group_offline"` // TODO not filled yet + ValidatorOffline []uint64 `json:"validator_offline"` // validator indices + GroupEfficiencyBelow float64 `json:"group_efficiency_below,omitempty"` // fill with the `group_efficiency_below` threshold if event is present ProposalMissed []IndexSlots `json:"proposal_missed"` ProposalDone []IndexBlocks `json:"proposal_done"` UpcomingProposals []IndexSlots `json:"upcoming_proposals"` @@ -72,9 +72,7 @@ type NotificationValidatorDashboardDetail struct { AttestationMissed []IndexEpoch `json:"attestation_missed"` // index (epoch) Withdrawal []NotificationEventWithdrawal `json:"withdrawal"` ValidatorOfflineReminder []uint64 `json:"validator_offline_reminder"` // validator indices; TODO not filled yet - GroupOfflineReminder bool `json:"group_offline_reminder"` // TODO not filled yet ValidatorBackOnline []NotificationEventValidatorBackOnline `json:"validator_back_online"` - GroupBackOnline uint64 `json:"group_back_online"` // TODO not filled yet MinimumCollateralReached []Address `json:"min_collateral_reached"` // node addresses MaximumCollateralReached []Address `json:"max_collateral_reached"` // node addresses } @@ -205,8 +203,8 @@ type NotificationSettingsValidatorDashboard struct { IsRealTimeModeEnabled bool `json:"is_real_time_mode_enabled"` IsValidatorOfflineSubscribed bool `json:"is_validator_offline_subscribed"` - IsGroupOfflineSubscribed bool `json:"is_group_offline_subscribed"` - GroupOfflineThreshold float64 `json:"group_offline_threshold" faker:"boundary_start=0, boundary_end=1"` + IsGroupEfficiencyBelowSubscribed bool `json:"is_group_efficiency_below_subscribed"` + GroupEfficiencyBelowThreshold float64 `json:"group_efficiency_below_threshold" faker:"boundary_start=0, boundary_end=1"` IsAttestationsMissedSubscribed bool `json:"is_attestations_missed_subscribed"` IsBlockProposalSubscribed bool `json:"is_block_proposal_subscribed"` IsUpcomingBlockProposalSubscribed bool `json:"is_upcoming_block_proposal_subscribed"` diff --git a/backend/pkg/api/types/user.go b/backend/pkg/api/types/user.go index d31ac8e0e..b22a00e25 100644 --- a/backend/pkg/api/types/user.go +++ b/backend/pkg/api/types/user.go @@ -117,25 +117,25 @@ type ExtraDashboardValidatorsPremiumAddon struct { } type PremiumPerks struct { - AdFree bool `json:"ad_free"` // note that this is somhow redunant, since there is already ApiPerks.NoAds - ValidatorDashboards uint64 `json:"validator_dashboards"` - ValidatorsPerDashboard uint64 `json:"validators_per_dashboard"` - ValidatorGroupsPerDashboard uint64 `json:"validator_groups_per_dashboard"` - ShareCustomDashboards bool `json:"share_custom_dashboards"` - ManageDashboardViaApi bool `json:"manage_dashboard_via_api"` - BulkAdding bool `json:"bulk_adding"` - ChartHistorySeconds ChartHistorySeconds `json:"chart_history_seconds"` - EmailNotificationsPerDay uint64 `json:"email_notifications_per_day"` - ConfigureNotificationsViaApi bool `json:"configure_notifications_via_api"` - ValidatorGroupNotifications uint64 `json:"validator_group_notifications"` - WebhookEndpoints uint64 `json:"webhook_endpoints"` - MobileAppCustomThemes bool `json:"mobile_app_custom_themes"` - MobileAppWidget bool `json:"mobile_app_widget"` - MonitorMachines uint64 `json:"monitor_machines"` - MachineMonitoringHistorySeconds uint64 `json:"machine_monitoring_history_seconds"` - NotificationsMachineCustomThreshold bool `json:"notifications_machine_custom_threshold"` - NotificationsValidatorDashboardRealTimeMode bool `json:"notifications_validator_dashboard_real_time_mode"` - NotificationsValidatorDashboardGroupOffline bool `json:"notifications_validator_dashboard_group_offline"` + AdFree bool `json:"ad_free"` // note that this is somhow redunant, since there is already ApiPerks.NoAds + ValidatorDashboards uint64 `json:"validator_dashboards"` + ValidatorsPerDashboard uint64 `json:"validators_per_dashboard"` + ValidatorGroupsPerDashboard uint64 `json:"validator_groups_per_dashboard"` + ShareCustomDashboards bool `json:"share_custom_dashboards"` + ManageDashboardViaApi bool `json:"manage_dashboard_via_api"` + BulkAdding bool `json:"bulk_adding"` + ChartHistorySeconds ChartHistorySeconds `json:"chart_history_seconds"` + EmailNotificationsPerDay uint64 `json:"email_notifications_per_day"` + ConfigureNotificationsViaApi bool `json:"configure_notifications_via_api"` + ValidatorGroupNotifications uint64 `json:"validator_group_notifications"` + WebhookEndpoints uint64 `json:"webhook_endpoints"` + MobileAppCustomThemes bool `json:"mobile_app_custom_themes"` + MobileAppWidget bool `json:"mobile_app_widget"` + MonitorMachines uint64 `json:"monitor_machines"` + MachineMonitoringHistorySeconds uint64 `json:"machine_monitoring_history_seconds"` + NotificationsMachineCustomThreshold bool `json:"notifications_machine_custom_threshold"` + NotificationsValidatorDashboardRealTimeMode bool `json:"notifications_validator_dashboard_real_time_mode"` + NotificationsValidatorDashboardGroupEfficiency bool `json:"notifications_validator_dashboard_group_offline"` } // TODO @patrick post-beta StripeCreateCheckoutSession and StripeCustomerPortal are currently served from v1 (loadbalanced), Once V1 is not affected by this anymore, consider wrapping this with ApiDataResponse diff --git a/backend/pkg/commons/db/user.go b/backend/pkg/commons/db/user.go index fd7712bfa..bf62ee8e2 100644 --- a/backend/pkg/commons/db/user.go +++ b/backend/pkg/commons/db/user.go @@ -37,17 +37,17 @@ var freeTierProduct t.PremiumProduct = t.PremiumProduct{ Daily: 0, Weekly: 0, }, - EmailNotificationsPerDay: 10, - ConfigureNotificationsViaApi: false, - ValidatorGroupNotifications: 1, - WebhookEndpoints: 1, - MobileAppCustomThemes: false, - MobileAppWidget: false, - MonitorMachines: 1, - MachineMonitoringHistorySeconds: 3600 * 3, - NotificationsMachineCustomThreshold: false, - NotificationsValidatorDashboardRealTimeMode: false, - NotificationsValidatorDashboardGroupOffline: false, + EmailNotificationsPerDay: 10, + ConfigureNotificationsViaApi: false, + ValidatorGroupNotifications: 1, + WebhookEndpoints: 1, + MobileAppCustomThemes: false, + MobileAppWidget: false, + MonitorMachines: 1, + MachineMonitoringHistorySeconds: 3600 * 3, + NotificationsMachineCustomThreshold: false, + NotificationsValidatorDashboardRealTimeMode: false, + NotificationsValidatorDashboardGroupEfficiency: false, }, PricePerMonthEur: 0, PricePerYearEur: 0, @@ -69,17 +69,17 @@ var adminPerks = t.PremiumPerks{ Daily: maxJsInt, Weekly: maxJsInt, }, - EmailNotificationsPerDay: maxJsInt, - ConfigureNotificationsViaApi: true, - ValidatorGroupNotifications: maxJsInt, - WebhookEndpoints: maxJsInt, - MobileAppCustomThemes: true, - MobileAppWidget: true, - MonitorMachines: maxJsInt, - MachineMonitoringHistorySeconds: maxJsInt, - NotificationsMachineCustomThreshold: true, - NotificationsValidatorDashboardRealTimeMode: true, - NotificationsValidatorDashboardGroupOffline: true, + EmailNotificationsPerDay: maxJsInt, + ConfigureNotificationsViaApi: true, + ValidatorGroupNotifications: maxJsInt, + WebhookEndpoints: maxJsInt, + MobileAppCustomThemes: true, + MobileAppWidget: true, + MonitorMachines: maxJsInt, + MachineMonitoringHistorySeconds: maxJsInt, + NotificationsMachineCustomThreshold: true, + NotificationsValidatorDashboardRealTimeMode: true, + NotificationsValidatorDashboardGroupEfficiency: true, } func GetUserInfo(ctx context.Context, userId uint64, userDbReader *sqlx.DB) (*t.UserInfo, error) { @@ -350,17 +350,17 @@ func GetProductSummary(ctx context.Context) (*t.ProductSummary, error) { // TODO Daily: month, Weekly: 0, }, - EmailNotificationsPerDay: 15, - ConfigureNotificationsViaApi: false, - ValidatorGroupNotifications: 3, - WebhookEndpoints: 3, - MobileAppCustomThemes: true, - MobileAppWidget: true, - MonitorMachines: 2, - MachineMonitoringHistorySeconds: 3600 * 24 * 30, - NotificationsMachineCustomThreshold: true, - NotificationsValidatorDashboardRealTimeMode: true, - NotificationsValidatorDashboardGroupOffline: true, + EmailNotificationsPerDay: 15, + ConfigureNotificationsViaApi: false, + ValidatorGroupNotifications: 3, + WebhookEndpoints: 3, + MobileAppCustomThemes: true, + MobileAppWidget: true, + MonitorMachines: 2, + MachineMonitoringHistorySeconds: 3600 * 24 * 30, + NotificationsMachineCustomThreshold: true, + NotificationsValidatorDashboardRealTimeMode: true, + NotificationsValidatorDashboardGroupEfficiency: true, }, PricePerMonthEur: 9.99, PricePerYearEur: 107.88, @@ -385,17 +385,17 @@ func GetProductSummary(ctx context.Context) (*t.ProductSummary, error) { // TODO Daily: 2 * month, Weekly: 8 * week, }, - EmailNotificationsPerDay: 20, - ConfigureNotificationsViaApi: false, - ValidatorGroupNotifications: 10, - WebhookEndpoints: 10, - MobileAppCustomThemes: true, - MobileAppWidget: true, - MonitorMachines: 10, - MachineMonitoringHistorySeconds: 3600 * 24 * 30, - NotificationsMachineCustomThreshold: true, - NotificationsValidatorDashboardRealTimeMode: true, - NotificationsValidatorDashboardGroupOffline: true, + EmailNotificationsPerDay: 20, + ConfigureNotificationsViaApi: false, + ValidatorGroupNotifications: 10, + WebhookEndpoints: 10, + MobileAppCustomThemes: true, + MobileAppWidget: true, + MonitorMachines: 10, + MachineMonitoringHistorySeconds: 3600 * 24 * 30, + NotificationsMachineCustomThreshold: true, + NotificationsValidatorDashboardRealTimeMode: true, + NotificationsValidatorDashboardGroupEfficiency: true, }, PricePerMonthEur: 29.99, PricePerYearEur: 311.88, @@ -420,17 +420,17 @@ func GetProductSummary(ctx context.Context) (*t.ProductSummary, error) { // TODO Daily: 12 * month, Weekly: maxJsInt, }, - EmailNotificationsPerDay: 50, - ConfigureNotificationsViaApi: true, - ValidatorGroupNotifications: 60, - WebhookEndpoints: 30, - MobileAppCustomThemes: true, - MobileAppWidget: true, - MonitorMachines: 10, - MachineMonitoringHistorySeconds: 3600 * 24 * 30, - NotificationsMachineCustomThreshold: true, - NotificationsValidatorDashboardRealTimeMode: true, - NotificationsValidatorDashboardGroupOffline: true, + EmailNotificationsPerDay: 50, + ConfigureNotificationsViaApi: true, + ValidatorGroupNotifications: 60, + WebhookEndpoints: 30, + MobileAppCustomThemes: true, + MobileAppWidget: true, + MonitorMachines: 10, + MachineMonitoringHistorySeconds: 3600 * 24 * 30, + NotificationsMachineCustomThreshold: true, + NotificationsValidatorDashboardRealTimeMode: true, + NotificationsValidatorDashboardGroupEfficiency: true, }, PricePerMonthEur: 49.99, PricePerYearEur: 479.88, diff --git a/backend/pkg/commons/types/frontend.go b/backend/pkg/commons/types/frontend.go index b45f1b06d..5feb2a5f4 100644 --- a/backend/pkg/commons/types/frontend.go +++ b/backend/pkg/commons/types/frontend.go @@ -86,7 +86,8 @@ const ( // Validator dashboard events ValidatorIsOfflineEventName EventName = "validator_is_offline" ValidatorIsOnlineEventName EventName = "validator_is_online" - GroupIsOfflineEventName EventName = "group_is_offline" + GroupIsOfflineEventName EventName = "group_is_offline" // TODO @BACKEND TEAM: probably remove this and all relevant code + ValidatorGroupEfficiencyEventName EventName = "validator_group_efficiency" ValidatorMissedAttestationEventName EventName = "validator_attestation_missed" ValidatorProposalEventName EventName = "validator_proposal" ValidatorUpcomingProposalEventName EventName = "validator_proposal_upcoming" diff --git a/frontend/types/api/notifications.ts b/frontend/types/api/notifications.ts index 7abe5c7f9..e6d01d51f 100644 --- a/frontend/types/api/notifications.ts +++ b/frontend/types/api/notifications.ts @@ -43,7 +43,7 @@ export interface NotificationDashboardsTableRow { group_id: number /* uint64 */; group_name: string; entity_count: number /* uint64 */; - event_types: ('validator_online' | 'validator_offline' | 'group_online' | 'group_offline' | 'attestation_missed' | 'proposal_success' | 'proposal_missed' | 'proposal_upcoming' | 'max_collateral' | 'min_collateral' | 'sync' | 'withdrawal' | 'validator_got_slashed' | 'validator_has_slashed' | 'incoming_tx' | 'outgoing_tx' | 'transfer_erc20' | 'transfer_erc721' | 'transfer_erc1155')[]; + event_types: ('validator_online' | 'validator_offline' | 'group_efficiency_below' | 'attestation_missed' | 'proposal_success' | 'proposal_missed' | 'proposal_upcoming' | 'max_collateral' | 'min_collateral' | 'sync' | 'withdrawal' | 'validator_got_slashed' | 'validator_has_slashed' | 'incoming_tx' | 'outgoing_tx' | 'transfer_erc20' | 'transfer_erc721' | 'transfer_erc1155')[]; } export type InternalGetUserNotificationDashboardsResponse = ApiPagingResponse; export interface NotificationEventValidatorBackOnline { @@ -59,7 +59,7 @@ export interface NotificationValidatorDashboardDetail { dashboard_name: string; group_name: string; validator_offline: number /* uint64 */[]; // validator indices - group_offline: boolean; // TODO not filled yet + group_efficiency_below?: number /* float64 */; // fill with the `group_efficiency_below` threshold if event is present proposal_missed: IndexSlots[]; proposal_done: IndexBlocks[]; upcoming_proposals: IndexSlots[]; @@ -68,9 +68,7 @@ export interface NotificationValidatorDashboardDetail { attestation_missed: IndexEpoch[]; // index (epoch) withdrawal: NotificationEventWithdrawal[]; validator_offline_reminder: number /* uint64 */[]; // validator indices; TODO not filled yet - group_offline_reminder: boolean; // TODO not filled yet validator_back_online: NotificationEventValidatorBackOnline[]; - group_back_online: number /* uint64 */; // TODO not filled yet min_collateral_reached: Address[]; // node addresses max_collateral_reached: Address[]; // node addresses } @@ -191,8 +189,8 @@ export interface NotificationSettingsValidatorDashboard { is_webhook_discord_enabled: boolean; is_real_time_mode_enabled: boolean; is_validator_offline_subscribed: boolean; - is_group_offline_subscribed: boolean; - group_offline_threshold: number /* float64 */; + is_group_efficiency_below_subscribed: boolean; + group_efficiency_below_threshold: number /* float64 */; is_attestations_missed_subscribed: boolean; is_block_proposal_subscribed: boolean; is_upcoming_block_proposal_subscribed: boolean; From 94271bff5ee6bb0ac525de897fb5a59bc1794af0 Mon Sep 17 00:00:00 2001 From: Lucca Dukic <109136188+LuccaBitfly@users.noreply.github.com> Date: Tue, 22 Oct 2024 10:38:22 +0200 Subject: [PATCH 089/124] fix: remove all mentions of group offline event See: BEDS-847 --- backend/pkg/api/data_access/notifications.go | 15 +-------------- backend/pkg/commons/types/frontend.go | 4 +--- 2 files changed, 2 insertions(+), 17 deletions(-) diff --git a/backend/pkg/api/data_access/notifications.go b/backend/pkg/api/data_access/notifications.go index 76baf3091..8f2e9fa72 100644 --- a/backend/pkg/api/data_access/notifications.go +++ b/backend/pkg/api/data_access/notifications.go @@ -568,19 +568,6 @@ func (d *DataAccessService) GetValidatorDashboardNotificationDetails(ctx context continue } notificationDetails.ValidatorBackOnline = append(notificationDetails.ValidatorBackOnline, t.NotificationEventValidatorBackOnline{Index: curNotification.ValidatorIndex, EpochCount: curNotification.Epoch}) - case types.ValidatorGroupIsOfflineEventName: - // TODO type / collection not present yet, skipping - /*curNotification, ok := not.(*notification.validatorGroupIsOfflineNotification) - if !ok { - return nil, fmt.Errorf("failed to cast notification to validatorGroupIsOfflineNotification") - } - if curNotification.Status == 0 { - notificationDetails.GroupOffline = ... - notificationDetails.GroupOfflineReminder = ... - } else { - notificationDetails.GroupBackOnline = ... - } - */ case types.ValidatorReceivedWithdrawalEventName: curNotification, ok := notification.(*n.ValidatorWithdrawalNotification) if !ok { @@ -1878,7 +1865,7 @@ func (d *DataAccessService) GetNotificationSettingsDashboards(ctx context.Contex switch eventName { case types.ValidatorIsOfflineEventName: settings.IsValidatorOfflineSubscribed = true - case types.GroupIsOfflineEventName: + case types.ValidatorGroupEfficiencyEventName: settings.IsGroupEfficiencyBelowSubscribed = true settings.GroupEfficiencyBelowThreshold = event.Threshold case types.ValidatorMissedAttestationEventName: diff --git a/backend/pkg/commons/types/frontend.go b/backend/pkg/commons/types/frontend.go index 5feb2a5f4..7fafe28be 100644 --- a/backend/pkg/commons/types/frontend.go +++ b/backend/pkg/commons/types/frontend.go @@ -68,8 +68,7 @@ const ( ValidatorMissedProposalEventName EventName = "validator_proposal_missed" ValidatorExecutedProposalEventName EventName = "validator_proposal_submitted" - ValidatorDidSlashEventName EventName = "validator_did_slash" - ValidatorGroupIsOfflineEventName EventName = "validator_group_is_offline" + ValidatorDidSlashEventName EventName = "validator_did_slash" ValidatorReceivedDepositEventName EventName = "validator_received_deposit" NetworkSlashingEventName EventName = "network_slashing" @@ -86,7 +85,6 @@ const ( // Validator dashboard events ValidatorIsOfflineEventName EventName = "validator_is_offline" ValidatorIsOnlineEventName EventName = "validator_is_online" - GroupIsOfflineEventName EventName = "group_is_offline" // TODO @BACKEND TEAM: probably remove this and all relevant code ValidatorGroupEfficiencyEventName EventName = "validator_group_efficiency" ValidatorMissedAttestationEventName EventName = "validator_attestation_missed" ValidatorProposalEventName EventName = "validator_proposal" From a293168b79e7744c34979dc510f5fabb724a2ba2 Mon Sep 17 00:00:00 2001 From: Lucca Dukic <109136188+LuccaBitfly@users.noreply.github.com> Date: Tue, 22 Oct 2024 10:39:15 +0200 Subject: [PATCH 090/124] refactor: remove notifications realtime mode from backend See: BEDS-847 --- backend/pkg/api/data_access/notifications.go | 6 +----- backend/pkg/api/handlers/public.go | 7 ------- backend/pkg/api/types/notifications.go | 1 - backend/pkg/api/types/user.go | 3 +-- backend/pkg/commons/db/user.go | 5 ----- frontend/types/api/notifications.ts | 1 - frontend/types/api/user.ts | 3 +-- 7 files changed, 3 insertions(+), 23 deletions(-) diff --git a/backend/pkg/api/data_access/notifications.go b/backend/pkg/api/data_access/notifications.go index 8f2e9fa72..ee812ce09 100644 --- a/backend/pkg/api/data_access/notifications.go +++ b/backend/pkg/api/data_access/notifications.go @@ -1759,7 +1759,6 @@ func (d *DataAccessService) GetNotificationSettingsDashboards(ctx context.Contex Network uint64 `db:"network"` WebhookUrl sql.NullString `db:"webhook_target"` IsWebhookDiscordEnabled sql.NullBool `db:"discord_webhook"` - IsRealTimeModeEnabled sql.NullBool `db:"realtime_notifications"` }{} wg.Go(func() error { err := d.alloyReader.SelectContext(ctx, &valDashboards, ` @@ -1771,7 +1770,6 @@ func (d *DataAccessService) GetNotificationSettingsDashboards(ctx context.Contex d.network, g.webhook_target, (g.webhook_format = $1) AS discord_webhook, - g.realtime_notifications FROM users_val_dashboards d INNER JOIN users_val_dashboards_groups g ON d.id = g.dashboard_id WHERE d.user_id = $2`, DiscordWebhookFormat, userId) @@ -1932,7 +1930,6 @@ func (d *DataAccessService) GetNotificationSettingsDashboards(ctx context.Contex if valSettings, ok := resultMap[key].Settings.(*t.NotificationSettingsValidatorDashboard); ok { valSettings.WebhookUrl = valDashboard.WebhookUrl.String valSettings.IsWebhookDiscordEnabled = valDashboard.IsWebhookDiscordEnabled.Bool - valSettings.IsRealTimeModeEnabled = valDashboard.IsRealTimeModeEnabled.Bool } } @@ -2159,8 +2156,7 @@ func (d *DataAccessService) UpdateNotificationSettingsValidatorDashboard(ctx con SET webhook_target = NULLIF($1, ''), webhook_format = CASE WHEN $2 THEN $3 ELSE NULL END, - realtime_notifications = CASE WHEN $4 THEN TRUE ELSE NULL END - WHERE dashboard_id = $5 AND id = $6`, settings.WebhookUrl, settings.IsWebhookDiscordEnabled, DiscordWebhookFormat, settings.IsRealTimeModeEnabled, dashboardId, groupId) + WHERE dashboard_id = $4 AND id = $5`, settings.WebhookUrl, settings.IsWebhookDiscordEnabled, DiscordWebhookFormat, dashboardId, groupId) if err != nil { return err } diff --git a/backend/pkg/api/handlers/public.go b/backend/pkg/api/handlers/public.go index 13ae20af1..92b650d75 100644 --- a/backend/pkg/api/handlers/public.go +++ b/backend/pkg/api/handlers/public.go @@ -2499,9 +2499,6 @@ func (h *HandlerService) PublicGetUserNotificationSettingsDashboards(w http.Resp settings.IsGroupEfficiencyBelowSubscribed = false settings.GroupEfficiencyBelowThreshold = defaultSettings.GroupEfficiencyBelowThreshold } - if !userInfo.PremiumPerks.NotificationsValidatorDashboardRealTimeMode && settings.IsRealTimeModeEnabled { - settings.IsRealTimeModeEnabled = false - } data[i].Settings = settings } response := types.InternalGetUserNotificationSettingsDashboardsResponse{ @@ -2557,10 +2554,6 @@ func (h *HandlerService) PublicPutUserNotificationSettingsValidatorDashboard(w h returnForbidden(w, r, errors.New("user does not have premium perks to subscribe group efficiency event")) return } - if !userInfo.PremiumPerks.NotificationsValidatorDashboardRealTimeMode && req.IsRealTimeModeEnabled { - returnForbidden(w, r, errors.New("user does not have premium perks to subscribe real time mode")) - return - } err = h.getDataAccessor(r).UpdateNotificationSettingsValidatorDashboard(r.Context(), userId, dashboardId, groupId, req) if err != nil { diff --git a/backend/pkg/api/types/notifications.go b/backend/pkg/api/types/notifications.go index 21cb7cd6b..777cad8b3 100644 --- a/backend/pkg/api/types/notifications.go +++ b/backend/pkg/api/types/notifications.go @@ -200,7 +200,6 @@ type InternalGetUserNotificationSettingsResponse ApiDataResponse[NotificationSet type NotificationSettingsValidatorDashboard struct { WebhookUrl string `json:"webhook_url" faker:"url"` IsWebhookDiscordEnabled bool `json:"is_webhook_discord_enabled"` - IsRealTimeModeEnabled bool `json:"is_real_time_mode_enabled"` IsValidatorOfflineSubscribed bool `json:"is_validator_offline_subscribed"` IsGroupEfficiencyBelowSubscribed bool `json:"is_group_efficiency_below_subscribed"` diff --git a/backend/pkg/api/types/user.go b/backend/pkg/api/types/user.go index b22a00e25..8e4a30dc7 100644 --- a/backend/pkg/api/types/user.go +++ b/backend/pkg/api/types/user.go @@ -134,8 +134,7 @@ type PremiumPerks struct { MonitorMachines uint64 `json:"monitor_machines"` MachineMonitoringHistorySeconds uint64 `json:"machine_monitoring_history_seconds"` NotificationsMachineCustomThreshold bool `json:"notifications_machine_custom_threshold"` - NotificationsValidatorDashboardRealTimeMode bool `json:"notifications_validator_dashboard_real_time_mode"` - NotificationsValidatorDashboardGroupEfficiency bool `json:"notifications_validator_dashboard_group_offline"` + NotificationsValidatorDashboardGroupEfficiency bool `json:"notifications_validator_dashboard_group_efficiency"` } // TODO @patrick post-beta StripeCreateCheckoutSession and StripeCustomerPortal are currently served from v1 (loadbalanced), Once V1 is not affected by this anymore, consider wrapping this with ApiDataResponse diff --git a/backend/pkg/commons/db/user.go b/backend/pkg/commons/db/user.go index bf62ee8e2..8ad275f8b 100644 --- a/backend/pkg/commons/db/user.go +++ b/backend/pkg/commons/db/user.go @@ -46,7 +46,6 @@ var freeTierProduct t.PremiumProduct = t.PremiumProduct{ MonitorMachines: 1, MachineMonitoringHistorySeconds: 3600 * 3, NotificationsMachineCustomThreshold: false, - NotificationsValidatorDashboardRealTimeMode: false, NotificationsValidatorDashboardGroupEfficiency: false, }, PricePerMonthEur: 0, @@ -78,7 +77,6 @@ var adminPerks = t.PremiumPerks{ MonitorMachines: maxJsInt, MachineMonitoringHistorySeconds: maxJsInt, NotificationsMachineCustomThreshold: true, - NotificationsValidatorDashboardRealTimeMode: true, NotificationsValidatorDashboardGroupEfficiency: true, } @@ -359,7 +357,6 @@ func GetProductSummary(ctx context.Context) (*t.ProductSummary, error) { // TODO MonitorMachines: 2, MachineMonitoringHistorySeconds: 3600 * 24 * 30, NotificationsMachineCustomThreshold: true, - NotificationsValidatorDashboardRealTimeMode: true, NotificationsValidatorDashboardGroupEfficiency: true, }, PricePerMonthEur: 9.99, @@ -394,7 +391,6 @@ func GetProductSummary(ctx context.Context) (*t.ProductSummary, error) { // TODO MonitorMachines: 10, MachineMonitoringHistorySeconds: 3600 * 24 * 30, NotificationsMachineCustomThreshold: true, - NotificationsValidatorDashboardRealTimeMode: true, NotificationsValidatorDashboardGroupEfficiency: true, }, PricePerMonthEur: 29.99, @@ -429,7 +425,6 @@ func GetProductSummary(ctx context.Context) (*t.ProductSummary, error) { // TODO MonitorMachines: 10, MachineMonitoringHistorySeconds: 3600 * 24 * 30, NotificationsMachineCustomThreshold: true, - NotificationsValidatorDashboardRealTimeMode: true, NotificationsValidatorDashboardGroupEfficiency: true, }, PricePerMonthEur: 49.99, diff --git a/frontend/types/api/notifications.ts b/frontend/types/api/notifications.ts index e6d01d51f..230ac73e0 100644 --- a/frontend/types/api/notifications.ts +++ b/frontend/types/api/notifications.ts @@ -187,7 +187,6 @@ export type InternalGetUserNotificationSettingsResponse = ApiDataResponse Date: Tue, 22 Oct 2024 11:18:22 +0200 Subject: [PATCH 091/124] fix: remove commas from queries See: BEDS-847 --- backend/pkg/api/data_access/notifications.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/pkg/api/data_access/notifications.go b/backend/pkg/api/data_access/notifications.go index ee812ce09..d9c9f71fa 100644 --- a/backend/pkg/api/data_access/notifications.go +++ b/backend/pkg/api/data_access/notifications.go @@ -1769,7 +1769,7 @@ func (d *DataAccessService) GetNotificationSettingsDashboards(ctx context.Contex g.name AS group_name, d.network, g.webhook_target, - (g.webhook_format = $1) AS discord_webhook, + (g.webhook_format = $1) AS discord_webhook FROM users_val_dashboards d INNER JOIN users_val_dashboards_groups g ON d.id = g.dashboard_id WHERE d.user_id = $2`, DiscordWebhookFormat, userId) @@ -2155,7 +2155,7 @@ func (d *DataAccessService) UpdateNotificationSettingsValidatorDashboard(ctx con UPDATE users_val_dashboards_groups SET webhook_target = NULLIF($1, ''), - webhook_format = CASE WHEN $2 THEN $3 ELSE NULL END, + webhook_format = CASE WHEN $2 THEN $3 ELSE NULL END WHERE dashboard_id = $4 AND id = $5`, settings.WebhookUrl, settings.IsWebhookDiscordEnabled, DiscordWebhookFormat, dashboardId, groupId) if err != nil { return err From df7d583065dd7f43959c89cd03c5bc16d8e633ea Mon Sep 17 00:00:00 2001 From: marcel-bitfly <174338434+marcel-bitfly@users.noreply.github.com> Date: Tue, 22 Oct 2024 15:11:56 +0200 Subject: [PATCH 092/124] feat(notifications): add `group efficiency` See: BEDS-656 --- frontend/components/bc/BcAccordion.vue | 19 ++++++++++----- .../NotificationsDashboardDialogEntity.vue | 24 +++++++++++++++++++ .../NotificationsDashboardsTable.vue | 6 ++--- .../NotificationsManagementDashboards.vue | 9 +++---- ...ificationsManagementSubscriptionDialog.vue | 19 +++++++++++---- .../playground/PlaygroundDialog.vue | 5 ++-- .../useNotificationsManagementDashboards.ts | 5 ++-- frontend/locales/en.json | 13 ++++++++-- 8 files changed, 72 insertions(+), 28 deletions(-) diff --git a/frontend/components/bc/BcAccordion.vue b/frontend/components/bc/BcAccordion.vue index aed38db72..a6c8dfb4b 100644 --- a/frontend/components/bc/BcAccordion.vue +++ b/frontend/components/bc/BcAccordion.vue @@ -4,6 +4,7 @@ import { FontAwesomeIcon } from '@fortawesome/vue-fontawesome' const props = defineProps<{ infoCopy?: string, + item?: T, items?: T[], open?: boolean, }>() @@ -56,17 +57,22 @@ const copyText = async () => {
  • - +
+ + + + + + user.value?.premium_perks.notifications_validator_dashboard_group_efficiency, +) function closeDialog(): void { dialogRef?.value.close() } @@ -17,10 +21,9 @@ function closeDialog(): void { const checkboxes = ref({ is_attestations_missed_subscribed: props.value?.is_attestations_missed_subscribed ?? false, is_block_proposal_subscribed: props.value?.is_block_proposal_subscribed ?? false, - is_group_offline_subscribed: props.value?.is_group_offline_subscribed ?? false, + is_group_efficiency_below_subscribed: props.value?.is_group_efficiency_below_subscribed ?? false, is_max_collateral_subscribed: props.value?.is_max_collateral_subscribed ?? false, is_min_collateral_subscribed: props.value?.is_min_collateral_subscribed ?? false, - is_real_time_mode_enabled: props.value?.is_real_time_mode_enabled ?? false, is_slashed_subscribed: props.value?.is_slashed_subscribed ?? false, is_sync_subscribed: props.value?.is_sync_subscribed ?? false, is_upcoming_block_proposal_subscribed: props.value?.is_upcoming_block_proposal_subscribed ?? false, @@ -28,7 +31,7 @@ const checkboxes = ref({ is_withdrawal_processed_subscribed: props.value?.is_withdrawal_processed_subscribed ?? false, }) const thresholds = ref({ - group_offline_threshold: formatFraction(props.value?.group_offline_threshold ?? 0), + group_efficiency_below_threshold: formatFraction(props.value?.group_efficiency_below_threshold ?? 0), max_collateral_threshold: formatFraction(props.value?.max_collateral_threshold ?? 0), min_collateral_threshold: formatFraction(props.value?.min_collateral_threshold ?? 0), }) @@ -43,7 +46,7 @@ watchDebounced([ ], () => { emit('change-settings', { ...checkboxes.value, - group_offline_threshold: Number(formatToFraction(thresholds.value.group_offline_threshold)), + group_efficiency_below_threshold: Number(formatToFraction(thresholds.value.group_efficiency_below_threshold)), max_collateral_threshold: Number(formatToFraction(thresholds.value.max_collateral_threshold)), min_collateral_threshold: Number(formatToFraction(thresholds.value.min_collateral_threshold)), }) @@ -118,6 +121,14 @@ watch(hasAllEvents, () => { v-model:checkbox="checkboxes.is_slashed_subscribed" :label="$t('notifications.subscriptions.validators.validator_got_slashed.label')" /> + { } const validatorSub: NotificationSettingsValidatorDashboard = { - group_offline_threshold: 0, // means "deactivated/unchecked" + group_efficiency_below_threshold: 0, is_attestations_missed_subscribed: true, is_block_proposal_subscribed: true, - is_group_offline_subscribed: true, + is_group_efficiency_below_subscribed: true, is_max_collateral_subscribed: false, is_min_collateral_subscribed: false, - is_real_time_mode_enabled: false, is_slashed_subscribed: false, is_sync_subscribed: true, is_upcoming_block_proposal_subscribed: false, diff --git a/frontend/composables/notifications/useNotificationsManagementDashboards.ts b/frontend/composables/notifications/useNotificationsManagementDashboards.ts index a27fbf32d..cb4b0b799 100644 --- a/frontend/composables/notifications/useNotificationsManagementDashboards.ts +++ b/frontend/composables/notifications/useNotificationsManagementDashboards.ts @@ -83,13 +83,12 @@ export function useNotificationsManagementDashboards() { return } const accountDashboarSettings = settings as NotificationSettingsValidatorDashboard - accountDashboarSettings.group_offline_threshold = 0 + accountDashboarSettings.group_efficiency_below_threshold = 0 accountDashboarSettings.is_attestations_missed_subscribed = false accountDashboarSettings.is_block_proposal_subscribed = false - accountDashboarSettings.is_group_offline_subscribed = false + accountDashboarSettings.is_group_efficiency_below_subscribed = false accountDashboarSettings.is_max_collateral_subscribed = false accountDashboarSettings.is_min_collateral_subscribed = false - accountDashboarSettings.is_real_time_mode_enabled = false accountDashboarSettings.is_slashed_subscribed = false accountDashboarSettings.is_sync_subscribed = false accountDashboarSettings.is_upcoming_block_proposal_subscribed = false diff --git a/frontend/locales/en.json b/frontend/locales/en.json index 66a5f0ec2..f2256ef24 100644 --- a/frontend/locales/en.json +++ b/frontend/locales/en.json @@ -616,6 +616,8 @@ }, "entity": { "attestation_missed":"Attestation missed", + "group_efficiency": "Group efficiency ", + "group_efficiency_text": "efficiency below {percentage}", "max_collateral":"Maximum collateral reached", "min_collateral":"Minimum collateral reached", "proposal_done": "Proposal done", @@ -650,8 +652,7 @@ }, "event_type": { "attestation_missed": "Attestation missed", - "group_offline": "Group offline", - "group_online": "Group online", + "group_efficiency_below": "Low group efficiency", "incoming_tx": "Incoming Tx.", "max_collateral": "Max. collateral", "min_collateral": "Min. collateral", @@ -814,6 +815,10 @@ "erc1155_token_transfers": { "label": "ERC1155 token transfers" }, + "group_efficiency": { + "info": "efficiency below {percentage}", + "label": "Group efficiency" + }, "ignore_spam_transactions": { "hint": [ "Ignores", @@ -844,6 +849,10 @@ "label": "Block proposal (missed & success)" }, "explanation": "All notifications are sent after network finality (~20min).", + "group_efficiency": { + "info": "Notifies you if your group's efficiency falls below {percentage}% in any given epoch", + "label": "Group efficiency below" + }, "max_collateral_reached": { "label": "Max collateral reached" }, From daa4c3edf032dad1e44ad537c70c0a32e44b7bcb Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Tue, 22 Oct 2024 14:06:45 +0000 Subject: [PATCH 093/124] feat(notification): implement group efficiency notification collection --- backend/cmd/misc/main.go | 42 ++--- backend/pkg/api/data_access/notifications.go | 1 + backend/pkg/notification/collection.go | 176 +++++++++++++++---- backend/pkg/notification/notifications.go | 10 +- backend/pkg/notification/pubkey_cache.go | 2 +- backend/pkg/notification/types.go | 35 ++-- 6 files changed, 195 insertions(+), 71 deletions(-) diff --git a/backend/cmd/misc/main.go b/backend/cmd/misc/main.go index 5555c2e7e..c567a0371 100644 --- a/backend/cmd/misc/main.go +++ b/backend/cmd/misc/main.go @@ -247,27 +247,27 @@ func Run() { } // clickhouse - // db.ClickHouseWriter, db.ClickHouseReader = db.MustInitDB(&types.DatabaseConfig{ - // Username: cfg.ClickHouse.WriterDatabase.Username, - // Password: cfg.ClickHouse.WriterDatabase.Password, - // Name: cfg.ClickHouse.WriterDatabase.Name, - // Host: cfg.ClickHouse.WriterDatabase.Host, - // Port: cfg.ClickHouse.WriterDatabase.Port, - // MaxOpenConns: cfg.ClickHouse.WriterDatabase.MaxOpenConns, - // SSL: true, - // MaxIdleConns: cfg.ClickHouse.WriterDatabase.MaxIdleConns, - // }, &types.DatabaseConfig{ - // Username: cfg.ClickHouse.ReaderDatabase.Username, - // Password: cfg.ClickHouse.ReaderDatabase.Password, - // Name: cfg.ClickHouse.ReaderDatabase.Name, - // Host: cfg.ClickHouse.ReaderDatabase.Host, - // Port: cfg.ClickHouse.ReaderDatabase.Port, - // MaxOpenConns: cfg.ClickHouse.ReaderDatabase.MaxOpenConns, - // SSL: true, - // MaxIdleConns: cfg.ClickHouse.ReaderDatabase.MaxIdleConns, - // }, "clickhouse", "clickhouse") - // defer db.ClickHouseReader.Close() - // defer db.ClickHouseWriter.Close() + db.ClickHouseWriter, db.ClickHouseReader = db.MustInitDB(&types.DatabaseConfig{ + Username: cfg.ClickHouse.WriterDatabase.Username, + Password: cfg.ClickHouse.WriterDatabase.Password, + Name: cfg.ClickHouse.WriterDatabase.Name, + Host: cfg.ClickHouse.WriterDatabase.Host, + Port: cfg.ClickHouse.WriterDatabase.Port, + MaxOpenConns: cfg.ClickHouse.WriterDatabase.MaxOpenConns, + SSL: true, + MaxIdleConns: cfg.ClickHouse.WriterDatabase.MaxIdleConns, + }, &types.DatabaseConfig{ + Username: cfg.ClickHouse.ReaderDatabase.Username, + Password: cfg.ClickHouse.ReaderDatabase.Password, + Name: cfg.ClickHouse.ReaderDatabase.Name, + Host: cfg.ClickHouse.ReaderDatabase.Host, + Port: cfg.ClickHouse.ReaderDatabase.Port, + MaxOpenConns: cfg.ClickHouse.ReaderDatabase.MaxOpenConns, + SSL: true, + MaxIdleConns: cfg.ClickHouse.ReaderDatabase.MaxIdleConns, + }, "clickhouse", "clickhouse") + defer db.ClickHouseReader.Close() + defer db.ClickHouseWriter.Close() // Initialize the persistent redis client if requires.Redis { diff --git a/backend/pkg/api/data_access/notifications.go b/backend/pkg/api/data_access/notifications.go index f37948ac0..21f3e0ebb 100644 --- a/backend/pkg/api/data_access/notifications.go +++ b/backend/pkg/api/data_access/notifications.go @@ -63,6 +63,7 @@ func (*DataAccessService) registerNotificationInterfaceTypes() { once.Do(func() { gob.Register(&n.ValidatorProposalNotification{}) gob.Register(&n.ValidatorUpcomingProposalNotification{}) + gob.Register(&n.ValidatorGroupEfficiencyNotification{}) gob.Register(&n.ValidatorAttestationNotification{}) gob.Register(&n.ValidatorIsOfflineNotification{}) gob.Register(&n.ValidatorIsOnlineNotification{}) diff --git a/backend/pkg/notification/collection.go b/backend/pkg/notification/collection.go index d5f4b3b8a..3093e9bcb 100644 --- a/backend/pkg/notification/collection.go +++ b/backend/pkg/notification/collection.go @@ -12,7 +12,6 @@ import ( "time" gcp_bigtable "cloud.google.com/go/bigtable" - "github.com/doug-martin/goqu/v9" "github.com/ethereum/go-ethereum/common" "github.com/gobitfly/beaconchain/pkg/commons/cache" "github.com/gobitfly/beaconchain/pkg/commons/db" @@ -22,6 +21,7 @@ import ( "github.com/gobitfly/beaconchain/pkg/commons/services" "github.com/gobitfly/beaconchain/pkg/commons/types" "github.com/gobitfly/beaconchain/pkg/commons/utils" + cTypes "github.com/gobitfly/beaconchain/pkg/consapi/types" "github.com/gobitfly/beaconchain/pkg/exporter/modules" "github.com/lib/pq" "github.com/rocket-pool/rocketpool-go/utils/eth" @@ -49,6 +49,7 @@ func notificationCollector() { once.Do(func() { gob.Register(&ValidatorProposalNotification{}) gob.Register(&ValidatorUpcomingProposalNotification{}) + gob.Register(&ValidatorGroupEfficiencyNotification{}) gob.Register(&ValidatorAttestationNotification{}) gob.Register(&ValidatorIsOfflineNotification{}) gob.Register(&ValidatorIsOnlineNotification{}) @@ -62,13 +63,13 @@ func notificationCollector() { gob.Register(&SyncCommitteeSoonNotification{}) }) + mc, err := modules.GetModuleContext() + if err != nil { + log.Fatal(err, "error getting module context", 0) + } + go func() { log.Infof("starting head notification collector") - mc, err := modules.GetModuleContext() - if err != nil { - log.Fatal(err, "error getting module context", 0) - } - for ; ; time.Sleep(time.Second * 30) { // get the head epoch head, err := mc.ConsClient.GetChainHead() @@ -160,7 +161,7 @@ func notificationCollector() { log.Infof("collecting notifications for epoch %v", epoch) // Network DB Notifications (network related) - notifications, err := collectNotifications(epoch) + notifications, err := collectNotifications(epoch, mc) if err != nil { log.Error(err, "error collection notifications", 0) @@ -279,7 +280,7 @@ func collectUpcomingBlockProposalNotifications(notificationsByUserID types.Notif return nil } -func collectNotifications(epoch uint64) (types.NotificationsPerUserId, error) { +func collectNotifications(epoch uint64, mc modules.ModuleContext) (types.NotificationsPerUserId, error) { notificationsByUserID := types.NotificationsPerUserId{} start := time.Now() var err error @@ -310,12 +311,12 @@ func collectNotifications(epoch uint64) (types.NotificationsPerUserId, error) { // The following functions will collect the notifications and add them to the // notificationsByUserID map. The notifications will be queued and sent later // by the notification sender process - err = collectGroupEfficiencyNotifications(notificationsByUserID, epoch) + err = collectGroupEfficiencyNotifications(notificationsByUserID, epoch, mc) if err != nil { metrics.Errors.WithLabelValues("notifications_collect_group_efficiency").Inc() return nil, fmt.Errorf("error collecting validator_group_efficiency notifications: %v", err) } - log.Infof("collecting attestation & offline notifications took: %v", time.Since(start)) + log.Infof("collecting group efficiency notifications took: %v", time.Since(start)) err = collectAttestationAndOfflineValidatorNotifications(notificationsByUserID, epoch) if err != nil { @@ -469,7 +470,7 @@ func collectUserDbNotifications(epoch uint64) (types.NotificationsPerUserId, err return notificationsByUserID, nil } -func collectGroupEfficiencyNotifications(notificationsByUserID types.NotificationsPerUserId, epoch uint64) error { +func collectGroupEfficiencyNotifications(notificationsByUserID types.NotificationsPerUserId, epoch uint64, mc modules.ModuleContext) error { type dbResult struct { ValidatorIndex uint64 `db:"validator_index"` AttestationReward decimal.Decimal `db:"attestations_reward"` @@ -480,32 +481,67 @@ func collectGroupEfficiencyNotifications(notificationsByUserID types.Notificatio SyncScheduled uint64 `db:"sync_scheduled"` } - var queryResult []*dbResult - clickhouseTable := "validator_dashboard_data_epoch" - // retrieve efficiency data for the epoch - ds := goqu.Dialect("postgres"). - From(goqu.L(fmt.Sprintf(`%s AS r FINAL`, clickhouseTable))). - Select( - goqu.L("validator_index"), - goqu.L("COALESCE(r.attestations_reward, 0) AS attestations_reward"), - goqu.L("COALESCE(r.attestations_ideal_reward, 0) AS attestations_ideal_reward"), - goqu.L("COALESCE(r.blocks_proposed, 0) AS blocks_proposed"), - goqu.L("COALESCE(r.blocks_scheduled, 0) AS blocks_scheduled"), - goqu.L("COALESCE(r.sync_executed, 0) AS sync_executed"), - goqu.L("COALESCE(r.sync_scheduled, 0) AS sync_scheduled")). - Where(goqu.L("r.epoch = ?", epoch)) - query, args, err := ds.Prepared(true).ToSQL() + // retrieve rewards for the epoch + log.Info("retrieving validator metadata") + validators, err := mc.CL.GetValidators(epoch*utils.Config.Chain.ClConfig.SlotsPerEpoch, nil, []cTypes.ValidatorStatus{cTypes.Active}) + if err != nil { + return fmt.Errorf("error getting validators: %w", err) + } + effectiveBalanceMap := make(map[uint64]uint64) + activeValidatorsMap := make(map[uint64]struct{}) + for _, validator := range validators.Data { + effectiveBalanceMap[validator.Index] = validator.Validator.EffectiveBalance + activeValidatorsMap[validator.Index] = struct{}{} + } + log.Info("retrieving attestation reward data") + attestationRewards, err := mc.CL.GetAttestationRewards(epoch) if err != nil { - return fmt.Errorf("error preparing query: %v", err) + return fmt.Errorf("error getting attestation rewards: %w", err) + } + + efficiencyMap := make(map[types.ValidatorIndex]*dbResult, len(attestationRewards.Data.TotalRewards)) + + idealRewardsMap := make(map[uint64]decimal.Decimal) + for _, reward := range attestationRewards.Data.IdealRewards { + idealRewardsMap[uint64(reward.EffectiveBalance)] = decimal.NewFromInt(int64(reward.Head) + int64(reward.Target) + int64(reward.Source) + int64(reward.InclusionDelay) + int64(reward.Inactivity)) + } + for _, reward := range attestationRewards.Data.TotalRewards { + efficiencyMap[types.ValidatorIndex(reward.ValidatorIndex)] = &dbResult{ + ValidatorIndex: reward.ValidatorIndex, + AttestationReward: decimal.NewFromInt(int64(reward.Head) + int64(reward.Target) + int64(reward.Source) + int64(reward.InclusionDelay) + int64(reward.Inactivity)), + AttestationIdealReward: idealRewardsMap[effectiveBalanceMap[reward.ValidatorIndex]], + } } - err = db.ClickHouseReader.Select(&queryResult, query, args...) + log.Info("retrieving block proposal data") + proposalAssignments, err := mc.CL.GetPropoalAssignments(epoch) if err != nil { - return fmt.Errorf("error retrieving data from table %s: %v", clickhouseTable, err) + return fmt.Errorf("error getting proposal assignments: %w", err) + } + for _, assignment := range proposalAssignments.Data { + efficiencyMap[types.ValidatorIndex(assignment.ValidatorIndex)].BlocksScheduled++ } - if len(queryResult) == 0 { - return fmt.Errorf("no efficiency data found for epoch %v", epoch) + for slot := epoch * utils.Config.Chain.ClConfig.SlotsPerEpoch; slot < (epoch+1)*utils.Config.Chain.ClConfig.SlotsPerEpoch; slot++ { + header, err := mc.CL.GetBlockHeader(slot) + log.Infof("retrieving data for slot %v", slot) + if err != nil && strings.Contains(err.Error(), "NOT_FOUND") { + continue + } else if err != nil { + return fmt.Errorf("error getting block header for slot %v: %w", slot, err) + } + efficiencyMap[types.ValidatorIndex(header.Data.Header.Message.ProposerIndex)].BlocksProposed++ + + syncRewards, err := mc.CL.GetSyncRewards(slot) + if err != nil { + return fmt.Errorf("error getting sync rewards for slot %v: %w", slot, err) + } + for _, reward := range syncRewards.Data { + efficiencyMap[types.ValidatorIndex(reward.ValidatorIndex)].SyncScheduled++ + if reward.Reward > 0 { + efficiencyMap[types.ValidatorIndex(reward.ValidatorIndex)].SyncExecuted++ + } + } } subMap, err := GetSubsForEventFilter(types.ValidatorGroupEfficiencyEventName, "", nil, nil) @@ -554,10 +590,72 @@ func collectGroupEfficiencyNotifications(notificationsByUserID types.Notificatio } } - efficiencyMap := make(map[types.ValidatorIndex]*dbResult) - for _, row := range queryResult { - efficiencyMap[types.ValidatorIndex(row.ValidatorIndex)] = row - } + // The commented code below can be used to validate data retrieved from the node against + // data in clickhouse + // var queryResult []*dbResult + // clickhouseTable := "validator_dashboard_data_epoch" + // // retrieve efficiency data for the epoch + // log.Infof("retrieving efficiency data for epoch %v", epoch) + // ds := goqu.Dialect("postgres"). + // From(goqu.L(fmt.Sprintf(`%s AS r`, clickhouseTable))). + // Select( + // goqu.L("validator_index"), + // goqu.L("COALESCE(r.attestations_reward, 0) AS attestations_reward"), + // goqu.L("COALESCE(r.attestations_ideal_reward, 0) AS attestations_ideal_reward"), + // goqu.L("COALESCE(r.blocks_proposed, 0) AS blocks_proposed"), + // goqu.L("COALESCE(r.blocks_scheduled, 0) AS blocks_scheduled"), + // goqu.L("COALESCE(r.sync_executed, 0) AS sync_executed"), + // goqu.L("COALESCE(r.sync_scheduled, 0) AS sync_scheduled")). + // Where(goqu.L("r.epoch_timestamp = ?", utils.EpochToTime(epoch))) + // query, args, err := ds.Prepared(true).ToSQL() + // if err != nil { + // return fmt.Errorf("error preparing query: %v", err) + // } + + // err = db.ClickHouseReader.Select(&queryResult, query, args...) + // if err != nil { + // return fmt.Errorf("error retrieving data from table %s: %v", clickhouseTable, err) + // } + + // if len(queryResult) == 0 { + // return fmt.Errorf("no efficiency data found for epoch %v", epoch) + // } + + // log.Infof("retrieved %v efficiency data rows", len(queryResult)) + + // for _, row := range queryResult { + // if _, ok := activeValidatorsMap[row.ValidatorIndex]; !ok { + // continue + // } + // existing := efficiencyMap[types.ValidatorIndex(row.ValidatorIndex)] + + // if existing == nil { + // existing = &dbResult{ + // ValidatorIndex: row.ValidatorIndex, + // AttestationReward: decimal.Decimal{}, + // AttestationIdealReward: decimal.Decimal{}, + // } + // } + // if !existing.AttestationIdealReward.Equal(row.AttestationIdealReward) { + // log.Fatal(fmt.Errorf("ideal reward mismatch for validator %v: %v != %v", row.ValidatorIndex, existing.AttestationIdealReward, row.AttestationIdealReward), "ideal reward mismatch", 0) + // } + // if !existing.AttestationReward.Equal(row.AttestationReward) { + // log.Fatal(fmt.Errorf("attestation reward mismatch for validator %v: %v != %v", row.ValidatorIndex, existing.AttestationReward, row.AttestationReward), "attestation reward mismatch", 0) + // } + // if existing.BlocksProposed != row.BlocksProposed { + // log.Fatal(fmt.Errorf("blocks proposed mismatch for validator %v: %v != %v", row.ValidatorIndex, existing.BlocksProposed, row.BlocksProposed), "blocks proposed mismatch", 0) + // } + // if existing.BlocksScheduled != row.BlocksScheduled { + // log.Fatal(fmt.Errorf("blocks scheduled mismatch for validator %v: %v != %v", row.ValidatorIndex, existing.BlocksScheduled, row.BlocksScheduled), "blocks scheduled mismatch", 0) + // } + // if existing.SyncExecuted != row.SyncExecuted { + // log.Fatal(fmt.Errorf("sync executed mismatch for validator %v: %v != %v", row.ValidatorIndex, existing.SyncExecuted, row.SyncExecuted), "sync executed mismatch", 0) + // } + // if existing.SyncScheduled != row.SyncScheduled { + // log.Fatal(fmt.Errorf("sync scheduled mismatch for validator %v: %v != %v", row.ValidatorIndex, existing.SyncScheduled, row.SyncScheduled), "sync scheduled mismatch", 0) + // } + // efficiencyMap[types.ValidatorIndex(row.ValidatorIndex)] = row + // } for userId, dashboards := range dashboardMap { for dashboardId, groups := range dashboards { @@ -597,7 +695,9 @@ func collectGroupEfficiencyNotifications(notificationsByUserID types.Notificatio efficiency := utils.CalculateTotalEfficiency(attestationEfficiency, proposerEfficiency, syncEfficiency) - if efficiency < groupDetails.Subscription.EventThreshold { + log.Infof("efficiency: %v, threshold: %v", efficiency, groupDetails.Subscription.EventThreshold*100) + + if efficiency < groupDetails.Subscription.EventThreshold*100 { log.Infof("creating group efficiency notification for user %v, dashboard %v, group %v in epoch %v", userId, dashboardId, groupId, epoch) n := &ValidatorGroupEfficiencyNotification{ NotificationBaseImpl: types.NotificationBaseImpl{ @@ -611,7 +711,7 @@ func collectGroupEfficiencyNotifications(notificationsByUserID types.Notificatio DashboardGroupId: groupDetails.Subscription.DashboardGroupId, DashboardGroupName: groupDetails.Subscription.DashboardGroupName, }, - Threshold: groupDetails.Subscription.EventThreshold, + Threshold: groupDetails.Subscription.EventThreshold * 100, Efficiency: efficiency, } notificationsByUserID.AddNotification(n) @@ -621,6 +721,8 @@ func collectGroupEfficiencyNotifications(notificationsByUserID types.Notificatio } } + log.Info("done collecting group efficiency notifications") + return nil } func collectBlockProposalNotifications(notificationsByUserID types.NotificationsPerUserId, status uint64, eventName types.EventName, epoch uint64) error { diff --git a/backend/pkg/notification/notifications.go b/backend/pkg/notification/notifications.go index 084bdf4f2..7bb71162a 100644 --- a/backend/pkg/notification/notifications.go +++ b/backend/pkg/notification/notifications.go @@ -3,15 +3,21 @@ package notification import ( "github.com/gobitfly/beaconchain/pkg/commons/log" "github.com/gobitfly/beaconchain/pkg/commons/types" + "github.com/gobitfly/beaconchain/pkg/exporter/modules" ) // Used for isolated testing func GetNotificationsForEpoch(pubkeyCachePath string, epoch uint64) (types.NotificationsPerUserId, error) { - err := initPubkeyCache(pubkeyCachePath) + mc, err := modules.GetModuleContext() + if err != nil { + log.Fatal(err, "error getting module context", 0) + } + + err = initPubkeyCache(pubkeyCachePath) if err != nil { log.Fatal(err, "error initializing pubkey cache path for notifications", 0) } - return collectNotifications(epoch) + return collectNotifications(epoch, mc) } // Used for isolated testing diff --git a/backend/pkg/notification/pubkey_cache.go b/backend/pkg/notification/pubkey_cache.go index 483884cf5..37f3f5ea2 100644 --- a/backend/pkg/notification/pubkey_cache.go +++ b/backend/pkg/notification/pubkey_cache.go @@ -72,7 +72,7 @@ func GetIndexForPubkey(pubkey []byte) (uint64, error) { if err != nil { return 0, err } - log.Infof("serving index %d for validator %x from db", index, pubkey) + // log.Infof("serving index %d for validator %x from db", index, pubkey) return index, nil } else if err != nil { return 0, err diff --git a/backend/pkg/notification/types.go b/backend/pkg/notification/types.go index d7b146649..02289ecca 100644 --- a/backend/pkg/notification/types.go +++ b/backend/pkg/notification/types.go @@ -51,7 +51,7 @@ func formatSlotLink(format types.NotificationFormat, slot interface{}) string { return "" } -func formatDashboardAndGroupLink(format types.NotificationFormat, n types.Notification) string { +func formatValidatorPrefixedDashboardAndGroupLink(format types.NotificationFormat, n types.Notification) string { dashboardAndGroupInfo := "" if n.GetDashboardId() != nil { switch format { @@ -66,6 +66,21 @@ func formatDashboardAndGroupLink(format types.NotificationFormat, n types.Notifi return dashboardAndGroupInfo } +func formatPureDashboardAndGroupLink(format types.NotificationFormat, n types.Notification) string { + dashboardAndGroupInfo := "" + if n.GetDashboardId() != nil { + switch format { + case types.NotifciationFormatHtml: + dashboardAndGroupInfo = fmt.Sprintf(`Group %[2]v in Dashboard %[3]v`, utils.Config.Frontend.SiteDomain, n.GetDashboardGroupName(), n.GetDashboardName(), *n.GetDashboardId()) + case types.NotifciationFormatText: + dashboardAndGroupInfo = fmt.Sprintf(`Group %[1]v in Dashboard %[2]v`, n.GetDashboardGroupName(), n.GetDashboardName()) + case types.NotifciationFormatMarkdown: + dashboardAndGroupInfo = fmt.Sprintf(`Group **%[1]v** in Dashboard [%[2]v](https://%[3]v/dashboard/%[4]v)`, n.GetDashboardGroupName(), n.GetDashboardName(), utils.Config.Frontend.SiteDomain, *n.GetDashboardId()) + } + } + return dashboardAndGroupInfo +} + type ValidatorProposalNotification struct { types.NotificationBaseImpl @@ -83,7 +98,7 @@ func (n *ValidatorProposalNotification) GetEntitiyId() string { func (n *ValidatorProposalNotification) GetInfo(format types.NotificationFormat) string { vali := formatValidatorLink(format, n.ValidatorIndex) slot := formatSlotLink(format, n.Slot) - dashboardAndGroupInfo := formatDashboardAndGroupLink(format, n) + dashboardAndGroupInfo := formatValidatorPrefixedDashboardAndGroupLink(format, n) switch n.Status { case 0: @@ -149,7 +164,7 @@ func (n *ValidatorUpcomingProposalNotification) GetEntitiyId() string { func (n *ValidatorUpcomingProposalNotification) GetInfo(format types.NotificationFormat) string { vali := formatValidatorLink(format, n.ValidatorIndex) slot := formatSlotLink(format, n.Slot) - dashboardAndGroupInfo := formatDashboardAndGroupLink(format, n) + dashboardAndGroupInfo := formatValidatorPrefixedDashboardAndGroupLink(format, n) return fmt.Sprintf(`New scheduled block proposal at slot %s for Validator %s%s.`, slot, vali, dashboardAndGroupInfo) } @@ -184,7 +199,7 @@ func (n *ValidatorIsOfflineNotification) GetEntitiyId() string { func (n *ValidatorIsOfflineNotification) GetInfo(format types.NotificationFormat) string { vali := formatValidatorLink(format, n.ValidatorIndex) epoch := formatEpochLink(format, n.LatestState) - dashboardAndGroupInfo := formatDashboardAndGroupLink(format, n) + dashboardAndGroupInfo := formatValidatorPrefixedDashboardAndGroupLink(format, n) return fmt.Sprintf(`Validator %v%v is offline since epoch %s.`, vali, dashboardAndGroupInfo, epoch) } @@ -214,7 +229,7 @@ func (n *ValidatorIsOnlineNotification) GetEntitiyId() string { func (n *ValidatorIsOnlineNotification) GetInfo(format types.NotificationFormat) string { vali := formatValidatorLink(format, n.ValidatorIndex) epoch := formatEpochLink(format, n.Epoch) - dashboardAndGroupInfo := formatDashboardAndGroupLink(format, n) + dashboardAndGroupInfo := formatValidatorPrefixedDashboardAndGroupLink(format, n) return fmt.Sprintf(`Validator %v%v is back online since epoch %v.`, vali, dashboardAndGroupInfo, epoch) } @@ -243,9 +258,9 @@ func (n *ValidatorGroupEfficiencyNotification) GetEntitiyId() string { // Overwrite specific methods func (n *ValidatorGroupEfficiencyNotification) GetInfo(format types.NotificationFormat) string { - dashboardAndGroupInfo := formatDashboardAndGroupLink(format, n) + dashboardAndGroupInfo := formatPureDashboardAndGroupLink(format, n) epoch := formatEpochLink(format, n.Epoch) - return fmt.Sprintf(`%s%s efficiency of %.2f is below the threhold of %.2f in epoch %s.`, dashboardAndGroupInfo, n.Efficiency, n.Threshold, epoch) + return fmt.Sprintf(`%s efficiency of %.2f%% is below the threhold of %.2f%% in epoch %s.`, dashboardAndGroupInfo, n.Efficiency, n.Threshold, epoch) } func (n *ValidatorGroupEfficiencyNotification) GetTitle() string { @@ -273,7 +288,7 @@ func (n *ValidatorAttestationNotification) GetEntitiyId() string { } func (n *ValidatorAttestationNotification) GetInfo(format types.NotificationFormat) string { - dashboardAndGroupInfo := formatDashboardAndGroupLink(format, n) + dashboardAndGroupInfo := formatValidatorPrefixedDashboardAndGroupLink(format, n) vali := formatValidatorLink(format, n.ValidatorIndex) epoch := formatEpochLink(format, n.Epoch) @@ -347,7 +362,7 @@ func (n *ValidatorGotSlashedNotification) GetEntitiyId() string { } func (n *ValidatorGotSlashedNotification) GetInfo(format types.NotificationFormat) string { - dashboardAndGroupInfo := formatDashboardAndGroupLink(format, n) + dashboardAndGroupInfo := formatValidatorPrefixedDashboardAndGroupLink(format, n) vali := formatValidatorLink(format, n.ValidatorIndex) epoch := formatEpochLink(format, n.Epoch) slasher := formatValidatorLink(format, n.Slasher) @@ -383,7 +398,7 @@ func (n *ValidatorWithdrawalNotification) GetEntitiyId() string { } func (n *ValidatorWithdrawalNotification) GetInfo(format types.NotificationFormat) string { - dashboardAndGroupInfo := formatDashboardAndGroupLink(format, n) + dashboardAndGroupInfo := formatValidatorPrefixedDashboardAndGroupLink(format, n) vali := formatValidatorLink(format, n.ValidatorIndex) amount := utils.FormatClCurrencyString(n.Amount, utils.Config.Frontend.MainCurrency, 6, true, false, false) generalPart := fmt.Sprintf(`An automatic withdrawal of %s has been processed for validator %s%s.`, amount, vali, dashboardAndGroupInfo) From 2d23a28dcb26e275a84741c33452480ee83f32e5 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Tue, 22 Oct 2024 14:12:32 +0000 Subject: [PATCH 094/124] fix(notifications): correct title for efficiency notifications --- backend/pkg/notification/queuing.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/backend/pkg/notification/queuing.go b/backend/pkg/notification/queuing.go index 90cb78bca..803a618c8 100644 --- a/backend/pkg/notification/queuing.go +++ b/backend/pkg/notification/queuing.go @@ -380,6 +380,9 @@ func RenderEmailsForUserEvents(epoch uint64, notificationsByUserID types.Notific case types.ValidatorExecutedProposalEventName: //nolint:gosec // this is a static string bodySummary += template.HTML(fmt.Sprintf("%s: %d validator%s, Reward: %.3f ETH", types.EventLabel[event], count, plural, totalBlockReward)) + case types.ValidatorGroupEfficiencyEventName: + //nolint:gosec // this is a static string + bodySummary += template.HTML(fmt.Sprintf("%s: %d Group%s", types.EventLabel[event], count, plural)) default: //nolint:gosec // this is a static string bodySummary += template.HTML(fmt.Sprintf("%s: %d Validator%s", types.EventLabel[event], count, plural)) @@ -513,6 +516,8 @@ func RenderPushMessagesForUserEvents(epoch uint64, notificationsByUserID types.N bodySummary += fmt.Sprintf("%s: %d machine%s", types.EventLabel[event], count, plural) case types.ValidatorExecutedProposalEventName: bodySummary += fmt.Sprintf("%s: %d validator%s, Reward: %.3f ETH", types.EventLabel[event], count, plural, totalBlockReward) + case types.ValidatorGroupEfficiencyEventName: + bodySummary += fmt.Sprintf("%s: %d group%s", types.EventLabel[event], count, plural) default: bodySummary += fmt.Sprintf("%s: %d validator%s", types.EventLabel[event], count, plural) } From e711eda731d393612f32cf3d8f41b72c757224d2 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Tue, 22 Oct 2024 14:28:12 +0000 Subject: [PATCH 095/124] fix(notifications): improve efficiency message --- backend/pkg/notification/types.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/pkg/notification/types.go b/backend/pkg/notification/types.go index 02289ecca..754955cda 100644 --- a/backend/pkg/notification/types.go +++ b/backend/pkg/notification/types.go @@ -260,7 +260,7 @@ func (n *ValidatorGroupEfficiencyNotification) GetEntitiyId() string { func (n *ValidatorGroupEfficiencyNotification) GetInfo(format types.NotificationFormat) string { dashboardAndGroupInfo := formatPureDashboardAndGroupLink(format, n) epoch := formatEpochLink(format, n.Epoch) - return fmt.Sprintf(`%s efficiency of %.2f%% is below the threhold of %.2f%% in epoch %s.`, dashboardAndGroupInfo, n.Efficiency, n.Threshold, epoch) + return fmt.Sprintf(`%s efficiency of %.2f%% was below the threhold of %.2f%% in epoch %s.`, dashboardAndGroupInfo, n.Efficiency, n.Threshold, epoch) } func (n *ValidatorGroupEfficiencyNotification) GetTitle() string { From 9e091f248598dedcbefac4216a81d33ad4d0e6a5 Mon Sep 17 00:00:00 2001 From: benji-bitfly Date: Tue, 22 Oct 2024 16:11:29 +0200 Subject: [PATCH 096/124] fix(NotificationMachinesTable): add `machines` data key - Pass the `data-key` prop to ensure proper handling in TreeTable. This is required by the `ui library`. See: BEDS-256 --- frontend/.vscode/settings.json | 1 + .../components/notifications/NotificationsMachinesTable.vue | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/frontend/.vscode/settings.json b/frontend/.vscode/settings.json index 6c8cfe14e..8b64c0da4 100644 --- a/frontend/.vscode/settings.json +++ b/frontend/.vscode/settings.json @@ -7,6 +7,7 @@ "DashboardChartSummaryChartFilter", "DashboardGroupManagementModal", "DashboardValidatorManagmentModal", + "NotificationMachinesTable", "NotificationsClientsTable", "NotificationsDashboardDialogEntity", "NotificationsDashboardTable", diff --git a/frontend/components/notifications/NotificationsMachinesTable.vue b/frontend/components/notifications/NotificationsMachinesTable.vue index 508aa086a..d58c6dfed 100644 --- a/frontend/components/notifications/NotificationsMachinesTable.vue +++ b/frontend/components/notifications/NotificationsMachinesTable.vue @@ -45,7 +45,7 @@ const machineEvent = (eventType: NotificationMachinesTableRow['event_type']) => Date: Tue, 22 Oct 2024 14:46:16 +0000 Subject: [PATCH 097/124] feat(notifications): speed up efficiency notifications --- backend/pkg/notification/collection.go | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/backend/pkg/notification/collection.go b/backend/pkg/notification/collection.go index 3093e9bcb..ba6b0568f 100644 --- a/backend/pkg/notification/collection.go +++ b/backend/pkg/notification/collection.go @@ -21,7 +21,7 @@ import ( "github.com/gobitfly/beaconchain/pkg/commons/services" "github.com/gobitfly/beaconchain/pkg/commons/types" "github.com/gobitfly/beaconchain/pkg/commons/utils" - cTypes "github.com/gobitfly/beaconchain/pkg/consapi/types" + constypes "github.com/gobitfly/beaconchain/pkg/consapi/types" "github.com/gobitfly/beaconchain/pkg/exporter/modules" "github.com/lib/pq" "github.com/rocket-pool/rocketpool-go/utils/eth" @@ -483,7 +483,7 @@ func collectGroupEfficiencyNotifications(notificationsByUserID types.Notificatio // retrieve rewards for the epoch log.Info("retrieving validator metadata") - validators, err := mc.CL.GetValidators(epoch*utils.Config.Chain.ClConfig.SlotsPerEpoch, nil, []cTypes.ValidatorStatus{cTypes.Active}) + validators, err := mc.CL.GetValidators(epoch*utils.Config.Chain.ClConfig.SlotsPerEpoch, nil, []constypes.ValidatorStatus{constypes.Active}) if err != nil { return fmt.Errorf("error getting validators: %w", err) } @@ -522,24 +522,26 @@ func collectGroupEfficiencyNotifications(notificationsByUserID types.Notificatio efficiencyMap[types.ValidatorIndex(assignment.ValidatorIndex)].BlocksScheduled++ } + syncAssignments, err := mc.CL.GetSyncCommitteesAssignments(nil, epoch*utils.Config.Chain.ClConfig.SlotsPerEpoch) + if err != nil { + return fmt.Errorf("error getting sync committee assignments: %w", err) + } + for slot := epoch * utils.Config.Chain.ClConfig.SlotsPerEpoch; slot < (epoch+1)*utils.Config.Chain.ClConfig.SlotsPerEpoch; slot++ { - header, err := mc.CL.GetBlockHeader(slot) log.Infof("retrieving data for slot %v", slot) + s, err := mc.CL.GetSlot(slot) if err != nil && strings.Contains(err.Error(), "NOT_FOUND") { continue } else if err != nil { return fmt.Errorf("error getting block header for slot %v: %w", slot, err) } - efficiencyMap[types.ValidatorIndex(header.Data.Header.Message.ProposerIndex)].BlocksProposed++ + efficiencyMap[types.ValidatorIndex(s.Data.Message.ProposerIndex)].BlocksProposed++ - syncRewards, err := mc.CL.GetSyncRewards(slot) - if err != nil { - return fmt.Errorf("error getting sync rewards for slot %v: %w", slot, err) - } - for _, reward := range syncRewards.Data { - efficiencyMap[types.ValidatorIndex(reward.ValidatorIndex)].SyncScheduled++ - if reward.Reward > 0 { - efficiencyMap[types.ValidatorIndex(reward.ValidatorIndex)].SyncExecuted++ + for i, validatorIndex := range syncAssignments.Data.Validators { + efficiencyMap[types.ValidatorIndex(validatorIndex)].SyncScheduled++ + + if utils.BitAtVector(s.Data.Message.Body.SyncAggregate.SyncCommitteeBits, i) { + efficiencyMap[types.ValidatorIndex(validatorIndex)].SyncExecuted++ } } } From df411a9716c2f8b2dd86a5a912e6af072470a3a3 Mon Sep 17 00:00:00 2001 From: marcel-bitfly <174338434+marcel-bitfly@users.noreply.github.com> Date: Wed, 23 Oct 2024 09:39:46 +0200 Subject: [PATCH 098/124] fix(NotificationsManagementSubscriptionDialog): only show `premium gem` for users that do not have `group efficiency perk` --- .../management/NotificationsManagementSubscriptionDialog.vue | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/frontend/components/notifications/management/NotificationsManagementSubscriptionDialog.vue b/frontend/components/notifications/management/NotificationsManagementSubscriptionDialog.vue index 2cebb69c7..7ad997313 100644 --- a/frontend/components/notifications/management/NotificationsManagementSubscriptionDialog.vue +++ b/frontend/components/notifications/management/NotificationsManagementSubscriptionDialog.vue @@ -127,7 +127,7 @@ watch(hasAllEvents, () => { has-unit :info="$t('notifications.subscriptions.validators.group_efficiency.info', { percentage: thresholds.group_efficiency_below_threshold })" :label="$t('notifications.subscriptions.validators.group_efficiency.label')" - :has-premium-gem="hasPremiumPerkGroupEfficiency" + :has-premium-gem="!hasPremiumPerkGroupEfficiency" /> Date: Wed, 23 Oct 2024 10:48:57 +0200 Subject: [PATCH 099/124] feat: implement proper test notification handling See: BEDS-94 --- backend/pkg/api/auth.go | 1 - backend/pkg/api/data_access/dummy.go | 11 ++++++- backend/pkg/api/data_access/notifications.go | 17 ++++++++++ backend/pkg/api/handlers/public.go | 33 ++++++++++++++++++-- 4 files changed, 57 insertions(+), 5 deletions(-) diff --git a/backend/pkg/api/auth.go b/backend/pkg/api/auth.go index 698c15eeb..13c4d860d 100644 --- a/backend/pkg/api/auth.go +++ b/backend/pkg/api/auth.go @@ -17,7 +17,6 @@ var day time.Duration = time.Hour * 24 var sessionDuration time.Duration = day * 365 func newSessionManager(cfg *types.Config) *scs.SessionManager { - // TODO: replace redis with user db down the line (or replace sessions with oauth2) pool := &redis.Pool{ MaxIdle: 10, Dial: func() (redis.Conn, error) { diff --git a/backend/pkg/api/data_access/dummy.go b/backend/pkg/api/data_access/dummy.go index 256dd0b22..0ec429593 100644 --- a/backend/pkg/api/data_access/dummy.go +++ b/backend/pkg/api/data_access/dummy.go @@ -54,7 +54,6 @@ func randomEthDecimal() decimal.Decimal { // must pass a pointer to the data func commonFakeData(a interface{}) error { - // TODO fake decimal.Decimal return faker.FakeData(a, options.WithRandomMapAndSliceMaxSize(5), options.WithRandomFloatBoundaries(interfaces.RandomFloatBoundary{Start: 0, End: 1})) } @@ -780,3 +779,13 @@ func (d *DummyService) PostUserMachineMetrics(ctx context.Context, userID uint64 func (d *DummyService) GetValidatorDashboardMobileValidators(ctx context.Context, dashboardId t.VDBId, period enums.TimePeriod, cursor string, colSort t.Sort[enums.VDBMobileValidatorsColumn], search string, limit uint64) ([]t.MobileValidatorDashboardValidatorsTableRow, *t.Paging, error) { return getDummyWithPaging[t.MobileValidatorDashboardValidatorsTableRow]() } + +func (d *DummyService) QueueTestEmailNotification(ctx context.Context, userId uint64) error { + return nil +} +func (d *DummyService) QueueTestPushNotification(ctx context.Context, userId uint64) error { + return nil +} +func (d *DummyService) QueueTestWebhookNotification(ctx context.Context, userId uint64, webhookUrl string, isDiscordWebhook bool) error { + return nil +} diff --git a/backend/pkg/api/data_access/notifications.go b/backend/pkg/api/data_access/notifications.go index 74f777c3b..c20f191ec 100644 --- a/backend/pkg/api/data_access/notifications.go +++ b/backend/pkg/api/data_access/notifications.go @@ -56,6 +56,10 @@ type NotificationsRepository interface { GetNotificationSettingsDashboards(ctx context.Context, userId uint64, cursor string, colSort t.Sort[enums.NotificationSettingsDashboardColumn], search string, limit uint64) ([]t.NotificationSettingsDashboardsTableRow, *t.Paging, error) UpdateNotificationSettingsValidatorDashboard(ctx context.Context, userId uint64, dashboardId t.VDBIdPrimary, groupId uint64, settings t.NotificationSettingsValidatorDashboard) error UpdateNotificationSettingsAccountDashboard(ctx context.Context, userId uint64, dashboardId t.VDBIdPrimary, groupId uint64, settings t.NotificationSettingsAccountDashboard) error + + QueueTestEmailNotification(ctx context.Context, userId uint64) error + QueueTestPushNotification(ctx context.Context, userId uint64) error + QueueTestWebhookNotification(ctx context.Context, userId uint64, webhookUrl string, isDiscordWebhook bool) error } func (*DataAccessService) registerNotificationInterfaceTypes() { @@ -2259,3 +2263,16 @@ func (d *DataAccessService) AddOrRemoveEvent(eventsToInsert *[]goqu.Record, even *eventsToDelete = append(*eventsToDelete, goqu.Ex{"user_id": userId, "event_name": fullEventName, "event_filter": eventFilter}) } } + +func (d *DataAccessService) QueueTestEmailNotification(ctx context.Context, userId uint64) error { + // TODO: @Data Access + return nil +} +func (d *DataAccessService) QueueTestPushNotification(ctx context.Context, userId uint64) error { + // TODO: @Data Access + return nil +} +func (d *DataAccessService) QueueTestWebhookNotification(ctx context.Context, userId uint64, webhookUrl string, isDiscordWebhook bool) error { + // TODO: @Data Access + return nil +} diff --git a/backend/pkg/api/handlers/public.go b/backend/pkg/api/handlers/public.go index 92b650d75..9c7f5d5e2 100644 --- a/backend/pkg/api/handlers/public.go +++ b/backend/pkg/api/handlers/public.go @@ -2648,7 +2648,16 @@ func (h *HandlerService) PublicPutUserNotificationSettingsAccountDashboard(w htt // @Success 204 // @Router /users/me/notifications/test-email [post] func (h *HandlerService) PublicPostUserNotificationsTestEmail(w http.ResponseWriter, r *http.Request) { - // TODO + userId, err := GetUserIdByContext(r) + if err != nil { + handleErr(w, r, err) + return + } + err = h.getDataAccessor(r).QueueTestEmailNotification(r.Context(), userId) + if err != nil { + handleErr(w, r, err) + return + } returnNoContent(w, r) } @@ -2661,7 +2670,16 @@ func (h *HandlerService) PublicPostUserNotificationsTestEmail(w http.ResponseWri // @Success 204 // @Router /users/me/notifications/test-push [post] func (h *HandlerService) PublicPostUserNotificationsTestPush(w http.ResponseWriter, r *http.Request) { - // TODO + userId, err := GetUserIdByContext(r) + if err != nil { + handleErr(w, r, err) + return + } + err = h.getDataAccessor(r).QueueTestPushNotification(r.Context(), userId) + if err != nil { + handleErr(w, r, err) + return + } returnNoContent(w, r) } @@ -2678,6 +2696,11 @@ func (h *HandlerService) PublicPostUserNotificationsTestPush(w http.ResponseWrit // @Router /users/me/notifications/test-webhook [post] func (h *HandlerService) PublicPostUserNotificationsTestWebhook(w http.ResponseWriter, r *http.Request) { var v validationError + userId, err := GetUserIdByContext(r) + if err != nil { + handleErr(w, r, err) + return + } type request struct { WebhookUrl string `json:"webhook_url"` IsDiscordWebhookEnabled bool `json:"is_discord_webhook_enabled,omitempty"` @@ -2691,7 +2714,11 @@ func (h *HandlerService) PublicPostUserNotificationsTestWebhook(w http.ResponseW handleErr(w, r, v) return } - // TODO + err = h.getDataAccessor(r).QueueTestWebhookNotification(r.Context(), userId, req.WebhookUrl, req.IsDiscordWebhookEnabled) + if err != nil { + handleErr(w, r, err) + return + } returnNoContent(w, r) } From 149f640180f3556fb1d1fe03ee1ddabc10a7f3a0 Mon Sep 17 00:00:00 2001 From: Lucca Dukic <109136188+LuccaBitfly@users.noreply.github.com> Date: Wed, 23 Oct 2024 10:52:09 +0200 Subject: [PATCH 100/124] feat: log requesting user id when logging err See: BEDS-94 --- backend/pkg/api/handlers/handler_service.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/backend/pkg/api/handlers/handler_service.go b/backend/pkg/api/handlers/handler_service.go index 4e7435351..71a43e9ad 100644 --- a/backend/pkg/api/handlers/handler_service.go +++ b/backend/pkg/api/handlers/handler_service.go @@ -315,11 +315,17 @@ func returnGone(w http.ResponseWriter, r *http.Request, err error) { const maxBodySize = 10 * 1024 func logApiError(r *http.Request, err error, callerSkip int, additionalInfos ...log.Fields) { - body, _ := io.ReadAll(io.LimitReader(r.Body, maxBodySize)) requestFields := log.Fields{ "request_endpoint": r.Method + " " + r.URL.Path, - "request_query": r.URL.RawQuery, - "request_body": string(body), + } + if len(r.URL.RawQuery) > 0 { + requestFields["request_query"] = r.URL.RawQuery + } + if body, _ := io.ReadAll(io.LimitReader(r.Body, maxBodySize)); len(body) > 0 { + requestFields["request_body"] = string(body) + } + if userId, _ := GetUserIdByContext(r); userId != 0 { + requestFields["request_user_id"] = userId } log.Error(err, "error handling request", callerSkip+1, append(additionalInfos, requestFields)...) } From 8a2bad3b00fd5b5f340bb7ec13557f230704e405 Mon Sep 17 00:00:00 2001 From: remoterami <142154971+remoterami@users.noreply.github.com> Date: Wed, 23 Oct 2024 12:54:09 +0200 Subject: [PATCH 101/124] handling invalid network name --- backend/pkg/api/data_access/notifications.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/backend/pkg/api/data_access/notifications.go b/backend/pkg/api/data_access/notifications.go index c20f191ec..e0dc9b717 100644 --- a/backend/pkg/api/data_access/notifications.go +++ b/backend/pkg/api/data_access/notifications.go @@ -1379,6 +1379,11 @@ func (d *DataAccessService) GetNotificationSettings(ctx context.Context, userId networkName := eventSplit[0] networkEvent := types.EventName(eventSplit[1]) + if _, ok := networksSettings[networkName]; !ok { + log.Error(nil, fmt.Sprintf("network is not defined: %s", networkName), 0) + continue + } + switch networkEvent { case types.RocketpoolNewClaimRoundStartedEventName: networksSettings[networkName].Settings.IsNewRewardRoundSubscribed = true From 264de3c781bcf93eb9d046ca835bf8796f67018f Mon Sep 17 00:00:00 2001 From: remoterami <142154971+remoterami@users.noreply.github.com> Date: Wed, 23 Oct 2024 13:34:41 +0200 Subject: [PATCH 102/124] returning error to api --- backend/pkg/api/data_access/notifications.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/backend/pkg/api/data_access/notifications.go b/backend/pkg/api/data_access/notifications.go index e0dc9b717..868efb171 100644 --- a/backend/pkg/api/data_access/notifications.go +++ b/backend/pkg/api/data_access/notifications.go @@ -1380,8 +1380,7 @@ func (d *DataAccessService) GetNotificationSettings(ctx context.Context, userId networkEvent := types.EventName(eventSplit[1]) if _, ok := networksSettings[networkName]; !ok { - log.Error(nil, fmt.Sprintf("network is not defined: %s", networkName), 0) - continue + return nil, fmt.Errorf("network is not defined: %s", networkName) } switch networkEvent { From 76f315a9f4f717672f149e7eb521a8f4ebbee2bc Mon Sep 17 00:00:00 2001 From: Lucca Dukic <109136188+LuccaBitfly@users.noreply.github.com> Date: Thu, 24 Oct 2024 08:02:26 +0200 Subject: [PATCH 103/124] fix: consistent naming for webhook notification parameters See: BEDS-94 --- backend/pkg/api/handlers/public.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/pkg/api/handlers/public.go b/backend/pkg/api/handlers/public.go index 9c7f5d5e2..b35b5ec5f 100644 --- a/backend/pkg/api/handlers/public.go +++ b/backend/pkg/api/handlers/public.go @@ -2703,7 +2703,7 @@ func (h *HandlerService) PublicPostUserNotificationsTestWebhook(w http.ResponseW } type request struct { WebhookUrl string `json:"webhook_url"` - IsDiscordWebhookEnabled bool `json:"is_discord_webhook_enabled,omitempty"` + IsWebhookDiscordEnabled bool `json:"is_webhook_discord_enabled,omitempty"` } var req request if err := v.checkBody(&req, r); err != nil { @@ -2714,7 +2714,7 @@ func (h *HandlerService) PublicPostUserNotificationsTestWebhook(w http.ResponseW handleErr(w, r, v) return } - err = h.getDataAccessor(r).QueueTestWebhookNotification(r.Context(), userId, req.WebhookUrl, req.IsDiscordWebhookEnabled) + err = h.getDataAccessor(r).QueueTestWebhookNotification(r.Context(), userId, req.WebhookUrl, req.IsWebhookDiscordEnabled) if err != nil { handleErr(w, r, err) return From 98044f451487c75ea3f52e9a6f7ef7cfff398515 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Thu, 24 Oct 2024 06:20:00 +0000 Subject: [PATCH 104/124] feat(notifications): implement test notifications --- backend/pkg/api/data_access/notifications.go | 10 +-- backend/pkg/commons/mail/mail.go | 10 +-- backend/pkg/notification/db.go | 5 +- backend/pkg/notification/queuing.go | 44 ++++++++++- backend/pkg/notification/sending.go | 79 +++++++++++++++++++- 5 files changed, 130 insertions(+), 18 deletions(-) diff --git a/backend/pkg/api/data_access/notifications.go b/backend/pkg/api/data_access/notifications.go index 868efb171..7ae5c2515 100644 --- a/backend/pkg/api/data_access/notifications.go +++ b/backend/pkg/api/data_access/notifications.go @@ -27,6 +27,7 @@ import ( "github.com/gobitfly/beaconchain/pkg/commons/log" "github.com/gobitfly/beaconchain/pkg/commons/types" "github.com/gobitfly/beaconchain/pkg/commons/utils" + "github.com/gobitfly/beaconchain/pkg/notification" n "github.com/gobitfly/beaconchain/pkg/notification" "github.com/lib/pq" "github.com/shopspring/decimal" @@ -2269,14 +2270,11 @@ func (d *DataAccessService) AddOrRemoveEvent(eventsToInsert *[]goqu.Record, even } func (d *DataAccessService) QueueTestEmailNotification(ctx context.Context, userId uint64) error { - // TODO: @Data Access - return nil + return notification.SendTestEmail(ctx, types.UserId(userId), d.userReader) } func (d *DataAccessService) QueueTestPushNotification(ctx context.Context, userId uint64) error { - // TODO: @Data Access - return nil + return notification.QueueTestPushNotification(ctx, types.UserId(userId), d.userReader, d.readerDb) } func (d *DataAccessService) QueueTestWebhookNotification(ctx context.Context, userId uint64, webhookUrl string, isDiscordWebhook bool) error { - // TODO: @Data Access - return nil + return notification.SendTestWebhookNotification(ctx, types.UserId(userId), webhookUrl, isDiscordWebhook) } diff --git a/backend/pkg/commons/mail/mail.go b/backend/pkg/commons/mail/mail.go index 60f27a14c..7e2491514 100644 --- a/backend/pkg/commons/mail/mail.go +++ b/backend/pkg/commons/mail/mail.go @@ -72,15 +72,9 @@ func createTextMessage(msg types.Email) string { // SendMailRateLimited sends an email to a given address with the given message. // It will return a ratelimit-error if the configured ratelimit is exceeded. -func SendMailRateLimited(content types.TransitEmailContent) error { +func SendMailRateLimited(content types.TransitEmailContent, maxEmailsPerDay int64, bucket string) error { sendThresholdReachedMail := false - maxEmailsPerDay := int64(0) - userInfo, err := db.GetUserInfo(context.Background(), uint64(content.UserId), db.FrontendReaderDB) - if err != nil { - return err - } - maxEmailsPerDay = int64(userInfo.PremiumPerks.EmailNotificationsPerDay) - count, err := db.CountSentMessage("n_mails", content.UserId) + count, err := db.CountSentMessage(bucket, content.UserId) if err != nil { return err } diff --git a/backend/pkg/notification/db.go b/backend/pkg/notification/db.go index 56c216fb5..d7ab55394 100644 --- a/backend/pkg/notification/db.go +++ b/backend/pkg/notification/db.go @@ -12,6 +12,7 @@ import ( "github.com/gobitfly/beaconchain/pkg/commons/log" "github.com/gobitfly/beaconchain/pkg/commons/types" "github.com/gobitfly/beaconchain/pkg/commons/utils" + "github.com/jmoiron/sqlx" "github.com/lib/pq" ) @@ -282,7 +283,7 @@ func GetSubsForEventFilter(eventName types.EventName, lastSentFilter string, las return subMap, nil } -func GetUserPushTokenByIds(ids []types.UserId) (map[types.UserId][]string, error) { +func GetUserPushTokenByIds(ids []types.UserId, userDbConn *sqlx.DB) (map[types.UserId][]string, error) { pushByID := map[types.UserId][]string{} if len(ids) == 0 { return pushByID, nil @@ -292,7 +293,7 @@ func GetUserPushTokenByIds(ids []types.UserId) (map[types.UserId][]string, error Token string `db:"notification_token"` } - err := db.FrontendWriterDB.Select(&rows, "SELECT DISTINCT ON (user_id, notification_token) user_id, notification_token FROM users_devices WHERE (user_id = ANY($1) AND user_id NOT IN (SELECT user_id from users_notification_channels WHERE active = false and channel = $2)) AND notify_enabled = true AND active = true AND notification_token IS NOT NULL AND LENGTH(notification_token) > 20 ORDER BY user_id, notification_token, id DESC", pq.Array(ids), types.PushNotificationChannel) + err := userDbConn.Select(&rows, "SELECT DISTINCT ON (user_id, notification_token) user_id, notification_token FROM users_devices WHERE (user_id = ANY($1) AND user_id NOT IN (SELECT user_id from users_notification_channels WHERE active = false and channel = $2)) AND notify_enabled = true AND active = true AND notification_token IS NOT NULL AND LENGTH(notification_token) > 20 ORDER BY user_id, notification_token, id DESC", pq.Array(ids), types.PushNotificationChannel) if err != nil { return nil, err } diff --git a/backend/pkg/notification/queuing.go b/backend/pkg/notification/queuing.go index 803a618c8..0a93b7846 100644 --- a/backend/pkg/notification/queuing.go +++ b/backend/pkg/notification/queuing.go @@ -3,6 +3,7 @@ package notification import ( "bytes" "compress/gzip" + "context" "database/sql" "encoding/gob" "fmt" @@ -450,7 +451,7 @@ func RenderPushMessagesForUserEvents(epoch uint64, notificationsByUserID types.N userIDs := slices.Collect(maps.Keys(notificationsByUserID)) - tokensByUserID, err := GetUserPushTokenByIds(userIDs) + tokensByUserID, err := GetUserPushTokenByIds(userIDs, db.FrontendReaderDB) if err != nil { metrics.Errors.WithLabelValues("notifications_send_push_notifications").Inc() return nil, fmt.Errorf("error when sending push-notifications: could not get tokens: %w", err) @@ -587,6 +588,47 @@ func QueuePushNotification(epoch uint64, notificationsByUserID types.Notificatio return nil } +func QueueTestPushNotification(ctx context.Context, userId types.UserId, userDbConn *sqlx.DB, networkDbConn *sqlx.DB) error { + count, err := db.CountSentMessage("n_test_push", userId) + if err != nil { + return err + } + if count > 10 { + return fmt.Errorf("rate limit has been exceeded") + } + tokens, err := GetUserPushTokenByIds([]types.UserId{userId}, userDbConn) + if err != nil { + return err + } + + messages := []*messaging.Message{} + for _, tokensOfUser := range tokens { + for _, token := range tokensOfUser { + log.Infof("sending test push to user %d with token %v", userId, token) + messages = append(messages, &messaging.Message{ + Notification: &messaging.Notification{ + Title: "Test Push", + Body: "This is a test push from beaconcha.in", + }, + Token: token, + }) + } + } + + if len(messages) == 0 { + return fmt.Errorf("no push tokens found for user %v", userId) + } + + transit := types.TransitPushContent{ + Messages: messages, + UserId: userId, + } + + _, err = networkDbConn.ExecContext(ctx, `INSERT INTO notification_queue (created, channel, content) VALUES (NOW(), 'push', $1)`, transit) + + return err +} + func QueueWebhookNotifications(notificationsByUserID types.NotificationsPerUserId, tx *sqlx.Tx) error { for userID, userNotifications := range notificationsByUserID { var webhooks []types.UserWebhook diff --git a/backend/pkg/notification/sending.go b/backend/pkg/notification/sending.go index 699f4c3f2..38e788c0c 100644 --- a/backend/pkg/notification/sending.go +++ b/backend/pkg/notification/sending.go @@ -18,6 +18,7 @@ import ( "github.com/gobitfly/beaconchain/pkg/commons/services" "github.com/gobitfly/beaconchain/pkg/commons/types" "github.com/gobitfly/beaconchain/pkg/commons/utils" + "github.com/jmoiron/sqlx" "github.com/lib/pq" ) @@ -154,7 +155,11 @@ func sendEmailNotifications() error { log.Infof("processing %v email notifications", len(notificationQueueItem)) for _, n := range notificationQueueItem { - err = mail.SendMailRateLimited(n.Content) + userInfo, err := db.GetUserInfo(context.Background(), uint64(n.Content.UserId), db.FrontendReaderDB) + if err != nil { + return err + } + err = mail.SendMailRateLimited(n.Content, int64(userInfo.PremiumPerks.EmailNotificationsPerDay), "n_emails") if err != nil { if !strings.Contains(err.Error(), "rate limit has been exceeded") { metrics.Errors.WithLabelValues("notifications_send_email").Inc() @@ -433,3 +438,75 @@ func sendDiscordNotifications() error { return nil } + +func SendTestEmail(ctx context.Context, userId types.UserId, dbConn *sqlx.DB) error { + var email string + err := dbConn.GetContext(ctx, &email, `SELECT email FROM users WHERE id = $1`, userId) + if err != nil { + return err + } + content := types.TransitEmailContent{ + UserId: userId, + Address: email, + Subject: "Test Email", + Email: types.Email{ + Title: "beaconcha.in - Test Email", + Body: "This is a test email from beaconcha.in", + }, + Attachments: []types.EmailAttachment{}, + CreatedTs: time.Now(), + } + err = mail.SendMailRateLimited(content, 10, "n_test_emails") + if err != nil { + return fmt.Errorf("error sending test email, err: %w", err) + } + + return nil +} + +func SendTestWebhookNotification(ctx context.Context, userId types.UserId, webhookUrl string, isDiscordWebhook bool) error { + count, err := db.CountSentMessage("n_test_push", userId) + if err != nil { + return err + } + if count > 10 { + return fmt.Errorf("rate limit has been exceeded") + } + + client := http.Client{Timeout: time.Second * 5} + + if isDiscordWebhook { + req := types.DiscordReq{ + Content: "This is a test notification from beaconcha.in", + } + reqBody := new(bytes.Buffer) + err := json.NewEncoder(reqBody).Encode(req) + if err != nil { + return fmt.Errorf("error marshalling discord webhook event: %w", err) + } + resp, err := client.Post(webhookUrl, "application/json", reqBody) + if err != nil { + return fmt.Errorf("error sending discord webhook request: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("error sending discord webhook request: %v", resp.Status) + } + } else { + // send a test webhook notification with the text "TEST" in the post body + reqBody := new(bytes.Buffer) + err := json.NewEncoder(reqBody).Encode(`{data: "TEST"}`) + if err != nil { + return fmt.Errorf("error marshalling webhook event: %w", err) + } + resp, err := client.Post(webhookUrl, "application/json", reqBody) + if err != nil { + return fmt.Errorf("error sending webhook request: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("error sending webhook request: %v", resp.Status) + } + } + return nil +} From 5395522261787f94a6043411368813a1b35b82e1 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Thu, 24 Oct 2024 06:24:04 +0000 Subject: [PATCH 105/124] feat(notifications): do not judge returned http code from webhook calls --- backend/pkg/notification/sending.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/backend/pkg/notification/sending.go b/backend/pkg/notification/sending.go index 38e788c0c..5d280bfae 100644 --- a/backend/pkg/notification/sending.go +++ b/backend/pkg/notification/sending.go @@ -489,9 +489,6 @@ func SendTestWebhookNotification(ctx context.Context, userId types.UserId, webho return fmt.Errorf("error sending discord webhook request: %w", err) } defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("error sending discord webhook request: %v", resp.Status) - } } else { // send a test webhook notification with the text "TEST" in the post body reqBody := new(bytes.Buffer) @@ -504,9 +501,6 @@ func SendTestWebhookNotification(ctx context.Context, userId types.UserId, webho return fmt.Errorf("error sending webhook request: %w", err) } defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - return fmt.Errorf("error sending webhook request: %v", resp.Status) - } } return nil } From f2807850433567dfc58967a379521c8c113d3326 Mon Sep 17 00:00:00 2001 From: Patrick Date: Thu, 24 Oct 2024 09:07:53 +0200 Subject: [PATCH 106/124] fix(ci): remove unneeded line in type-check (#1029) --- .github/workflows/backend-converted-types-check.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/backend-converted-types-check.yml b/.github/workflows/backend-converted-types-check.yml index a59593a02..b502e296e 100644 --- a/.github/workflows/backend-converted-types-check.yml +++ b/.github/workflows/backend-converted-types-check.yml @@ -41,7 +41,6 @@ jobs: newHash=$(find ../frontend/types/api -type f -print0 | sort -z | xargs -0 sha1sum | sha256sum | head -c 64) if [ "$currHash" != "$newHash" ]; then echo "frontend-types have changed, please commit the changes" - git diff --stat exit 1 fi From 7fd8571a9b6e8af1f0edf25eb5ce2ddaacf33ec9 Mon Sep 17 00:00:00 2001 From: marcel-bitfly <174338434+marcel-bitfly@users.noreply.github.com> Date: Thu, 24 Oct 2024 09:22:20 +0200 Subject: [PATCH 107/124] feat(BcInputUnit): set `decimal keyboard layout` on mobile --- frontend/.vscode/settings.json | 1 + frontend/components/bc/input/BcInputUnit.vue | 1 + 2 files changed, 2 insertions(+) diff --git a/frontend/.vscode/settings.json b/frontend/.vscode/settings.json index 8b64c0da4..b162c363d 100644 --- a/frontend/.vscode/settings.json +++ b/frontend/.vscode/settings.json @@ -1,6 +1,7 @@ { "conventionalCommits.scopes": [ "BcButton", + "BcInputUnit", "BcLink", "BcTablePager", "BcToggle", diff --git a/frontend/components/bc/input/BcInputUnit.vue b/frontend/components/bc/input/BcInputUnit.vue index a92b46b22..4f1f21f81 100644 --- a/frontend/components/bc/input/BcInputUnit.vue +++ b/frontend/components/bc/input/BcInputUnit.vue @@ -12,6 +12,7 @@ const input = defineModel() class="bc-input-unit__input" label-position="right" input-width="44px" + inputmode="decimal" :label="unit" type="number" v-bind="$attrs" From f6836e6d72b84f16975662cbebca006f8e909900 Mon Sep 17 00:00:00 2001 From: Lucca Dukic <109136188+LuccaBitfly@users.noreply.github.com> Date: Thu, 24 Oct 2024 10:25:59 +0200 Subject: [PATCH 108/124] refactor: pass and use context everywhere See: BEDS-868 --- backend/pkg/api/data_access/archiver.go | 2 +- backend/pkg/api/data_access/block.go | 18 +- backend/pkg/api/data_access/data_access.go | 11 +- backend/pkg/api/data_access/dummy.go | 272 +++++++++--------- backend/pkg/api/data_access/header.go | 14 +- backend/pkg/api/data_access/mobile.go | 54 ++-- backend/pkg/api/data_access/notifications.go | 2 +- backend/pkg/api/data_access/vdb_management.go | 2 +- backend/pkg/api/handlers/auth.go | 12 +- backend/pkg/api/handlers/backward_compat.go | 2 +- backend/pkg/api/handlers/handler_service.go | 2 +- backend/pkg/api/handlers/input_validation.go | 5 +- backend/pkg/api/handlers/internal.go | 7 +- backend/pkg/api/handlers/public.go | 11 +- 14 files changed, 206 insertions(+), 208 deletions(-) diff --git a/backend/pkg/api/data_access/archiver.go b/backend/pkg/api/data_access/archiver.go index 09fd605a3..5d6d80aa1 100644 --- a/backend/pkg/api/data_access/archiver.go +++ b/backend/pkg/api/data_access/archiver.go @@ -27,7 +27,7 @@ func (d *DataAccessService) GetValidatorDashboardsCountInfo(ctx context.Context) } var dbReturn []DashboardInfo - err := d.readerDb.Select(&dbReturn, ` + err := d.readerDb.SelectContext(ctx, &dbReturn, ` WITH dashboards_groups AS (SELECT dashboard_id, diff --git a/backend/pkg/api/data_access/block.go b/backend/pkg/api/data_access/block.go index 65e63b408..7410c765f 100644 --- a/backend/pkg/api/data_access/block.go +++ b/backend/pkg/api/data_access/block.go @@ -74,7 +74,7 @@ func (d *DataAccessService) GetBlockBlobs(ctx context.Context, chainId, block ui } func (d *DataAccessService) GetSlot(ctx context.Context, chainId, slot uint64) (*t.BlockSummary, error) { - block, err := d.GetBlockHeightAt(slot) + block, err := d.GetBlockHeightAt(ctx, slot) if err != nil { return nil, err } @@ -82,7 +82,7 @@ func (d *DataAccessService) GetSlot(ctx context.Context, chainId, slot uint64) ( } func (d *DataAccessService) GetSlotOverview(ctx context.Context, chainId, slot uint64) (*t.BlockOverview, error) { - block, err := d.GetBlockHeightAt(slot) + block, err := d.GetBlockHeightAt(ctx, slot) if err != nil { return nil, err } @@ -90,7 +90,7 @@ func (d *DataAccessService) GetSlotOverview(ctx context.Context, chainId, slot u } func (d *DataAccessService) GetSlotTransactions(ctx context.Context, chainId, slot uint64) ([]t.BlockTransactionTableRow, error) { - block, err := d.GetBlockHeightAt(slot) + block, err := d.GetBlockHeightAt(ctx, slot) if err != nil { return nil, err } @@ -98,7 +98,7 @@ func (d *DataAccessService) GetSlotTransactions(ctx context.Context, chainId, sl } func (d *DataAccessService) GetSlotVotes(ctx context.Context, chainId, slot uint64) ([]t.BlockVoteTableRow, error) { - block, err := d.GetBlockHeightAt(slot) + block, err := d.GetBlockHeightAt(ctx, slot) if err != nil { return nil, err } @@ -106,7 +106,7 @@ func (d *DataAccessService) GetSlotVotes(ctx context.Context, chainId, slot uint } func (d *DataAccessService) GetSlotAttestations(ctx context.Context, chainId, slot uint64) ([]t.BlockAttestationTableRow, error) { - block, err := d.GetBlockHeightAt(slot) + block, err := d.GetBlockHeightAt(ctx, slot) if err != nil { return nil, err } @@ -114,7 +114,7 @@ func (d *DataAccessService) GetSlotAttestations(ctx context.Context, chainId, sl } func (d *DataAccessService) GetSlotWithdrawals(ctx context.Context, chainId, slot uint64) ([]t.BlockWithdrawalTableRow, error) { - block, err := d.GetBlockHeightAt(slot) + block, err := d.GetBlockHeightAt(ctx, slot) if err != nil { return nil, err } @@ -122,7 +122,7 @@ func (d *DataAccessService) GetSlotWithdrawals(ctx context.Context, chainId, slo } func (d *DataAccessService) GetSlotBlsChanges(ctx context.Context, chainId, slot uint64) ([]t.BlockBlsChangeTableRow, error) { - block, err := d.GetBlockHeightAt(slot) + block, err := d.GetBlockHeightAt(ctx, slot) if err != nil { return nil, err } @@ -130,7 +130,7 @@ func (d *DataAccessService) GetSlotBlsChanges(ctx context.Context, chainId, slot } func (d *DataAccessService) GetSlotVoluntaryExits(ctx context.Context, chainId, slot uint64) ([]t.BlockVoluntaryExitTableRow, error) { - block, err := d.GetBlockHeightAt(slot) + block, err := d.GetBlockHeightAt(ctx, slot) if err != nil { return nil, err } @@ -138,7 +138,7 @@ func (d *DataAccessService) GetSlotVoluntaryExits(ctx context.Context, chainId, } func (d *DataAccessService) GetSlotBlobs(ctx context.Context, chainId, slot uint64) ([]t.BlockBlobTableRow, error) { - block, err := d.GetBlockHeightAt(slot) + block, err := d.GetBlockHeightAt(ctx, slot) if err != nil { return nil, err } diff --git a/backend/pkg/api/data_access/data_access.go b/backend/pkg/api/data_access/data_access.go index 8bac59142..4f45f3fab 100644 --- a/backend/pkg/api/data_access/data_access.go +++ b/backend/pkg/api/data_access/data_access.go @@ -35,16 +35,15 @@ type DataAccessor interface { Close() - GetLatestFinalizedEpoch() (uint64, error) - GetLatestSlot() (uint64, error) - GetLatestBlock() (uint64, error) - GetBlockHeightAt(slot uint64) (uint64, error) - GetLatestExchangeRates() ([]t.EthConversionRate, error) + GetLatestFinalizedEpoch(ctx context.Context) (uint64, error) + GetLatestSlot(ctx context.Context) (uint64, error) + GetLatestBlock(ctx context.Context) (uint64, error) + GetLatestExchangeRates(ctx context.Context) ([]t.EthConversionRate, error) GetProductSummary(ctx context.Context) (*t.ProductSummary, error) GetFreeTierPerks(ctx context.Context) (*t.PremiumPerks, error) - GetValidatorsFromSlices(indices []uint64, publicKeys []string) ([]t.VDBValidator, error) + GetValidatorsFromSlices(ctx context.Context, indices []uint64, publicKeys []string) ([]t.VDBValidator, error) } type DataAccessService struct { diff --git a/backend/pkg/api/data_access/dummy.go b/backend/pkg/api/data_access/dummy.go index 0ec429593..387ea5c0d 100644 --- a/backend/pkg/api/data_access/dummy.go +++ b/backend/pkg/api/data_access/dummy.go @@ -53,8 +53,8 @@ func randomEthDecimal() decimal.Decimal { } // must pass a pointer to the data -func commonFakeData(a interface{}) error { - return faker.FakeData(a, options.WithRandomMapAndSliceMaxSize(5), options.WithRandomFloatBoundaries(interfaces.RandomFloatBoundary{Start: 0, End: 1})) +func populateWithFakeData(ctx context.Context, a interface{}) error { + return faker.FakeData(a, options.WithRandomMapAndSliceMaxSize(10), options.WithRandomFloatBoundaries(interfaces.RandomFloatBoundary{Start: 0, End: 1})) } func (d *DummyService) StartDataAccessServices() { @@ -62,25 +62,25 @@ func (d *DummyService) StartDataAccessServices() { } // used for any non-pointer data, e.g. all primitive types or slices -func getDummyData[T any]() (T, error) { +func getDummyData[T any](ctx context.Context) (T, error) { var r T - err := commonFakeData(&r) + err := populateWithFakeData(ctx, &r) return r, err } // used for any struct data that should be returned as a pointer -func getDummyStruct[T any]() (*T, error) { +func getDummyStruct[T any](ctx context.Context) (*T, error) { var r T - err := commonFakeData(&r) + err := populateWithFakeData(ctx, &r) return &r, err } // used for any table data that should be returned with paging -func getDummyWithPaging[T any]() ([]T, *t.Paging, error) { +func getDummyWithPaging[T any](ctx context.Context) ([]T, *t.Paging, error) { r := []T{} p := t.Paging{} - _ = commonFakeData(&r) - err := commonFakeData(&p) + _ = populateWithFakeData(ctx, &r) + err := populateWithFakeData(ctx, &p) return r, &p, err } @@ -88,32 +88,28 @@ func (d *DummyService) Close() { // nothing to close } -func (d *DummyService) GetLatestSlot() (uint64, error) { - return getDummyData[uint64]() +func (d *DummyService) GetLatestSlot(ctx context.Context) (uint64, error) { + return getDummyData[uint64](ctx) } -func (d *DummyService) GetLatestFinalizedEpoch() (uint64, error) { - return getDummyData[uint64]() +func (d *DummyService) GetLatestFinalizedEpoch(ctx context.Context) (uint64, error) { + return getDummyData[uint64](ctx) } -func (d *DummyService) GetLatestBlock() (uint64, error) { - return getDummyData[uint64]() +func (d *DummyService) GetLatestBlock(ctx context.Context) (uint64, error) { + return getDummyData[uint64](ctx) } -func (d *DummyService) GetBlockHeightAt(slot uint64) (uint64, error) { - return getDummyData[uint64]() -} - -func (d *DummyService) GetLatestExchangeRates() ([]t.EthConversionRate, error) { - return getDummyData[[]t.EthConversionRate]() +func (d *DummyService) GetLatestExchangeRates(ctx context.Context) ([]t.EthConversionRate, error) { + return getDummyData[[]t.EthConversionRate](ctx) } func (d *DummyService) GetUserByEmail(ctx context.Context, email string) (uint64, error) { - return getDummyData[uint64]() + return getDummyData[uint64](ctx) } func (d *DummyService) CreateUser(ctx context.Context, email, password string) (uint64, error) { - return getDummyData[uint64]() + return getDummyData[uint64](ctx) } func (d *DummyService) RemoveUser(ctx context.Context, userId uint64) error { @@ -129,11 +125,11 @@ func (d *DummyService) UpdateUserPassword(ctx context.Context, userId uint64, pa } func (d *DummyService) GetEmailConfirmationTime(ctx context.Context, userId uint64) (time.Time, error) { - return getDummyData[time.Time]() + return getDummyData[time.Time](ctx) } func (d *DummyService) GetPasswordResetTime(ctx context.Context, userId uint64) (time.Time, error) { - return getDummyData[time.Time]() + return getDummyData[time.Time](ctx) } func (d *DummyService) UpdateEmailConfirmationTime(ctx context.Context, userId uint64) error { @@ -157,66 +153,66 @@ func (d *DummyService) UpdatePasswordResetHash(ctx context.Context, userId uint6 } func (d *DummyService) GetUserInfo(ctx context.Context, userId uint64) (*t.UserInfo, error) { - return getDummyStruct[t.UserInfo]() + return getDummyStruct[t.UserInfo](ctx) } func (d *DummyService) GetUserCredentialInfo(ctx context.Context, userId uint64) (*t.UserCredentialInfo, error) { - return getDummyStruct[t.UserCredentialInfo]() + return getDummyStruct[t.UserCredentialInfo](ctx) } func (d *DummyService) GetUserIdByApiKey(ctx context.Context, apiKey string) (uint64, error) { - return getDummyData[uint64]() + return getDummyData[uint64](ctx) } func (d *DummyService) GetUserIdByConfirmationHash(ctx context.Context, hash string) (uint64, error) { - return getDummyData[uint64]() + return getDummyData[uint64](ctx) } func (d *DummyService) GetUserIdByResetHash(ctx context.Context, hash string) (uint64, error) { - return getDummyData[uint64]() + return getDummyData[uint64](ctx) } func (d *DummyService) GetProductSummary(ctx context.Context) (*t.ProductSummary, error) { - return getDummyStruct[t.ProductSummary]() + return getDummyStruct[t.ProductSummary](ctx) } func (d *DummyService) GetFreeTierPerks(ctx context.Context) (*t.PremiumPerks, error) { - return getDummyStruct[t.PremiumPerks]() + return getDummyStruct[t.PremiumPerks](ctx) } func (d *DummyService) GetValidatorDashboardUser(ctx context.Context, dashboardId t.VDBIdPrimary) (*t.DashboardUser, error) { - return getDummyStruct[t.DashboardUser]() + return getDummyStruct[t.DashboardUser](ctx) } func (d *DummyService) GetValidatorDashboardIdByPublicId(ctx context.Context, publicDashboardId t.VDBIdPublic) (*t.VDBIdPrimary, error) { - return getDummyStruct[t.VDBIdPrimary]() + return getDummyStruct[t.VDBIdPrimary](ctx) } func (d *DummyService) GetValidatorDashboardInfo(ctx context.Context, dashboardId t.VDBIdPrimary) (*t.ValidatorDashboard, error) { - r, err := getDummyStruct[t.ValidatorDashboard]() + r, err := getDummyStruct[t.ValidatorDashboard](ctx) // return semi-valid data to not break staging r.IsArchived = false return r, err } func (d *DummyService) GetValidatorDashboardName(ctx context.Context, dashboardId t.VDBIdPrimary) (string, error) { - return getDummyData[string]() + return getDummyData[string](ctx) } -func (d *DummyService) GetValidatorsFromSlices(indices []uint64, publicKeys []string) ([]t.VDBValidator, error) { - return getDummyData[[]t.VDBValidator]() +func (d *DummyService) GetValidatorsFromSlices(ctx context.Context, indices []uint64, publicKeys []string) ([]t.VDBValidator, error) { + return getDummyData[[]t.VDBValidator](ctx) } func (d *DummyService) GetUserDashboards(ctx context.Context, userId uint64) (*t.UserDashboardsData, error) { - return getDummyStruct[t.UserDashboardsData]() + return getDummyStruct[t.UserDashboardsData](ctx) } func (d *DummyService) CreateValidatorDashboard(ctx context.Context, userId uint64, name string, network uint64) (*t.VDBPostReturnData, error) { - return getDummyStruct[t.VDBPostReturnData]() + return getDummyStruct[t.VDBPostReturnData](ctx) } func (d *DummyService) GetValidatorDashboardOverview(ctx context.Context, dashboardId t.VDBId, protocolModes t.VDBProtocolModes) (*t.VDBOverviewData, error) { - return getDummyStruct[t.VDBOverviewData]() + return getDummyStruct[t.VDBOverviewData](ctx) } func (d *DummyService) RemoveValidatorDashboard(ctx context.Context, dashboardId t.VDBIdPrimary) error { @@ -228,7 +224,7 @@ func (d *DummyService) RemoveValidatorDashboards(ctx context.Context, dashboardI } func (d *DummyService) UpdateValidatorDashboardArchiving(ctx context.Context, dashboardId t.VDBIdPrimary, archivedReason *enums.VDBArchivedReason) (*t.VDBPostArchivingReturnData, error) { - return getDummyStruct[t.VDBPostArchivingReturnData]() + return getDummyStruct[t.VDBPostArchivingReturnData](ctx) } func (d *DummyService) UpdateValidatorDashboardsArchiving(ctx context.Context, dashboards []t.ArchiverDashboardArchiveReason) error { @@ -236,15 +232,15 @@ func (d *DummyService) UpdateValidatorDashboardsArchiving(ctx context.Context, d } func (d *DummyService) UpdateValidatorDashboardName(ctx context.Context, dashboardId t.VDBIdPrimary, name string) (*t.VDBPostReturnData, error) { - return getDummyStruct[t.VDBPostReturnData]() + return getDummyStruct[t.VDBPostReturnData](ctx) } func (d *DummyService) CreateValidatorDashboardGroup(ctx context.Context, dashboardId t.VDBIdPrimary, name string) (*t.VDBPostCreateGroupData, error) { - return getDummyStruct[t.VDBPostCreateGroupData]() + return getDummyStruct[t.VDBPostCreateGroupData](ctx) } func (d *DummyService) UpdateValidatorDashboardGroup(ctx context.Context, dashboardId t.VDBIdPrimary, groupId uint64, name string) (*t.VDBPostCreateGroupData, error) { - return getDummyStruct[t.VDBPostCreateGroupData]() + return getDummyStruct[t.VDBPostCreateGroupData](ctx) } func (d *DummyService) RemoveValidatorDashboardGroup(ctx context.Context, dashboardId t.VDBIdPrimary, groupId uint64) error { @@ -256,23 +252,23 @@ func (d *DummyService) GetValidatorDashboardGroupExists(ctx context.Context, das } func (d *DummyService) AddValidatorDashboardValidators(ctx context.Context, dashboardId t.VDBIdPrimary, groupId uint64, validators []t.VDBValidator) ([]t.VDBPostValidatorsData, error) { - return getDummyData[[]t.VDBPostValidatorsData]() + return getDummyData[[]t.VDBPostValidatorsData](ctx) } func (d *DummyService) AddValidatorDashboardValidatorsByDepositAddress(ctx context.Context, dashboardId t.VDBIdPrimary, groupId uint64, address string, limit uint64) ([]t.VDBPostValidatorsData, error) { - return getDummyData[[]t.VDBPostValidatorsData]() + return getDummyData[[]t.VDBPostValidatorsData](ctx) } func (d *DummyService) AddValidatorDashboardValidatorsByWithdrawalAddress(ctx context.Context, dashboardId t.VDBIdPrimary, groupId uint64, address string, limit uint64) ([]t.VDBPostValidatorsData, error) { - return getDummyData[[]t.VDBPostValidatorsData]() + return getDummyData[[]t.VDBPostValidatorsData](ctx) } func (d *DummyService) AddValidatorDashboardValidatorsByGraffiti(ctx context.Context, dashboardId t.VDBIdPrimary, groupId uint64, graffiti string, limit uint64) ([]t.VDBPostValidatorsData, error) { - return getDummyData[[]t.VDBPostValidatorsData]() + return getDummyData[[]t.VDBPostValidatorsData](ctx) } func (d *DummyService) GetValidatorDashboardValidators(ctx context.Context, dashboardId t.VDBId, groupId int64, cursor string, colSort t.Sort[enums.VDBManageValidatorsColumn], search string, limit uint64) ([]t.VDBManageValidatorsTableRow, *t.Paging, error) { - return getDummyWithPaging[t.VDBManageValidatorsTableRow]() + return getDummyWithPaging[t.VDBManageValidatorsTableRow](ctx) } func (d *DummyService) RemoveValidatorDashboardValidators(ctx context.Context, dashboardId t.VDBIdPrimary, validators []t.VDBValidator) error { @@ -280,15 +276,15 @@ func (d *DummyService) RemoveValidatorDashboardValidators(ctx context.Context, d } func (d *DummyService) CreateValidatorDashboardPublicId(ctx context.Context, dashboardId t.VDBIdPrimary, name string, shareGroups bool) (*t.VDBPublicId, error) { - return getDummyStruct[t.VDBPublicId]() + return getDummyStruct[t.VDBPublicId](ctx) } func (d *DummyService) GetValidatorDashboardPublicId(ctx context.Context, publicDashboardId t.VDBIdPublic) (*t.VDBPublicId, error) { - return getDummyStruct[t.VDBPublicId]() + return getDummyStruct[t.VDBPublicId](ctx) } func (d *DummyService) UpdateValidatorDashboardPublicId(ctx context.Context, publicDashboardId t.VDBIdPublic, name string, shareGroups bool) (*t.VDBPublicId, error) { - return getDummyStruct[t.VDBPublicId]() + return getDummyStruct[t.VDBPublicId](ctx) } func (d *DummyService) RemoveValidatorDashboardPublicId(ctx context.Context, publicDashboardId t.VDBIdPublic) error { @@ -299,76 +295,76 @@ func (d *DummyService) GetValidatorDashboardSlotViz(ctx context.Context, dashboa r := struct { Epochs []t.SlotVizEpoch `faker:"slice_len=4"` }{} - err := commonFakeData(&r) + err := populateWithFakeData(ctx, &r) return r.Epochs, err } func (d *DummyService) GetValidatorDashboardSummary(ctx context.Context, dashboardId t.VDBId, period enums.TimePeriod, cursor string, colSort t.Sort[enums.VDBSummaryColumn], search string, limit uint64, protocolModes t.VDBProtocolModes) ([]t.VDBSummaryTableRow, *t.Paging, error) { - return getDummyWithPaging[t.VDBSummaryTableRow]() + return getDummyWithPaging[t.VDBSummaryTableRow](ctx) } func (d *DummyService) GetValidatorDashboardGroupSummary(ctx context.Context, dashboardId t.VDBId, groupId int64, period enums.TimePeriod, protocolModes t.VDBProtocolModes) (*t.VDBGroupSummaryData, error) { - return getDummyStruct[t.VDBGroupSummaryData]() + return getDummyStruct[t.VDBGroupSummaryData](ctx) } func (d *DummyService) GetValidatorDashboardSummaryChart(ctx context.Context, dashboardId t.VDBId, groupIds []int64, efficiency enums.VDBSummaryChartEfficiencyType, aggregation enums.ChartAggregation, afterTs uint64, beforeTs uint64) (*t.ChartData[int, float64], error) { - return getDummyStruct[t.ChartData[int, float64]]() + return getDummyStruct[t.ChartData[int, float64]](ctx) } func (d *DummyService) GetValidatorDashboardSummaryValidators(ctx context.Context, dashboardId t.VDBId, groupId int64) (*t.VDBGeneralSummaryValidators, error) { - return getDummyStruct[t.VDBGeneralSummaryValidators]() + return getDummyStruct[t.VDBGeneralSummaryValidators](ctx) } func (d *DummyService) GetValidatorDashboardSyncSummaryValidators(ctx context.Context, dashboardId t.VDBId, groupId int64, period enums.TimePeriod) (*t.VDBSyncSummaryValidators, error) { - return getDummyStruct[t.VDBSyncSummaryValidators]() + return getDummyStruct[t.VDBSyncSummaryValidators](ctx) } func (d *DummyService) GetValidatorDashboardSlashingsSummaryValidators(ctx context.Context, dashboardId t.VDBId, groupId int64, period enums.TimePeriod) (*t.VDBSlashingsSummaryValidators, error) { - return getDummyStruct[t.VDBSlashingsSummaryValidators]() + return getDummyStruct[t.VDBSlashingsSummaryValidators](ctx) } func (d *DummyService) GetValidatorDashboardProposalSummaryValidators(ctx context.Context, dashboardId t.VDBId, groupId int64, period enums.TimePeriod) (*t.VDBProposalSummaryValidators, error) { - return getDummyStruct[t.VDBProposalSummaryValidators]() + return getDummyStruct[t.VDBProposalSummaryValidators](ctx) } func (d *DummyService) GetValidatorDashboardRewards(ctx context.Context, dashboardId t.VDBId, cursor string, colSort t.Sort[enums.VDBRewardsColumn], search string, limit uint64, protocolModes t.VDBProtocolModes) ([]t.VDBRewardsTableRow, *t.Paging, error) { - return getDummyWithPaging[t.VDBRewardsTableRow]() + return getDummyWithPaging[t.VDBRewardsTableRow](ctx) } func (d *DummyService) GetValidatorDashboardGroupRewards(ctx context.Context, dashboardId t.VDBId, groupId int64, epoch uint64, protocolModes t.VDBProtocolModes) (*t.VDBGroupRewardsData, error) { - return getDummyStruct[t.VDBGroupRewardsData]() + return getDummyStruct[t.VDBGroupRewardsData](ctx) } func (d *DummyService) GetValidatorDashboardRewardsChart(ctx context.Context, dashboardId t.VDBId, protocolModes t.VDBProtocolModes) (*t.ChartData[int, decimal.Decimal], error) { - return getDummyStruct[t.ChartData[int, decimal.Decimal]]() + return getDummyStruct[t.ChartData[int, decimal.Decimal]](ctx) } func (d *DummyService) GetValidatorDashboardDuties(ctx context.Context, dashboardId t.VDBId, epoch uint64, groupId int64, cursor string, colSort t.Sort[enums.VDBDutiesColumn], search string, limit uint64, protocolModes t.VDBProtocolModes) ([]t.VDBEpochDutiesTableRow, *t.Paging, error) { - return getDummyWithPaging[t.VDBEpochDutiesTableRow]() + return getDummyWithPaging[t.VDBEpochDutiesTableRow](ctx) } func (d *DummyService) GetValidatorDashboardBlocks(ctx context.Context, dashboardId t.VDBId, cursor string, colSort t.Sort[enums.VDBBlocksColumn], search string, limit uint64, protocolModes t.VDBProtocolModes) ([]t.VDBBlocksTableRow, *t.Paging, error) { - return getDummyWithPaging[t.VDBBlocksTableRow]() + return getDummyWithPaging[t.VDBBlocksTableRow](ctx) } func (d *DummyService) GetValidatorDashboardHeatmap(ctx context.Context, dashboardId t.VDBId, protocolModes t.VDBProtocolModes, aggregation enums.ChartAggregation, afterTs uint64, beforeTs uint64) (*t.VDBHeatmap, error) { - return getDummyStruct[t.VDBHeatmap]() + return getDummyStruct[t.VDBHeatmap](ctx) } func (d *DummyService) GetValidatorDashboardGroupHeatmap(ctx context.Context, dashboardId t.VDBId, groupId uint64, protocolModes t.VDBProtocolModes, aggregation enums.ChartAggregation, timestamp uint64) (*t.VDBHeatmapTooltipData, error) { - return getDummyStruct[t.VDBHeatmapTooltipData]() + return getDummyStruct[t.VDBHeatmapTooltipData](ctx) } func (d *DummyService) GetValidatorDashboardElDeposits(ctx context.Context, dashboardId t.VDBId, cursor string, limit uint64) ([]t.VDBExecutionDepositsTableRow, *t.Paging, error) { - return getDummyWithPaging[t.VDBExecutionDepositsTableRow]() + return getDummyWithPaging[t.VDBExecutionDepositsTableRow](ctx) } func (d *DummyService) GetValidatorDashboardClDeposits(ctx context.Context, dashboardId t.VDBId, cursor string, limit uint64) ([]t.VDBConsensusDepositsTableRow, *t.Paging, error) { - return getDummyWithPaging[t.VDBConsensusDepositsTableRow]() + return getDummyWithPaging[t.VDBConsensusDepositsTableRow](ctx) } func (d *DummyService) GetValidatorDashboardTotalElDeposits(ctx context.Context, dashboardId t.VDBId) (*t.VDBTotalExecutionDepositsData, error) { - return getDummyStruct[t.VDBTotalExecutionDepositsData]() + return getDummyStruct[t.VDBTotalExecutionDepositsData](ctx) } func (d *DummyService) GetValidatorDashboardTotalClDeposits(ctx context.Context, dashboardId t.VDBId) (*t.VDBTotalConsensusDepositsData, error) { - return getDummyStruct[t.VDBTotalConsensusDepositsData]() + return getDummyStruct[t.VDBTotalConsensusDepositsData](ctx) } func (d *DummyService) GetValidatorDashboardWithdrawals(ctx context.Context, dashboardId t.VDBId, cursor string, colSort t.Sort[enums.VDBWithdrawalsColumn], search string, limit uint64, protocolModes t.VDBProtocolModes) ([]t.VDBWithdrawalsTableRow, *t.Paging, error) { @@ -376,19 +372,19 @@ func (d *DummyService) GetValidatorDashboardWithdrawals(ctx context.Context, das } func (d *DummyService) GetValidatorDashboardTotalWithdrawals(ctx context.Context, dashboardId t.VDBId, search string, protocolModes t.VDBProtocolModes) (*t.VDBTotalWithdrawalsData, error) { - return getDummyStruct[t.VDBTotalWithdrawalsData]() + return getDummyStruct[t.VDBTotalWithdrawalsData](ctx) } func (d *DummyService) GetValidatorDashboardRocketPool(ctx context.Context, dashboardId t.VDBId, cursor string, colSort t.Sort[enums.VDBRocketPoolColumn], search string, limit uint64) ([]t.VDBRocketPoolTableRow, *t.Paging, error) { - return getDummyWithPaging[t.VDBRocketPoolTableRow]() + return getDummyWithPaging[t.VDBRocketPoolTableRow](ctx) } func (d *DummyService) GetValidatorDashboardTotalRocketPool(ctx context.Context, dashboardId t.VDBId, search string) (*t.VDBRocketPoolTableRow, error) { - return getDummyStruct[t.VDBRocketPoolTableRow]() + return getDummyStruct[t.VDBRocketPoolTableRow](ctx) } func (d *DummyService) GetValidatorDashboardRocketPoolMinipools(ctx context.Context, dashboardId t.VDBId, node string, cursor string, colSort t.Sort[enums.VDBRocketPoolMinipoolsColumn], search string, limit uint64) ([]t.VDBRocketPoolMinipoolsTableRow, *t.Paging, error) { - return getDummyWithPaging[t.VDBRocketPoolMinipoolsTableRow]() + return getDummyWithPaging[t.VDBRocketPoolMinipoolsTableRow](ctx) } func (d *DummyService) GetAllNetworks() ([]t.NetworkInfo, error) { @@ -492,82 +488,82 @@ func (d *DummyService) GetAllClients() ([]t.ClientInfo, error) { } func (d *DummyService) GetSearchValidatorByIndex(ctx context.Context, chainId, index uint64) (*t.SearchValidator, error) { - return getDummyStruct[t.SearchValidator]() + return getDummyStruct[t.SearchValidator](ctx) } func (d *DummyService) GetSearchValidatorByPublicKey(ctx context.Context, chainId uint64, publicKey []byte) (*t.SearchValidator, error) { - return getDummyStruct[t.SearchValidator]() + return getDummyStruct[t.SearchValidator](ctx) } func (d *DummyService) GetSearchValidatorsByDepositAddress(ctx context.Context, chainId uint64, address []byte) (*t.SearchValidatorsByDepositAddress, error) { - return getDummyStruct[t.SearchValidatorsByDepositAddress]() + return getDummyStruct[t.SearchValidatorsByDepositAddress](ctx) } func (d *DummyService) GetSearchValidatorsByDepositEnsName(ctx context.Context, chainId uint64, ensName string) (*t.SearchValidatorsByDepositEnsName, error) { - return getDummyStruct[t.SearchValidatorsByDepositEnsName]() + return getDummyStruct[t.SearchValidatorsByDepositEnsName](ctx) } func (d *DummyService) GetSearchValidatorsByWithdrawalCredential(ctx context.Context, chainId uint64, credential []byte) (*t.SearchValidatorsByWithdrwalCredential, error) { - return getDummyStruct[t.SearchValidatorsByWithdrwalCredential]() + return getDummyStruct[t.SearchValidatorsByWithdrwalCredential](ctx) } func (d *DummyService) GetSearchValidatorsByWithdrawalEnsName(ctx context.Context, chainId uint64, ensName string) (*t.SearchValidatorsByWithrawalEnsName, error) { - return getDummyStruct[t.SearchValidatorsByWithrawalEnsName]() + return getDummyStruct[t.SearchValidatorsByWithrawalEnsName](ctx) } func (d *DummyService) GetSearchValidatorsByGraffiti(ctx context.Context, chainId uint64, graffiti string) (*t.SearchValidatorsByGraffiti, error) { - return getDummyStruct[t.SearchValidatorsByGraffiti]() + return getDummyStruct[t.SearchValidatorsByGraffiti](ctx) } func (d *DummyService) GetUserValidatorDashboardCount(ctx context.Context, userId uint64, active bool) (uint64, error) { - return getDummyData[uint64]() + return getDummyData[uint64](ctx) } func (d *DummyService) GetValidatorDashboardGroupCount(ctx context.Context, dashboardId t.VDBIdPrimary) (uint64, error) { - return getDummyData[uint64]() + return getDummyData[uint64](ctx) } func (d *DummyService) GetValidatorDashboardValidatorsCount(ctx context.Context, dashboardId t.VDBIdPrimary) (uint64, error) { - return getDummyData[uint64]() + return getDummyData[uint64](ctx) } func (d *DummyService) GetValidatorDashboardPublicIdCount(ctx context.Context, dashboardId t.VDBIdPrimary) (uint64, error) { - return getDummyData[uint64]() + return getDummyData[uint64](ctx) } func (d *DummyService) GetNotificationOverview(ctx context.Context, userId uint64) (*t.NotificationOverviewData, error) { - return getDummyStruct[t.NotificationOverviewData]() + return getDummyStruct[t.NotificationOverviewData](ctx) } func (d *DummyService) GetDashboardNotifications(ctx context.Context, userId uint64, chainIds []uint64, cursor string, colSort t.Sort[enums.NotificationDashboardsColumn], search string, limit uint64) ([]t.NotificationDashboardsTableRow, *t.Paging, error) { - return getDummyWithPaging[t.NotificationDashboardsTableRow]() + return getDummyWithPaging[t.NotificationDashboardsTableRow](ctx) } func (d *DummyService) GetValidatorDashboardNotificationDetails(ctx context.Context, dashboardId t.VDBIdPrimary, groupId uint64, epoch uint64, search string) (*t.NotificationValidatorDashboardDetail, error) { - return getDummyStruct[t.NotificationValidatorDashboardDetail]() + return getDummyStruct[t.NotificationValidatorDashboardDetail](ctx) } func (d *DummyService) GetAccountDashboardNotificationDetails(ctx context.Context, dashboardId uint64, groupId uint64, epoch uint64, search string) (*t.NotificationAccountDashboardDetail, error) { - return getDummyStruct[t.NotificationAccountDashboardDetail]() + return getDummyStruct[t.NotificationAccountDashboardDetail](ctx) } func (d *DummyService) GetMachineNotifications(ctx context.Context, userId uint64, cursor string, colSort t.Sort[enums.NotificationMachinesColumn], search string, limit uint64) ([]t.NotificationMachinesTableRow, *t.Paging, error) { - return getDummyWithPaging[t.NotificationMachinesTableRow]() + return getDummyWithPaging[t.NotificationMachinesTableRow](ctx) } func (d *DummyService) GetClientNotifications(ctx context.Context, userId uint64, cursor string, colSort t.Sort[enums.NotificationClientsColumn], search string, limit uint64) ([]t.NotificationClientsTableRow, *t.Paging, error) { - return getDummyWithPaging[t.NotificationClientsTableRow]() + return getDummyWithPaging[t.NotificationClientsTableRow](ctx) } func (d *DummyService) GetRocketPoolNotifications(ctx context.Context, userId uint64, cursor string, colSort t.Sort[enums.NotificationRocketPoolColumn], search string, limit uint64) ([]t.NotificationRocketPoolTableRow, *t.Paging, error) { - return getDummyWithPaging[t.NotificationRocketPoolTableRow]() + return getDummyWithPaging[t.NotificationRocketPoolTableRow](ctx) } func (d *DummyService) GetNetworkNotifications(ctx context.Context, userId uint64, cursor string, colSort t.Sort[enums.NotificationNetworksColumn], limit uint64) ([]t.NotificationNetworksTableRow, *t.Paging, error) { - return getDummyWithPaging[t.NotificationNetworksTableRow]() + return getDummyWithPaging[t.NotificationNetworksTableRow](ctx) } func (d *DummyService) GetNotificationSettings(ctx context.Context, userId uint64) (*t.NotificationSettings, error) { - return getDummyStruct[t.NotificationSettings]() + return getDummyStruct[t.NotificationSettings](ctx) } func (d *DummyService) GetNotificationSettingsDefaultValues(ctx context.Context) (*t.NotificationSettingsDefaultValues, error) { - return getDummyStruct[t.NotificationSettingsDefaultValues]() + return getDummyStruct[t.NotificationSettingsDefaultValues](ctx) } func (d *DummyService) UpdateNotificationSettingsGeneral(ctx context.Context, userId uint64, settings t.NotificationSettingsGeneral) error { return nil @@ -583,11 +579,11 @@ func (d *DummyService) DeleteNotificationSettingsPairedDevice(ctx context.Contex } func (d *DummyService) UpdateNotificationSettingsClients(ctx context.Context, userId uint64, clientId uint64, IsSubscribed bool) (*t.NotificationSettingsClient, error) { - return getDummyStruct[t.NotificationSettingsClient]() + return getDummyStruct[t.NotificationSettingsClient](ctx) } func (d *DummyService) GetNotificationSettingsDashboards(ctx context.Context, userId uint64, cursor string, colSort t.Sort[enums.NotificationSettingsDashboardColumn], search string, limit uint64) ([]t.NotificationSettingsDashboardsTableRow, *t.Paging, error) { - r, p, err := getDummyWithPaging[t.NotificationSettingsDashboardsTableRow]() + r, p, err := getDummyWithPaging[t.NotificationSettingsDashboardsTableRow](ctx) for i, n := range r { var settings interface{} if n.IsAccountDashboard { @@ -595,7 +591,7 @@ func (d *DummyService) GetNotificationSettingsDashboards(ctx context.Context, us } else { settings = t.NotificationSettingsValidatorDashboard{} } - _ = commonFakeData(&settings) + _ = populateWithFakeData(ctx, &settings) r[i].Settings = settings } return r, p, err @@ -611,7 +607,7 @@ func (d *DummyService) CreateAdConfiguration(ctx context.Context, key, jquerySel } func (d *DummyService) GetAdConfigurations(ctx context.Context, keys []string) ([]t.AdConfigurationData, error) { - return getDummyData[[]t.AdConfigurationData]() + return getDummyData[[]t.AdConfigurationData](ctx) } func (d *DummyService) UpdateAdConfiguration(ctx context.Context, key, jquerySelector string, insertMode enums.AdInsertMode, refreshInterval uint64, forAllUsers bool, bannerId uint64, htmlContent string, enabled bool) error { @@ -623,128 +619,128 @@ func (d *DummyService) RemoveAdConfiguration(ctx context.Context, key string) er } func (d *DummyService) GetLatestExportedChartTs(ctx context.Context, aggregation enums.ChartAggregation) (uint64, error) { - return getDummyData[uint64]() + return getDummyData[uint64](ctx) } -func (d *DummyService) GetUserIdByRefreshToken(claimUserID, claimAppID, claimDeviceID uint64, hashedRefreshToken string) (uint64, error) { - return getDummyData[uint64]() +func (d *DummyService) GetUserIdByRefreshToken(ctx context.Context, claimUserID, claimAppID, claimDeviceID uint64, hashedRefreshToken string) (uint64, error) { + return getDummyData[uint64](ctx) } -func (d *DummyService) MigrateMobileSession(oldHashedRefreshToken, newHashedRefreshToken, deviceID, deviceName string) error { +func (d *DummyService) MigrateMobileSession(ctx context.Context, oldHashedRefreshToken, newHashedRefreshToken, deviceID, deviceName string) error { return nil } -func (d *DummyService) GetAppDataFromRedirectUri(callback string) (*t.OAuthAppData, error) { - return getDummyStruct[t.OAuthAppData]() +func (d *DummyService) GetAppDataFromRedirectUri(ctx context.Context, callback string) (*t.OAuthAppData, error) { + return getDummyStruct[t.OAuthAppData](ctx) } -func (d *DummyService) AddUserDevice(userID uint64, hashedRefreshToken string, deviceID, deviceName string, appID uint64) error { +func (d *DummyService) AddUserDevice(ctx context.Context, userID uint64, hashedRefreshToken string, deviceID, deviceName string, appID uint64) error { return nil } -func (d *DummyService) AddMobileNotificationToken(userID uint64, deviceID, notifyToken string) error { +func (d *DummyService) AddMobileNotificationToken(ctx context.Context, userID uint64, deviceID, notifyToken string) error { return nil } -func (d *DummyService) GetAppSubscriptionCount(userID uint64) (uint64, error) { - return getDummyData[uint64]() +func (d *DummyService) GetAppSubscriptionCount(ctx context.Context, userID uint64) (uint64, error) { + return getDummyData[uint64](ctx) } -func (d *DummyService) AddMobilePurchase(tx *sql.Tx, userID uint64, paymentDetails t.MobileSubscription, verifyResponse *userservice.VerifyResponse, extSubscriptionId string) error { +func (d *DummyService) AddMobilePurchase(ctx context.Context, tx *sql.Tx, userID uint64, paymentDetails t.MobileSubscription, verifyResponse *userservice.VerifyResponse, extSubscriptionId string) error { return nil } func (d *DummyService) GetBlockOverview(ctx context.Context, chainId, block uint64) (*t.BlockOverview, error) { - return getDummyStruct[t.BlockOverview]() + return getDummyStruct[t.BlockOverview](ctx) } func (d *DummyService) GetBlockTransactions(ctx context.Context, chainId, block uint64) ([]t.BlockTransactionTableRow, error) { - return getDummyData[[]t.BlockTransactionTableRow]() + return getDummyData[[]t.BlockTransactionTableRow](ctx) } func (d *DummyService) GetBlock(ctx context.Context, chainId, block uint64) (*t.BlockSummary, error) { - return getDummyStruct[t.BlockSummary]() + return getDummyStruct[t.BlockSummary](ctx) } func (d *DummyService) GetBlockVotes(ctx context.Context, chainId, block uint64) ([]t.BlockVoteTableRow, error) { - return getDummyData[[]t.BlockVoteTableRow]() + return getDummyData[[]t.BlockVoteTableRow](ctx) } func (d *DummyService) GetBlockAttestations(ctx context.Context, chainId, block uint64) ([]t.BlockAttestationTableRow, error) { - return getDummyData[[]t.BlockAttestationTableRow]() + return getDummyData[[]t.BlockAttestationTableRow](ctx) } func (d *DummyService) GetBlockWithdrawals(ctx context.Context, chainId, block uint64) ([]t.BlockWithdrawalTableRow, error) { - return getDummyData[[]t.BlockWithdrawalTableRow]() + return getDummyData[[]t.BlockWithdrawalTableRow](ctx) } func (d *DummyService) GetBlockBlsChanges(ctx context.Context, chainId, block uint64) ([]t.BlockBlsChangeTableRow, error) { - return getDummyData[[]t.BlockBlsChangeTableRow]() + return getDummyData[[]t.BlockBlsChangeTableRow](ctx) } func (d *DummyService) GetBlockVoluntaryExits(ctx context.Context, chainId, block uint64) ([]t.BlockVoluntaryExitTableRow, error) { - return getDummyData[[]t.BlockVoluntaryExitTableRow]() + return getDummyData[[]t.BlockVoluntaryExitTableRow](ctx) } func (d *DummyService) GetBlockBlobs(ctx context.Context, chainId, block uint64) ([]t.BlockBlobTableRow, error) { - return getDummyData[[]t.BlockBlobTableRow]() + return getDummyData[[]t.BlockBlobTableRow](ctx) } func (d *DummyService) GetSlot(ctx context.Context, chainId, block uint64) (*t.BlockSummary, error) { - return getDummyStruct[t.BlockSummary]() + return getDummyStruct[t.BlockSummary](ctx) } func (d *DummyService) GetSlotOverview(ctx context.Context, chainId, block uint64) (*t.BlockOverview, error) { - return getDummyStruct[t.BlockOverview]() + return getDummyStruct[t.BlockOverview](ctx) } func (d *DummyService) GetSlotTransactions(ctx context.Context, chainId, block uint64) ([]t.BlockTransactionTableRow, error) { - return getDummyData[[]t.BlockTransactionTableRow]() + return getDummyData[[]t.BlockTransactionTableRow](ctx) } func (d *DummyService) GetSlotVotes(ctx context.Context, chainId, block uint64) ([]t.BlockVoteTableRow, error) { - return getDummyData[[]t.BlockVoteTableRow]() + return getDummyData[[]t.BlockVoteTableRow](ctx) } func (d *DummyService) GetSlotAttestations(ctx context.Context, chainId, block uint64) ([]t.BlockAttestationTableRow, error) { - return getDummyData[[]t.BlockAttestationTableRow]() + return getDummyData[[]t.BlockAttestationTableRow](ctx) } func (d *DummyService) GetSlotWithdrawals(ctx context.Context, chainId, block uint64) ([]t.BlockWithdrawalTableRow, error) { - return getDummyData[[]t.BlockWithdrawalTableRow]() + return getDummyData[[]t.BlockWithdrawalTableRow](ctx) } func (d *DummyService) GetSlotBlsChanges(ctx context.Context, chainId, block uint64) ([]t.BlockBlsChangeTableRow, error) { - return getDummyData[[]t.BlockBlsChangeTableRow]() + return getDummyData[[]t.BlockBlsChangeTableRow](ctx) } func (d *DummyService) GetSlotVoluntaryExits(ctx context.Context, chainId, block uint64) ([]t.BlockVoluntaryExitTableRow, error) { - return getDummyData[[]t.BlockVoluntaryExitTableRow]() + return getDummyData[[]t.BlockVoluntaryExitTableRow](ctx) } func (d *DummyService) GetSlotBlobs(ctx context.Context, chainId, block uint64) ([]t.BlockBlobTableRow, error) { - return getDummyData[[]t.BlockBlobTableRow]() + return getDummyData[[]t.BlockBlobTableRow](ctx) } func (d *DummyService) GetValidatorDashboardsCountInfo(ctx context.Context) (map[uint64][]t.ArchiverDashboard, error) { - return getDummyData[map[uint64][]t.ArchiverDashboard]() + return getDummyData[map[uint64][]t.ArchiverDashboard](ctx) } func (d *DummyService) GetRocketPoolOverview(ctx context.Context) (*t.RocketPoolData, error) { - return getDummyStruct[t.RocketPoolData]() + return getDummyStruct[t.RocketPoolData](ctx) } func (d *DummyService) GetApiWeights(ctx context.Context) ([]t.ApiWeightItem, error) { - return getDummyData[[]t.ApiWeightItem]() + return getDummyData[[]t.ApiWeightItem](ctx) } func (d *DummyService) GetHealthz(ctx context.Context, showAll bool) t.HealthzData { - r, _ := getDummyData[t.HealthzData]() + r, _ := getDummyData[t.HealthzData](ctx) return r } func (d *DummyService) GetLatestBundleForNativeVersion(ctx context.Context, nativeVersion uint64) (*t.MobileAppBundleStats, error) { - return getDummyStruct[t.MobileAppBundleStats]() + return getDummyStruct[t.MobileAppBundleStats](ctx) } func (d *DummyService) IncrementBundleDeliveryCount(ctx context.Context, bundleVerison uint64) error { @@ -752,11 +748,11 @@ func (d *DummyService) IncrementBundleDeliveryCount(ctx context.Context, bundleV } func (d *DummyService) GetValidatorDashboardMobileWidget(ctx context.Context, dashboardId t.VDBIdPrimary) (*t.MobileWidgetData, error) { - return getDummyStruct[t.MobileWidgetData]() + return getDummyStruct[t.MobileWidgetData](ctx) } func (d *DummyService) GetUserMachineMetrics(ctx context.Context, userID uint64, limit int, offset int) (*t.MachineMetricsData, error) { - data, err := getDummyStruct[t.MachineMetricsData]() + data, err := getDummyStruct[t.MachineMetricsData](ctx) if err != nil { return nil, err } @@ -777,7 +773,7 @@ func (d *DummyService) PostUserMachineMetrics(ctx context.Context, userID uint64 } func (d *DummyService) GetValidatorDashboardMobileValidators(ctx context.Context, dashboardId t.VDBId, period enums.TimePeriod, cursor string, colSort t.Sort[enums.VDBMobileValidatorsColumn], search string, limit uint64) ([]t.MobileValidatorDashboardValidatorsTableRow, *t.Paging, error) { - return getDummyWithPaging[t.MobileValidatorDashboardValidatorsTableRow]() + return getDummyWithPaging[t.MobileValidatorDashboardValidatorsTableRow](ctx) } func (d *DummyService) QueueTestEmailNotification(ctx context.Context, userId uint64) error { diff --git a/backend/pkg/api/data_access/header.go b/backend/pkg/api/data_access/header.go index 1a91ad832..d06b41439 100644 --- a/backend/pkg/api/data_access/header.go +++ b/backend/pkg/api/data_access/header.go @@ -12,24 +12,24 @@ import ( "github.com/gobitfly/beaconchain/pkg/commons/utils" ) -func (d *DataAccessService) GetLatestSlot() (uint64, error) { +func (d *DataAccessService) GetLatestSlot(ctx context.Context) (uint64, error) { latestSlot := cache.LatestSlot.Get() return latestSlot, nil } -func (d *DataAccessService) GetLatestFinalizedEpoch() (uint64, error) { +func (d *DataAccessService) GetLatestFinalizedEpoch(ctx context.Context) (uint64, error) { finalizedEpoch := cache.LatestFinalizedEpoch.Get() return finalizedEpoch, nil } -func (d *DataAccessService) GetLatestBlock() (uint64, error) { +func (d *DataAccessService) GetLatestBlock(ctx context.Context) (uint64, error) { // @DATA-ACCESS implement - return d.dummy.GetLatestBlock() + return d.dummy.GetLatestBlock(ctx) } -func (d *DataAccessService) GetBlockHeightAt(slot uint64) (uint64, error) { +func (d *DataAccessService) GetBlockHeightAt(ctx context.Context, slot uint64) (uint64, error) { // @DATA-ACCESS implement; return error if no block at slot - return d.dummy.GetBlockHeightAt(slot) + return getDummyData[uint64](ctx) } // returns the block number of the latest existing block at or before the given slot @@ -69,7 +69,7 @@ func (d *DataAccessService) GetLatestBlockHeightsForEpoch(ctx context.Context, e return res, nil } -func (d *DataAccessService) GetLatestExchangeRates() ([]t.EthConversionRate, error) { +func (d *DataAccessService) GetLatestExchangeRates(ctx context.Context) ([]t.EthConversionRate, error) { result := []t.EthConversionRate{} availableCurrencies := price.GetAvailableCurrencies() diff --git a/backend/pkg/api/data_access/mobile.go b/backend/pkg/api/data_access/mobile.go index dc526746b..373688439 100644 --- a/backend/pkg/api/data_access/mobile.go +++ b/backend/pkg/api/data_access/mobile.go @@ -18,25 +18,25 @@ import ( ) type AppRepository interface { - GetUserIdByRefreshToken(claimUserID, claimAppID, claimDeviceID uint64, hashedRefreshToken string) (uint64, error) - MigrateMobileSession(oldHashedRefreshToken, newHashedRefreshToken, deviceID, deviceName string) error - AddUserDevice(userID uint64, hashedRefreshToken string, deviceID, deviceName string, appID uint64) error - GetAppDataFromRedirectUri(callback string) (*t.OAuthAppData, error) - AddMobileNotificationToken(userID uint64, deviceID, notifyToken string) error - GetAppSubscriptionCount(userID uint64) (uint64, error) - AddMobilePurchase(tx *sql.Tx, userID uint64, paymentDetails t.MobileSubscription, verifyResponse *userservice.VerifyResponse, extSubscriptionId string) error + GetUserIdByRefreshToken(ctx context.Context, claimUserID, claimAppID, claimDeviceID uint64, hashedRefreshToken string) (uint64, error) + MigrateMobileSession(ctx context.Context, oldHashedRefreshToken, newHashedRefreshToken, deviceID, deviceName string) error + AddUserDevice(ctx context.Context, userID uint64, hashedRefreshToken string, deviceID, deviceName string, appID uint64) error + GetAppDataFromRedirectUri(ctx context.Context, callback string) (*t.OAuthAppData, error) + AddMobileNotificationToken(ctx context.Context, userID uint64, deviceID, notifyToken string) error + GetAppSubscriptionCount(ctx context.Context, userID uint64) (uint64, error) + AddMobilePurchase(ctx context.Context, tx *sql.Tx, userID uint64, paymentDetails t.MobileSubscription, verifyResponse *userservice.VerifyResponse, extSubscriptionId string) error GetLatestBundleForNativeVersion(ctx context.Context, nativeVersion uint64) (*t.MobileAppBundleStats, error) IncrementBundleDeliveryCount(ctx context.Context, bundleVerison uint64) error GetValidatorDashboardMobileValidators(ctx context.Context, dashboardId t.VDBId, period enums.TimePeriod, cursor string, colSort t.Sort[enums.VDBMobileValidatorsColumn], search string, limit uint64) ([]t.MobileValidatorDashboardValidatorsTableRow, *t.Paging, error) } // GetUserIdByRefreshToken basically used to confirm the claimed user id with the refresh token. Returns the userId if successful -func (d *DataAccessService) GetUserIdByRefreshToken(claimUserID, claimAppID, claimDeviceID uint64, hashedRefreshToken string) (uint64, error) { +func (d *DataAccessService) GetUserIdByRefreshToken(ctx context.Context, claimUserID, claimAppID, claimDeviceID uint64, hashedRefreshToken string) (uint64, error) { if hashedRefreshToken == "" { // sanity return 0, errors.New("empty refresh token") } var userID uint64 - err := d.userWriter.Get(&userID, + err := d.userWriter.GetContext(ctx, &userID, `SELECT user_id FROM users_devices WHERE user_id = $1 AND refresh_token = $2 AND app_id = $3 AND id = $4 AND active = true`, claimUserID, hashedRefreshToken, claimAppID, claimDeviceID) if errors.Is(err, sql.ErrNoRows) { @@ -45,8 +45,8 @@ func (d *DataAccessService) GetUserIdByRefreshToken(claimUserID, claimAppID, cla return userID, err } -func (d *DataAccessService) MigrateMobileSession(oldHashedRefreshToken, newHashedRefreshToken, deviceID, deviceName string) error { - result, err := d.userWriter.Exec("UPDATE users_devices SET refresh_token = $2, device_identifier = $3, device_name = $4 WHERE refresh_token = $1", oldHashedRefreshToken, newHashedRefreshToken, deviceID, deviceName) +func (d *DataAccessService) MigrateMobileSession(ctx context.Context, oldHashedRefreshToken, newHashedRefreshToken, deviceID, deviceName string) error { + result, err := d.userWriter.ExecContext(ctx, "UPDATE users_devices SET refresh_token = $2, device_identifier = $3, device_name = $4 WHERE refresh_token = $1", oldHashedRefreshToken, newHashedRefreshToken, deviceID, deviceName) if err != nil { return errors.Wrap(err, "Error updating refresh token") } @@ -63,21 +63,21 @@ func (d *DataAccessService) MigrateMobileSession(oldHashedRefreshToken, newHashe return err } -func (d *DataAccessService) GetAppDataFromRedirectUri(callback string) (*t.OAuthAppData, error) { +func (d *DataAccessService) GetAppDataFromRedirectUri(ctx context.Context, callback string) (*t.OAuthAppData, error) { data := t.OAuthAppData{} - err := d.userWriter.Get(&data, "SELECT id, app_name, redirect_uri, active, owner_id FROM oauth_apps WHERE active = true AND redirect_uri = $1", callback) + err := d.userWriter.GetContext(ctx, &data, "SELECT id, app_name, redirect_uri, active, owner_id FROM oauth_apps WHERE active = true AND redirect_uri = $1", callback) return &data, err } -func (d *DataAccessService) AddUserDevice(userID uint64, hashedRefreshToken string, deviceID, deviceName string, appID uint64) error { - _, err := d.userWriter.Exec("INSERT INTO users_devices (user_id, refresh_token, device_identifier, device_name, app_id, created_ts) VALUES($1, $2, $3, $4, $5, 'NOW()') ON CONFLICT DO NOTHING", +func (d *DataAccessService) AddUserDevice(ctx context.Context, userID uint64, hashedRefreshToken string, deviceID, deviceName string, appID uint64) error { + _, err := d.userWriter.ExecContext(ctx, "INSERT INTO users_devices (user_id, refresh_token, device_identifier, device_name, app_id, created_ts) VALUES($1, $2, $3, $4, $5, 'NOW()') ON CONFLICT DO NOTHING", userID, hashedRefreshToken, deviceID, deviceName, appID, ) return err } -func (d *DataAccessService) AddMobileNotificationToken(userID uint64, deviceID, notifyToken string) error { - _, err := d.userWriter.Exec("UPDATE users_devices SET notification_token = $1 WHERE user_id = $2 AND device_identifier = $3;", +func (d *DataAccessService) AddMobileNotificationToken(ctx context.Context, userID uint64, deviceID, notifyToken string) error { + _, err := d.userWriter.ExecContext(ctx, "UPDATE users_devices SET notification_token = $1 WHERE user_id = $2 AND device_identifier = $3;", notifyToken, userID, deviceID, ) if errors.Is(err, sql.ErrNoRows) { @@ -86,13 +86,13 @@ func (d *DataAccessService) AddMobileNotificationToken(userID uint64, deviceID, return err } -func (d *DataAccessService) GetAppSubscriptionCount(userID uint64) (uint64, error) { +func (d *DataAccessService) GetAppSubscriptionCount(ctx context.Context, userID uint64) (uint64, error) { var count uint64 - err := d.userReader.Get(&count, "SELECT COUNT(receipt) FROM users_app_subscriptions WHERE user_id = $1", userID) + err := d.userReader.GetContext(ctx, &count, "SELECT COUNT(receipt) FROM users_app_subscriptions WHERE user_id = $1", userID) return count, err } -func (d *DataAccessService) AddMobilePurchase(tx *sql.Tx, userID uint64, paymentDetails t.MobileSubscription, verifyResponse *userservice.VerifyResponse, extSubscriptionId string) error { +func (d *DataAccessService) AddMobilePurchase(ctx context.Context, tx *sql.Tx, userID uint64, paymentDetails t.MobileSubscription, verifyResponse *userservice.VerifyResponse, extSubscriptionId string) error { now := time.Now() nowTs := now.Unix() receiptHash := utils.HashAndEncode(verifyResponse.Receipt) @@ -103,11 +103,11 @@ func (d *DataAccessService) AddMobilePurchase(tx *sql.Tx, userID uint64, payment ON CONFLICT(receipt_hash) DO UPDATE SET product_id = $2, active = $7, updated_at = TO_TIMESTAMP($5);` var err error if tx == nil { - _, err = d.userWriter.Exec(query, + _, err = d.userWriter.ExecContext(ctx, query, userID, verifyResponse.ProductID, paymentDetails.PriceMicros, paymentDetails.Currency, nowTs, nowTs, verifyResponse.Valid, verifyResponse.Valid, paymentDetails.Transaction.Type, verifyResponse.Receipt, verifyResponse.ExpirationDate, verifyResponse.RejectReason, receiptHash, extSubscriptionId, ) } else { - _, err = tx.Exec(query, + _, err = tx.ExecContext(ctx, query, userID, verifyResponse.ProductID, paymentDetails.PriceMicros, paymentDetails.Currency, nowTs, nowTs, verifyResponse.Valid, verifyResponse.Valid, paymentDetails.Transaction.Type, verifyResponse.Receipt, verifyResponse.ExpirationDate, verifyResponse.RejectReason, receiptHash, extSubscriptionId, ) } @@ -117,7 +117,7 @@ func (d *DataAccessService) AddMobilePurchase(tx *sql.Tx, userID uint64, payment func (d *DataAccessService) GetLatestBundleForNativeVersion(ctx context.Context, nativeVersion uint64) (*t.MobileAppBundleStats, error) { var bundle t.MobileAppBundleStats - err := d.userReader.Get(&bundle, ` + err := d.userReader.GetContext(ctx, &bundle, ` WITH latest_native AS ( SELECT max(min_native_version) as max_native_version @@ -152,7 +152,7 @@ func (d *DataAccessService) GetLatestBundleForNativeVersion(ctx context.Context, } func (d *DataAccessService) IncrementBundleDeliveryCount(ctx context.Context, bundleVersion uint64) error { - _, err := d.userWriter.Exec("UPDATE mobile_app_bundles SET delivered_count = COALESCE(delivered_count, 0) + 1 WHERE bundle_version = $1", bundleVersion) + _, err := d.userWriter.ExecContext(ctx, "UPDATE mobile_app_bundles SET delivered_count = COALESCE(delivered_count, 0) + 1 WHERE bundle_version = $1", bundleVersion) return err } @@ -209,7 +209,7 @@ func (d *DataAccessService) GetValidatorDashboardMobileWidget(ctx context.Contex // RPL eg.Go(func() error { - rpNetworkStats, err := d.internal_rp_network_stats() + rpNetworkStats, err := d.getInternalRpNetworkStats(ctx) if err != nil { return fmt.Errorf("error retrieving rocketpool network stats: %w", err) } @@ -347,9 +347,9 @@ func (d *DataAccessService) GetValidatorDashboardMobileWidget(ctx context.Contex return &data, nil } -func (d *DataAccessService) internal_rp_network_stats() (*t.RPNetworkStats, error) { +func (d *DataAccessService) getInternalRpNetworkStats(ctx context.Context) (*t.RPNetworkStats, error) { var networkStats t.RPNetworkStats - err := d.alloyReader.Get(&networkStats, ` + err := d.alloyReader.GetContext(ctx, &networkStats, ` SELECT EXTRACT(EPOCH FROM claim_interval_time) / 3600 AS claim_interval_hours, node_operator_rewards, diff --git a/backend/pkg/api/data_access/notifications.go b/backend/pkg/api/data_access/notifications.go index 028d325da..5c9adc406 100644 --- a/backend/pkg/api/data_access/notifications.go +++ b/backend/pkg/api/data_access/notifications.go @@ -129,7 +129,7 @@ func (d *DataAccessService) GetNotificationOverview(ctx context.Context, userId }) // most notified groups - latestSlot, err := d.GetLatestSlot() + latestSlot, err := d.GetLatestSlot(ctx) if err != nil { return nil, err } diff --git a/backend/pkg/api/data_access/vdb_management.go b/backend/pkg/api/data_access/vdb_management.go index d152d6cac..91d15a24b 100644 --- a/backend/pkg/api/data_access/vdb_management.go +++ b/backend/pkg/api/data_access/vdb_management.go @@ -168,7 +168,7 @@ func (d *DataAccessService) GetValidatorDashboardName(ctx context.Context, dashb } // param validators: slice of validator public keys or indices -func (d *DataAccessService) GetValidatorsFromSlices(indices []t.VDBValidator, publicKeys []string) ([]t.VDBValidator, error) { +func (d *DataAccessService) GetValidatorsFromSlices(ctx context.Context, indices []t.VDBValidator, publicKeys []string) ([]t.VDBValidator, error) { if len(indices) == 0 && len(publicKeys) == 0 { return []t.VDBValidator{}, nil } diff --git a/backend/pkg/api/handlers/auth.go b/backend/pkg/api/handlers/auth.go index 3f3c2f3b3..d4e0ba44b 100644 --- a/backend/pkg/api/handlers/auth.go +++ b/backend/pkg/api/handlers/auth.go @@ -528,7 +528,7 @@ func (h *HandlerService) InternalPostMobileAuthorize(w http.ResponseWriter, r *h } // check if oauth app exists to validate whether redirect uri is valid - appInfo, err := h.daService.GetAppDataFromRedirectUri(req.RedirectURI) + appInfo, err := h.daService.GetAppDataFromRedirectUri(r.Context(), req.RedirectURI) if err != nil { callback := req.RedirectURI + "?error=invalid_request&error_description=missing_redirect_uri" + state http.Redirect(w, r, callback, http.StatusSeeOther) @@ -545,7 +545,7 @@ func (h *HandlerService) InternalPostMobileAuthorize(w http.ResponseWriter, r *h session := h.scs.Token(r.Context()) sanitizedDeviceName := html.EscapeString(clientName) - err = h.daService.AddUserDevice(userInfo.Id, utils.HashAndEncode(session+session), clientID, sanitizedDeviceName, appInfo.ID) + err = h.daService.AddUserDevice(r.Context(), userInfo.Id, utils.HashAndEncode(session+session), clientID, sanitizedDeviceName, appInfo.ID) if err != nil { log.Warnf("Error adding user device: %v", err) callback := req.RedirectURI + "?error=invalid_request&error_description=server_error" + state @@ -608,7 +608,7 @@ func (h *HandlerService) InternalPostMobileEquivalentExchange(w http.ResponseWri // invalidate old refresh token and replace with hashed session id sanitizedDeviceName := html.EscapeString(req.DeviceName) - err = h.daService.MigrateMobileSession(refreshTokenHashed, utils.HashAndEncode(session+session), req.DeviceID, sanitizedDeviceName) // salted with session + err = h.daService.MigrateMobileSession(r.Context(), refreshTokenHashed, utils.HashAndEncode(session+session), req.DeviceID, sanitizedDeviceName) // salted with session if err != nil { handleErr(w, r, err) return @@ -649,7 +649,7 @@ func (h *HandlerService) InternalPostUsersMeNotificationSettingsPairedDevicesTok return } - err = h.daService.AddMobileNotificationToken(user.Id, deviceID, req.Token) + err = h.daService.AddMobileNotificationToken(r.Context(), user.Id, deviceID, req.Token) if err != nil { handleErr(w, r, err) return @@ -689,7 +689,7 @@ func (h *HandlerService) InternalHandleMobilePurchase(w http.ResponseWriter, r * return } - subscriptionCount, err := h.daService.GetAppSubscriptionCount(user.Id) + subscriptionCount, err := h.daService.GetAppSubscriptionCount(r.Context(), user.Id) if err != nil { handleErr(w, r, err) return @@ -720,7 +720,7 @@ func (h *HandlerService) InternalHandleMobilePurchase(w http.ResponseWriter, r * } } - err = h.daService.AddMobilePurchase(nil, user.Id, req, validationResult, "") + err = h.daService.AddMobilePurchase(r.Context(), nil, user.Id, req, validationResult, "") if err != nil { handleErr(w, r, err) return diff --git a/backend/pkg/api/handlers/backward_compat.go b/backend/pkg/api/handlers/backward_compat.go index e9d823c69..8b8e69442 100644 --- a/backend/pkg/api/handlers/backward_compat.go +++ b/backend/pkg/api/handlers/backward_compat.go @@ -43,7 +43,7 @@ func (h *HandlerService) getTokenByRefresh(r *http.Request, refreshToken string) log.Infof("refresh token: %v, claims: %v, hashed refresh: %v", refreshToken, unsafeClaims, refreshTokenHashed) // confirm all claims via db lookup and refreshtoken check - userID, err := h.daService.GetUserIdByRefreshToken(unsafeClaims.UserID, unsafeClaims.AppID, unsafeClaims.DeviceID, refreshTokenHashed) + userID, err := h.daService.GetUserIdByRefreshToken(r.Context(), unsafeClaims.UserID, unsafeClaims.AppID, unsafeClaims.DeviceID, refreshTokenHashed) if err != nil { if err == sql.ErrNoRows { return 0, "", dataaccess.ErrNotFound diff --git a/backend/pkg/api/handlers/handler_service.go b/backend/pkg/api/handlers/handler_service.go index 71a43e9ad..fd8c5734e 100644 --- a/backend/pkg/api/handlers/handler_service.go +++ b/backend/pkg/api/handlers/handler_service.go @@ -119,7 +119,7 @@ func (h *HandlerService) getDashboardId(ctx context.Context, dashboardIdParam in } return &types.VDBId{Id: types.VDBIdPrimary(dashboardInfo.DashboardId), Validators: nil, AggregateGroups: !dashboardInfo.ShareSettings.ShareGroups}, nil case validatorSet: - validators, err := h.daService.GetValidatorsFromSlices(dashboardId.Indexes, dashboardId.PublicKeys) + validators, err := h.daService.GetValidatorsFromSlices(ctx, dashboardId.Indexes, dashboardId.PublicKeys) if err != nil { return nil, err } diff --git a/backend/pkg/api/handlers/input_validation.go b/backend/pkg/api/handlers/input_validation.go index eca791aa8..3545b0c63 100644 --- a/backend/pkg/api/handlers/input_validation.go +++ b/backend/pkg/api/handlers/input_validation.go @@ -269,10 +269,11 @@ func (h *HandlerService) validateBlockRequest(r *http.Request, paramName string) switch paramValue := mux.Vars(r)[paramName]; paramValue { // possibly add other values like "genesis", "finalized", hardforks etc. later case "latest": + ctx := r.Context() if paramName == "block" { - value, err = h.daService.GetLatestBlock() + value, err = h.daService.GetLatestBlock(ctx) } else if paramName == "slot" { - value, err = h.daService.GetLatestSlot() + value, err = h.daService.GetLatestSlot(ctx) } if err != nil { return 0, 0, err diff --git a/backend/pkg/api/handlers/internal.go b/backend/pkg/api/handlers/internal.go index 3093352f9..83bb4faf2 100644 --- a/backend/pkg/api/handlers/internal.go +++ b/backend/pkg/api/handlers/internal.go @@ -44,19 +44,20 @@ func (h *HandlerService) InternalGetRatelimitWeights(w http.ResponseWriter, r *h // Latest State func (h *HandlerService) InternalGetLatestState(w http.ResponseWriter, r *http.Request) { - latestSlot, err := h.daService.GetLatestSlot() + ctx := r.Context() + latestSlot, err := h.daService.GetLatestSlot(ctx) if err != nil { handleErr(w, r, err) return } - finalizedEpoch, err := h.daService.GetLatestFinalizedEpoch() + finalizedEpoch, err := h.daService.GetLatestFinalizedEpoch(ctx) if err != nil { handleErr(w, r, err) return } - exchangeRates, err := h.daService.GetLatestExchangeRates() + exchangeRates, err := h.daService.GetLatestExchangeRates(ctx) if err != nil { handleErr(w, r, err) return diff --git a/backend/pkg/api/handlers/public.go b/backend/pkg/api/handlers/public.go index b35b5ec5f..5a6473f0e 100644 --- a/backend/pkg/api/handlers/public.go +++ b/backend/pkg/api/handlers/public.go @@ -589,7 +589,7 @@ func (h *HandlerService) PublicPostValidatorDashboardValidators(w http.ResponseW handleErr(w, r, v) return } - validators, err := h.getDataAccessor(r).GetValidatorsFromSlices(indices, pubkeys) + validators, err := h.getDataAccessor(r).GetValidatorsFromSlices(ctx, indices, pubkeys) if err != nil { handleErr(w, r, err) return @@ -637,7 +637,7 @@ func (h *HandlerService) PublicPostValidatorDashboardValidators(w http.ResponseW // PublicGetValidatorDashboardValidators godoc // -// @Description Get a list of groups in a specified validator dashboard. +// @Description Get a list of validators in a specified validator dashboard. // @Tags Validator Dashboard // @Produce json // @Param dashboard_id path string true "The ID of the dashboard." @@ -647,7 +647,7 @@ func (h *HandlerService) PublicPostValidatorDashboardValidators(w http.ResponseW // @Param search query string false "Search for Address, ENS." // @Success 200 {object} types.GetValidatorDashboardValidatorsResponse // @Failure 400 {object} types.ApiErrorResponse -// @Router /validator-dashboards/{dashboard_id}/groups [get] +// @Router /validator-dashboards/{dashboard_id}/validators [get] func (h *HandlerService) PublicGetValidatorDashboardValidators(w http.ResponseWriter, r *http.Request) { var v validationError dashboardId, err := h.handleDashboardId(r.Context(), mux.Vars(r)["dashboard_id"]) @@ -703,12 +703,13 @@ func (h *HandlerService) PublicDeleteValidatorDashboardValidators(w http.Respons handleErr(w, r, v) return } - validators, err := h.getDataAccessor(r).GetValidatorsFromSlices(indices, publicKeys) + ctx := r.Context() + validators, err := h.getDataAccessor(r).GetValidatorsFromSlices(ctx, indices, publicKeys) if err != nil { handleErr(w, r, err) return } - err = h.getDataAccessor(r).RemoveValidatorDashboardValidators(r.Context(), dashboardId, validators) + err = h.getDataAccessor(r).RemoveValidatorDashboardValidators(ctx, dashboardId, validators) if err != nil { handleErr(w, r, err) return From ff0a4c8f3c25b22068637e52a7494e7ab242fa96 Mon Sep 17 00:00:00 2001 From: Lucca Dukic <109136188+LuccaBitfly@users.noreply.github.com> Date: Thu, 24 Oct 2024 10:29:58 +0200 Subject: [PATCH 109/124] refactor: move context keys to types pkg See: BEDS-868 --- backend/pkg/api/handlers/auth.go | 7 +------ backend/pkg/api/handlers/handler_service.go | 4 ++-- backend/pkg/api/handlers/middlewares.go | 14 ++++---------- backend/pkg/api/types/data_access.go | 8 ++++++++ 4 files changed, 15 insertions(+), 18 deletions(-) diff --git a/backend/pkg/api/handlers/auth.go b/backend/pkg/api/handlers/auth.go index d4e0ba44b..357a47cb5 100644 --- a/backend/pkg/api/handlers/auth.go +++ b/backend/pkg/api/handlers/auth.go @@ -34,11 +34,6 @@ const authConfirmEmailRateLimit = time.Minute * 2 const authResetEmailRateLimit = time.Minute * 2 const authEmailExpireTime = time.Minute * 30 -type ctxKey string - -const ctxUserIdKey ctxKey = "user_id" -const ctxIsMockedKey ctxKey = "is_mocked" - var errBadCredentials = newUnauthorizedErr("invalid email or password") func (h *HandlerService) getUserBySession(r *http.Request) (types.UserCredentialInfo, error) { @@ -203,7 +198,7 @@ func (h *HandlerService) GetUserIdByApiKey(r *http.Request) (uint64, error) { // if this is used, user ID should've been stored in context (by GetUserIdStoreMiddleware) func GetUserIdByContext(r *http.Request) (uint64, error) { - userId, ok := r.Context().Value(ctxUserIdKey).(uint64) + userId, ok := r.Context().Value(types.CtxUserIdKey).(uint64) if !ok { return 0, newUnauthorizedErr("user not authenticated") } diff --git a/backend/pkg/api/handlers/handler_service.go b/backend/pkg/api/handlers/handler_service.go index fd8c5734e..6e76c91db 100644 --- a/backend/pkg/api/handlers/handler_service.go +++ b/backend/pkg/api/handlers/handler_service.go @@ -225,7 +225,7 @@ func getMaxChartAge(aggregation enums.ChartAggregation, perkSeconds types.ChartH } func isUserAdmin(user *types.UserInfo) bool { - if user == nil { + if user == nil { // can happen for guest or shared dashboards return false } return user.UserGroup == types.UserGroupAdmin @@ -542,6 +542,6 @@ func (intOrString) JSONSchema() *jsonschema.Schema { } func isMocked(r *http.Request) bool { - isMocked, ok := r.Context().Value(ctxIsMockedKey).(bool) + isMocked, ok := r.Context().Value(types.CtxIsMockedKey).(bool) return ok && isMocked } diff --git a/backend/pkg/api/handlers/middlewares.go b/backend/pkg/api/handlers/middlewares.go index 54238cc59..d59cb33f4 100644 --- a/backend/pkg/api/handlers/middlewares.go +++ b/backend/pkg/api/handlers/middlewares.go @@ -29,7 +29,7 @@ func StoreUserIdMiddleware(next http.Handler, userIdFunc func(r *http.Request) ( // store user id in context ctx := r.Context() - ctx = context.WithValue(ctx, ctxUserIdKey, userId) + ctx = context.WithValue(ctx, types.CtxUserIdKey, userId) r = r.WithContext(ctx) next.ServeHTTP(w, r) }) @@ -53,7 +53,7 @@ func (h *HandlerService) StoreUserIdByApiKeyMiddleware(next http.Handler) http.H func (h *HandlerService) VDBAuthMiddleware(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { // if mock data is used, no need to check access - if isMockEnabled, ok := r.Context().Value(ctxIsMockedKey).(bool); ok && isMockEnabled { + if isMocked, ok := r.Context().Value(types.CtxIsMockedKey).(bool); ok && isMocked { next.ServeHTTP(w, r) return } @@ -71,12 +71,6 @@ func (h *HandlerService) VDBAuthMiddleware(next http.Handler) http.Handler { handleErr(w, r, err) return } - - // store user id in context - ctx := r.Context() - ctx = context.WithValue(ctx, ctxUserIdKey, userId) - r = r.WithContext(ctx) - dashboardUser, err := h.daService.GetValidatorDashboardUser(r.Context(), types.VDBIdPrimary(dashboardId)) if err != nil { handleErr(w, r, err) @@ -138,7 +132,7 @@ func (h *HandlerService) ManageNotificationsViaApiCheckMiddleware(next http.Hand // middleware check to return if specified dashboard is not archived (and accessible) func (h *HandlerService) VDBArchivedCheckMiddleware(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - if isMockEnabled, ok := r.Context().Value(ctxIsMockedKey).(bool); ok && isMockEnabled { + if isMocked, ok := r.Context().Value(types.CtxIsMockedKey).(bool); ok && isMocked { next.ServeHTTP(w, r) return } @@ -193,7 +187,7 @@ func (h *HandlerService) StoreIsMockedFlagMiddleware(next http.Handler) http.Han } // store isMocked flag in context ctx := r.Context() - ctx = context.WithValue(ctx, ctxIsMockedKey, true) + ctx = context.WithValue(ctx, types.CtxIsMockedKey, true) r = r.WithContext(ctx) next.ServeHTTP(w, r) }) diff --git a/backend/pkg/api/types/data_access.go b/backend/pkg/api/types/data_access.go index 6a2859f6f..00c909d12 100644 --- a/backend/pkg/api/types/data_access.go +++ b/backend/pkg/api/types/data_access.go @@ -322,3 +322,11 @@ type NotificationSettingsDefaultValues struct { GasBelowThreshold decimal.Decimal NetworkParticipationRateThreshold float64 } + +// ------------------------------ + +type CtxKey string + +const CtxUserIdKey CtxKey = "user_id" +const CtxIsMockedKey CtxKey = "is_mocked" +const CtxDashboardIdKey CtxKey = "dashboard_id" From e658294303a8ee595f14846cfb47c750cc69b0d7 Mon Sep 17 00:00:00 2001 From: Lucca Dukic <109136188+LuccaBitfly@users.noreply.github.com> Date: Thu, 24 Oct 2024 10:32:07 +0200 Subject: [PATCH 110/124] feat: store dashboard id in context in archiving middleware - avoids fetching from db twice when calling `handleDashboardId` both in archiving middleware and the actual handler See: BEDS-868 --- backend/pkg/api/handlers/handler_service.go | 4 ++++ backend/pkg/api/handlers/middlewares.go | 7 ++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/backend/pkg/api/handlers/handler_service.go b/backend/pkg/api/handlers/handler_service.go index 6e76c91db..167662579 100644 --- a/backend/pkg/api/handlers/handler_service.go +++ b/backend/pkg/api/handlers/handler_service.go @@ -138,6 +138,10 @@ func (h *HandlerService) getDashboardId(ctx context.Context, dashboardIdParam in // it should be used as the last validation step for all internal dashboard GET-handlers. // Modifying handlers (POST, PUT, DELETE) should only accept primary dashboard ids and just use checkPrimaryDashboardId. func (h *HandlerService) handleDashboardId(ctx context.Context, param string) (*types.VDBId, error) { + //check if dashboard id is stored in context + if dashboardId, ok := ctx.Value(types.CtxDashboardIdKey).(*types.VDBId); ok { + return dashboardId, nil + } // validate dashboard id param dashboardIdParam, err := parseDashboardId(param) if err != nil { diff --git a/backend/pkg/api/handlers/middlewares.go b/backend/pkg/api/handlers/middlewares.go index d59cb33f4..bac5f6822 100644 --- a/backend/pkg/api/handlers/middlewares.go +++ b/backend/pkg/api/handlers/middlewares.go @@ -141,7 +141,12 @@ func (h *HandlerService) VDBArchivedCheckMiddleware(next http.Handler) http.Hand handleErr(w, r, err) return } - if len(dashboardId.Validators) > 0 { + // store dashboard id in context + ctx := r.Context() + ctx = context.WithValue(ctx, types.CtxDashboardIdKey, dashboardId) + r = r.WithContext(ctx) + + if len(dashboardId.Validators) > 0 { // don't check guest dashboards next.ServeHTTP(w, r) return } From 9a1813340d1f61717fba865082ef5d1cf68b0952 Mon Sep 17 00:00:00 2001 From: Lucca Dukic <109136188+LuccaBitfly@users.noreply.github.com> Date: Thu, 24 Oct 2024 10:34:40 +0200 Subject: [PATCH 111/124] feat: implement mock_seed query param - allows for passind a seed when the is_mocked flag is being used, which will cause deterministic mocked data. - useful for some auth sensitive endpoints which rely on fetched data staying the same See: BEDS-868 --- backend/pkg/api/data_access/dummy.go | 11 +++++++++++ backend/pkg/api/handlers/middlewares.go | 15 ++++++++++++++- backend/pkg/api/types/data_access.go | 1 + 3 files changed, 26 insertions(+), 1 deletion(-) diff --git a/backend/pkg/api/data_access/dummy.go b/backend/pkg/api/data_access/dummy.go index 387ea5c0d..89f915e46 100644 --- a/backend/pkg/api/data_access/dummy.go +++ b/backend/pkg/api/data_access/dummy.go @@ -7,8 +7,11 @@ import ( "math/rand/v2" "reflect" "slices" + "sync" "time" + mathrand "math/rand" + "github.com/go-faker/faker/v4" "github.com/go-faker/faker/v4/pkg/interfaces" "github.com/go-faker/faker/v4/pkg/options" @@ -52,8 +55,16 @@ func randomEthDecimal() decimal.Decimal { return decimal } +var mockLock sync.Mutex = sync.Mutex{} + // must pass a pointer to the data func populateWithFakeData(ctx context.Context, a interface{}) error { + if seed, ok := ctx.Value(t.CtxMockSeedKey).(int64); ok { + mockLock.Lock() + defer mockLock.Unlock() + faker.SetRandomSource(mathrand.NewSource(seed)) + } + return faker.FakeData(a, options.WithRandomMapAndSliceMaxSize(10), options.WithRandomFloatBoundaries(interfaces.RandomFloatBoundary{Start: 0, End: 1})) } diff --git a/backend/pkg/api/handlers/middlewares.go b/backend/pkg/api/handlers/middlewares.go index bac5f6822..f24ae39a7 100644 --- a/backend/pkg/api/handlers/middlewares.go +++ b/backend/pkg/api/handlers/middlewares.go @@ -169,7 +169,17 @@ func (h *HandlerService) VDBArchivedCheckMiddleware(next http.Handler) http.Hand // note that mocked data is only returned by handlers that check for it. func (h *HandlerService) StoreIsMockedFlagMiddleware(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - isMocked, _ := strconv.ParseBool(r.URL.Query().Get("is_mocked")) + var v validationError + q := r.URL.Query() + isMocked := v.checkBool(q.Get("is_mocked"), "is_mocked") + var mockSeed int64 + if mockSeedStr := q.Get("mock_seed"); mockSeedStr != "" { + mockSeed = v.checkInt(mockSeedStr, "mock_seed") + } + if v.hasErrors() { + handleErr(w, r, v) + return + } if !isMocked { next.ServeHTTP(w, r) return @@ -193,6 +203,9 @@ func (h *HandlerService) StoreIsMockedFlagMiddleware(next http.Handler) http.Han // store isMocked flag in context ctx := r.Context() ctx = context.WithValue(ctx, types.CtxIsMockedKey, true) + if mockSeed != 0 { + ctx = context.WithValue(ctx, types.CtxMockSeedKey, mockSeed) + } r = r.WithContext(ctx) next.ServeHTTP(w, r) }) diff --git a/backend/pkg/api/types/data_access.go b/backend/pkg/api/types/data_access.go index 00c909d12..8cf95256e 100644 --- a/backend/pkg/api/types/data_access.go +++ b/backend/pkg/api/types/data_access.go @@ -329,4 +329,5 @@ type CtxKey string const CtxUserIdKey CtxKey = "user_id" const CtxIsMockedKey CtxKey = "is_mocked" +const CtxMockSeedKey CtxKey = "mock_seed" const CtxDashboardIdKey CtxKey = "dashboard_id" From 351dc83e7d9bd096963025478944161c414f5254 Mon Sep 17 00:00:00 2001 From: Lucca Dukic <109136188+LuccaBitfly@users.noreply.github.com> Date: Thu, 24 Oct 2024 13:01:10 +0200 Subject: [PATCH 112/124] fix: cap do not disturb timestamp if greater than maxInt32 See: BEDS-856 --- backend/pkg/api/handlers/public.go | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/pkg/api/handlers/public.go b/backend/pkg/api/handlers/public.go index 5a6473f0e..974060972 100644 --- a/backend/pkg/api/handlers/public.go +++ b/backend/pkg/api/handlers/public.go @@ -2215,6 +2215,7 @@ func (h *HandlerService) PublicPutUserNotificationSettingsGeneral(w http.Respons handleErr(w, r, v) return } + req.DoNotDisturbTimestamp = min(req.DoNotDisturbTimestamp, math.MaxInt32) // check premium perks userInfo, err := h.getDataAccessor(r).GetUserInfo(r.Context(), userId) From 103de90d84a29236f2431db89a210181e30f2138 Mon Sep 17 00:00:00 2001 From: Patrick Date: Thu, 24 Oct 2024 13:12:45 +0200 Subject: [PATCH 113/124] fix(evm_indexer): fix parsing flags, add version-flag, serve metrics if set in config (#1033) --- backend/cmd/evm_node_indexer/main.go | 51 ++++++++++++++++++++-------- 1 file changed, 36 insertions(+), 15 deletions(-) diff --git a/backend/cmd/evm_node_indexer/main.go b/backend/cmd/evm_node_indexer/main.go index b431720bc..4975bb3d0 100644 --- a/backend/cmd/evm_node_indexer/main.go +++ b/backend/cmd/evm_node_indexer/main.go @@ -13,6 +13,7 @@ import ( "fmt" "io" "net/http" + "os" "regexp" "strings" "sync/atomic" @@ -24,6 +25,7 @@ import ( "github.com/gobitfly/beaconchain/pkg/commons/db" "github.com/gobitfly/beaconchain/pkg/commons/hexutil" "github.com/gobitfly/beaconchain/pkg/commons/log" + "github.com/gobitfly/beaconchain/pkg/commons/metrics" "github.com/gobitfly/beaconchain/pkg/commons/types" "github.com/gobitfly/beaconchain/pkg/commons/utils" "github.com/gobitfly/beaconchain/pkg/commons/version" @@ -103,22 +105,32 @@ func init() { // main func Run() { + fs := flag.NewFlagSet("fs", flag.ExitOnError) + // read / set parameter - configPath := flag.String("config", "config/default.config.yml", "Path to the config file") - startBlockNumber := flag.Int64("start-block-number", -1, "trigger a REEXPORT, only working in combination with end-block-number, defined block is included, will be the first action done and will quite afterwards, ignore every other action") - endBlockNumber := flag.Int64("end-block-number", -1, "trigger a REEXPORT, only working in combination with start-block-number, defined block is included, will be the first action done and will quite afterwards, ignore every other action") - reorgDepth = flag.Int64("reorg.depth", 32, fmt.Sprintf("lookback to check and handle chain reorgs (MAX %s), you should NEVER reduce this after the first start, otherwise there will be unchecked areas", _formatInt64(MAX_REORG_DEPTH))) - concurrency := flag.Int64("concurrency", 8, "maximum threads used (running on maximum whenever possible)") - nodeRequestsAtOnce := flag.Int64("node-requests-at-once", 16, fmt.Sprintf("bulk size per node = bt = db request (MAX %s)", _formatInt64(MAX_NODE_REQUESTS_AT_ONCE))) - skipHoleCheck := flag.Bool("skip-hole-check", false, "skips the initial check for holes, doesn't go very well with only-hole-check") - onlyHoleCheck := flag.Bool("only-hole-check", false, "just check for holes and quit, can be used for a reexport running simulation to a normal setup, just remove entries in postgres and start with this flag, doesn't go very well with skip-hole-check") - noNewBlocks := flag.Bool("ignore-new-blocks", false, "there are no new blocks, at all") - noNewBlocksThresholdSeconds := flag.Int("fatal-if-no-new-block-for-x-seconds", 600, "will fatal if there is no new block for x seconds (MIN 30), will start throwing errors at 2/3 of the time, will start throwing warnings at 1/3 of the time, doesn't go very well with ignore-new-blocks") - discordWebhookBlockThreshold := flag.Int64("discord-block-threshold", 100000, "every x blocks an update is send to Discord") - discordWebhookReportUrl := flag.String("discord-url", "", "report progress to discord url") - discordWebhookUser := flag.String("discord-user", "", "report progress to discord user") - discordWebhookAddTextFatal := flag.String("discord-fatal-text", "", "this text will be added to the discord message in the case of an fatal") - flag.Parse() + configPath := fs.String("config", "config/default.config.yml", "Path to the config file") + versionFlag := fs.Bool("version", false, "print version and exit") + startBlockNumber := fs.Int64("start-block-number", -1, "trigger a REEXPORT, only working in combination with end-block-number, defined block is included, will be the first action done and will quite afterwards, ignore every other action") + endBlockNumber := fs.Int64("end-block-number", -1, "trigger a REEXPORT, only working in combination with start-block-number, defined block is included, will be the first action done and will quite afterwards, ignore every other action") + reorgDepth = fs.Int64("reorg.depth", 32, fmt.Sprintf("lookback to check and handle chain reorgs (MAX %s), you should NEVER reduce this after the first start, otherwise there will be unchecked areas", _formatInt64(MAX_REORG_DEPTH))) + concurrency := fs.Int64("concurrency", 8, "maximum threads used (running on maximum whenever possible)") + nodeRequestsAtOnce := fs.Int64("node-requests-at-once", 16, fmt.Sprintf("bulk size per node = bt = db request (MAX %s)", _formatInt64(MAX_NODE_REQUESTS_AT_ONCE))) + skipHoleCheck := fs.Bool("skip-hole-check", false, "skips the initial check for holes, doesn't go very well with only-hole-check") + onlyHoleCheck := fs.Bool("only-hole-check", false, "just check for holes and quit, can be used for a reexport running simulation to a normal setup, just remove entries in postgres and start with this flag, doesn't go very well with skip-hole-check") + noNewBlocks := fs.Bool("ignore-new-blocks", false, "there are no new blocks, at all") + noNewBlocksThresholdSeconds := fs.Int("fatal-if-no-new-block-for-x-seconds", 600, "will fatal if there is no new block for x seconds (MIN 30), will start throwing errors at 2/3 of the time, will start throwing warnings at 1/3 of the time, doesn't go very well with ignore-new-blocks") + discordWebhookBlockThreshold := fs.Int64("discord-block-threshold", 100000, "every x blocks an update is send to Discord") + discordWebhookReportUrl := fs.String("discord-url", "", "report progress to discord url") + discordWebhookUser := fs.String("discord-user", "", "report progress to discord user") + discordWebhookAddTextFatal := fs.String("discord-fatal-text", "", "this text will be added to the discord message in the case of an fatal") + err := fs.Parse(os.Args[2:]) + if err != nil { + log.Fatal(err, "error parsing flags", 0) + } + if *versionFlag { + log.Info(version.Version) + return + } // tell the user about all parameter { @@ -161,6 +173,15 @@ func Run() { } else { eth1RpcEndpoint = utils.Config.Eth1GethEndpoint } + + if utils.Config.Metrics.Enabled { + go func() { + log.Infof("serving metrics on %v", utils.Config.Metrics.Address) + if err := metrics.Serve(utils.Config.Metrics.Address, utils.Config.Metrics.Pprof, utils.Config.Metrics.PprofExtra); err != nil { + log.Fatal(err, "error serving metrics", 0) + } + }() + } } // check parameters From 4340a77bf2645063e7ef22063ff3f4d21e284d74 Mon Sep 17 00:00:00 2001 From: remoterami <142154971+remoterami@users.noreply.github.com> Date: Thu, 24 Oct 2024 13:56:31 +0200 Subject: [PATCH 114/124] applied CR feedback --- backend/pkg/api/data_access/general.go | 12 ++-- backend/pkg/api/data_access/vdb_blocks.go | 80 ++++++++++++++--------- 2 files changed, 54 insertions(+), 38 deletions(-) diff --git a/backend/pkg/api/data_access/general.go b/backend/pkg/api/data_access/general.go index d629d52df..9d7ee0d97 100644 --- a/backend/pkg/api/data_access/general.go +++ b/backend/pkg/api/data_access/general.go @@ -74,9 +74,9 @@ func applySortAndPagination(defaultColumns []types.SortColumn, primary types.Sor if cursor.IsReverse() { column.Desc = !column.Desc } - colOrder := column.Column.Asc() + colOrder := column.Column.Asc().NullsFirst() if column.Desc { - colOrder = column.Column.Desc() + colOrder = column.Column.Desc().NullsLast() } queryOrder = append(queryOrder, colOrder) } @@ -89,10 +89,10 @@ func applySortAndPagination(defaultColumns []types.SortColumn, primary types.Sor column := queryOrderColumns[i] var colWhere exp.Expression - // current convention is the psql default (ASC: nulls last, DESC: nulls first) - colWhere = goqu.Or(column.Column.Gt(column.Offset), column.Column.IsNull()) - if column.Desc { - colWhere = column.Column.Lt(column.Offset) + // current convention is opposite of the psql default (ASC: nulls first, DESC: nulls last) + colWhere = goqu.Or(column.Column.Lt(column.Offset), column.Column.IsNull()) + if !column.Desc { + colWhere = column.Column.Gt(column.Offset) if column.Offset == nil { colWhere = goqu.Or(colWhere, column.Column.IsNull()) } diff --git a/backend/pkg/api/data_access/vdb_blocks.go b/backend/pkg/api/data_access/vdb_blocks.go index 00495e265..b8c1ba352 100644 --- a/backend/pkg/api/data_access/vdb_blocks.go +++ b/backend/pkg/api/data_access/vdb_blocks.go @@ -10,6 +10,7 @@ import ( "time" "github.com/doug-martin/goqu/v9" + "github.com/doug-martin/goqu/v9/exp" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/gobitfly/beaconchain/pkg/api/enums" t "github.com/gobitfly/beaconchain/pkg/api/types" @@ -45,7 +46,7 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das searchGroup := regexp.MustCompile(`^[a-zA-Z0-9_\-.\ ]+$`).MatchString(search) searchIndex := regexp.MustCompile(`^[0-9]+$`).MatchString(search) - validators := goqu.T("users_val_dashboards_validators").As("validators") + validators := goqu.T("validators") blocks := goqu.T("blocks") groups := goqu.T("groups") @@ -65,21 +66,22 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das ) if dashboardId.Validators == nil { filteredValidatorsDs = filteredValidatorsDs. - From(validators). + From(goqu.T("users_val_dashboards_validators").As(validators.GetTable())). Where(validators.Col("dashboard_id").Eq(dashboardId.Id)) // apply search filters + searches := []exp.Expression{} if searchIndex { - filteredValidatorsDs = filteredValidatorsDs.Where(validators.Col("validator_index").Eq(search)) + searches = append(searches, validators.Col("validator_index").Eq(search)) } if searchGroup { filteredValidatorsDs = filteredValidatorsDs. InnerJoin(goqu.T("users_val_dashboards_groups").As(groups), goqu.On( validators.Col("group_id").Eq(groups.Col("id")), validators.Col("dashboard_id").Eq(groups.Col("dashboard_id")), - )). - Where( - goqu.L("LOWER(?)", groups.Col("name")).Like(strings.Replace(search, "_", "\\_", -1) + "%"), - ) + )) + searches = append(searches, + goqu.L("LOWER(?)", groups.Col("name")).Like(strings.Replace(strings.ToLower(search), "_", "\\_", -1)+"%"), + ) } if searchPubkey { index, ok := validatorMapping.ValidatorIndices[search] @@ -87,11 +89,15 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das // searched pubkey doesn't exist, don't even need to query anything return make([]t.VDBBlocksTableRow, 0), &t.Paging{}, nil } - - filteredValidatorsDs = filteredValidatorsDs. - Where(validators.Col("validator_index").Eq(index)) + searches = append(searches, + validators.Col("validator_index").Eq(index), + ) + } + if len(searches) > 0 { + filteredValidatorsDs = filteredValidatorsDs.Where(goqu.Or(searches...)) } } else { + validatorList := make([]t.VDBValidator, 0, len(dashboardId.Validators)) for _, validator := range dashboardId.Validators { if searchIndex && fmt.Sprint(validator) != search || searchPubkey && validator != validatorMapping.ValidatorIndices[search] { @@ -101,14 +107,19 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das Validator: validator, Group: t.DefaultGroupId, }) + validatorList = append(validatorList, validator) if searchIndex || searchPubkey { break } } filteredValidatorsDs = filteredValidatorsDs. From( - goqu.L("unnest(?)", pq.Array(filteredValidators)).As("validator_index"), - ).As("validators") // TODO ? + goqu.Dialect("postgres"). + From( + goqu.L("unnest(?::int[])", pq.Array(validatorList)).As("validator_index"), + ). + As(validators.GetTable()), + ) } // ------------------------------------- @@ -144,10 +155,17 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das ) // 2. Selects + groupIdQ := goqu.C("group_id").(exp.Aliaseable) + if dashboardId.Validators != nil { + groupIdQ = exp.NewLiteralExpression("?::int", t.DefaultGroupId) + } + groupId := groupIdQ.As("group_id") + blocksDs = blocksDs. SelectAppend( blocks.Col("epoch"), blocks.Col("slot"), + groupId, blocks.Col("status"), blocks.Col("exec_block_number"), blocks.Col("graffiti_text"), @@ -156,16 +174,7 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das goqu.L("cp.cl_attestations_reward / 1e9 + cp.cl_sync_aggregate_reward / 1e9 + cp.cl_slashing_inclusion_reward / 1e9").As("cl_reward"), ) - groupId := validators.Col("group_id") - if dashboardId.Validators != nil { - groupId = goqu.V(t.DefaultGroupId).As("group_id").GetAs() - } - blocksDs = blocksDs.SelectAppend(groupId) - - // 3. Limit - blocksDs = blocksDs.Limit(uint(limit + 1)) - - // 4. Sorting and pagination + // 3. Sorting and pagination defaultColumns := []t.SortColumn{ {Column: enums.VDBBlocksColumns.Slot.ToExpr(), Desc: true, Offset: currentCursor.Slot}, } @@ -185,18 +194,23 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das } order, directions := applySortAndPagination(defaultColumns, t.SortColumn{Column: colSort.Column.ToExpr(), Desc: colSort.Desc, Offset: offset}, currentCursor.GenericCursor) - blocksDs = goqu.From(blocksDs). // encapsulate so we can use selected fields - Order(order...) + blocksDs = goqu.Dialect("postgres").From(goqu.T("past_blocks_cte")). + With("past_blocks_cte", blocksDs). // encapsulate so we can use selected fields + Order(order...) if directions != nil { blocksDs = blocksDs.Where(directions) } + // 4. Limit + blocksDs = blocksDs.Limit(uint(limit + 1)) + // 5. Gather and supply scheduled blocks to let db do the sorting etc latestSlot := cache.LatestSlot.Get() onlyPrimarySort := colSort.Column == enums.VDBBlockSlot - if !(onlyPrimarySort || colSort.Column == enums.VDBBlockBlock) || !currentCursor.IsValid() || - currentCursor.Slot > latestSlot+1 && currentCursor.Reverse != colSort.Desc || - currentCursor.Slot < latestSlot+1 && currentCursor.Reverse == colSort.Desc { + if !(onlyPrimarySort || colSort.Column == enums.VDBBlockBlock) || + !currentCursor.IsValid() || + currentCursor.Slot > latestSlot+1 || + colSort.Desc == currentCursor.Reverse { dutiesInfo, err := d.services.GetCurrentDutiesInfo() if err == nil { if dashboardId.Validators == nil { @@ -213,11 +227,12 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das return make([]t.VDBBlocksTableRow, 0), &t.Paging{}, nil } - validatorSet := make(map[t.VDBValidator]bool) + validatorSet := make(map[t.VDBValidator]uint64) for _, v := range filteredValidators { - validatorSet[v.Validator] = true + validatorSet[v.Validator] = v.Group } var scheduledProposers []t.VDBValidator + var scheduledGroups []uint64 var scheduledEpochs []uint64 var scheduledSlots []uint64 // don't need if requested slots are in the past @@ -231,25 +246,26 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das continue } scheduledProposers = append(scheduledProposers, dutiesInfo.PropAssignmentsForSlot[slot]) + scheduledGroups = append(scheduledGroups, validatorSet[vali]) scheduledEpochs = append(scheduledEpochs, slot/utils.Config.Chain.ClConfig.SlotsPerEpoch) scheduledSlots = append(scheduledSlots, slot) } scheduledDs := goqu.Dialect("postgres"). From( - goqu.L("unnest(?::int[], ?::int[], ?::int[]) AS prov(validator_index, epoch, slot)", pq.Array(scheduledProposers), pq.Array(scheduledEpochs), pq.Array(scheduledSlots)), + goqu.L("unnest(?::int[], ?::int[], ?::int[], ?::int[]) AS prov(validator_index, group_id, epoch, slot)", pq.Array(scheduledProposers), pq.Array(scheduledGroups), pq.Array(scheduledEpochs), pq.Array(scheduledSlots)), ). Select( goqu.C("validator_index"), goqu.C("epoch"), goqu.C("slot"), + groupId, goqu.V("0").As("status"), goqu.V(nil).As("exec_block_number"), + goqu.V(nil).As("graffiti_text"), goqu.V(nil).As("fee_recipient"), goqu.V(nil).As("el_reward"), goqu.V(nil).As("cl_reward"), - goqu.V(nil).As("graffiti_text"), - goqu.V(t.DefaultGroupId).As("group_id"), ). As("scheduled_blocks") From a5bc4281350a1e46372e76cd972cc57bef4ebc1c Mon Sep 17 00:00:00 2001 From: Lucca Dukic <109136188+LuccaBitfly@users.noreply.github.com> Date: Wed, 23 Oct 2024 13:18:21 +0200 Subject: [PATCH 115/124] fix: implement proper auth check for editing paired devices See: BEDS-863 --- backend/pkg/api/data_access/dummy.go | 8 +++- backend/pkg/api/data_access/notifications.go | 45 +++++++++++++------- backend/pkg/api/handlers/public.go | 23 ++++++++-- 3 files changed, 56 insertions(+), 20 deletions(-) diff --git a/backend/pkg/api/data_access/dummy.go b/backend/pkg/api/data_access/dummy.go index 89f915e46..f1063084a 100644 --- a/backend/pkg/api/data_access/dummy.go +++ b/backend/pkg/api/data_access/dummy.go @@ -582,10 +582,10 @@ func (d *DummyService) UpdateNotificationSettingsGeneral(ctx context.Context, us func (d *DummyService) UpdateNotificationSettingsNetworks(ctx context.Context, userId uint64, chainId uint64, settings t.NotificationSettingsNetwork) error { return nil } -func (d *DummyService) UpdateNotificationSettingsPairedDevice(ctx context.Context, userId uint64, pairedDeviceId uint64, name string, IsNotificationsEnabled bool) error { +func (d *DummyService) UpdateNotificationSettingsPairedDevice(ctx context.Context, pairedDeviceId uint64, name string, IsNotificationsEnabled bool) error { return nil } -func (d *DummyService) DeleteNotificationSettingsPairedDevice(ctx context.Context, userId uint64, pairedDeviceId uint64) error { +func (d *DummyService) DeleteNotificationSettingsPairedDevice(ctx context.Context, pairedDeviceId uint64) error { return nil } @@ -796,3 +796,7 @@ func (d *DummyService) QueueTestPushNotification(ctx context.Context, userId uin func (d *DummyService) QueueTestWebhookNotification(ctx context.Context, userId uint64, webhookUrl string, isDiscordWebhook bool) error { return nil } + +func (d *DummyService) GetPairedDeviceUserId(ctx context.Context, pairedDeviceId uint64) (uint64, error) { + return getDummyData[uint64](ctx) +} diff --git a/backend/pkg/api/data_access/notifications.go b/backend/pkg/api/data_access/notifications.go index 5c9adc406..39b10eec9 100644 --- a/backend/pkg/api/data_access/notifications.go +++ b/backend/pkg/api/data_access/notifications.go @@ -51,8 +51,9 @@ type NotificationsRepository interface { GetNotificationSettingsDefaultValues(ctx context.Context) (*t.NotificationSettingsDefaultValues, error) UpdateNotificationSettingsGeneral(ctx context.Context, userId uint64, settings t.NotificationSettingsGeneral) error UpdateNotificationSettingsNetworks(ctx context.Context, userId uint64, chainId uint64, settings t.NotificationSettingsNetwork) error - UpdateNotificationSettingsPairedDevice(ctx context.Context, userId uint64, pairedDeviceId uint64, name string, IsNotificationsEnabled bool) error - DeleteNotificationSettingsPairedDevice(ctx context.Context, userId uint64, pairedDeviceId uint64) error + GetPairedDeviceUserId(ctx context.Context, pairedDeviceId uint64) (uint64, error) + UpdateNotificationSettingsPairedDevice(ctx context.Context, pairedDeviceId uint64, name string, IsNotificationsEnabled bool) error + DeleteNotificationSettingsPairedDevice(ctx context.Context, pairedDeviceId uint64) error UpdateNotificationSettingsClients(ctx context.Context, userId uint64, clientId uint64, IsSubscribed bool) (*t.NotificationSettingsClient, error) GetNotificationSettingsDashboards(ctx context.Context, userId uint64, cursor string, colSort t.Sort[enums.NotificationSettingsDashboardColumn], search string, limit uint64) ([]t.NotificationSettingsDashboardsTableRow, *t.Paging, error) UpdateNotificationSettingsValidatorDashboard(ctx context.Context, userId uint64, dashboardId t.VDBIdPrimary, groupId uint64, settings t.NotificationSettingsValidatorDashboard) error @@ -1637,47 +1638,61 @@ func (d *DataAccessService) UpdateNotificationSettingsNetworks(ctx context.Conte } return nil } -func (d *DataAccessService) UpdateNotificationSettingsPairedDevice(ctx context.Context, userId uint64, pairedDeviceId uint64, name string, IsNotificationsEnabled bool) error { + +func (d *DataAccessService) GetPairedDeviceUserId(ctx context.Context, pairedDeviceId uint64) (uint64, error) { + var userId uint64 + err := d.userReader.GetContext(context.Background(), &userId, ` + SELECT user_id + FROM users_devices + WHERE id = $1`, pairedDeviceId) + if err != nil { + if err == sql.ErrNoRows { + return 0, fmt.Errorf("%w, paired device with id %v not found", ErrNotFound, pairedDeviceId) + } + return 0, err + } + return userId, nil +} + +func (d *DataAccessService) UpdateNotificationSettingsPairedDevice(ctx context.Context, pairedDeviceId uint64, name string, IsNotificationsEnabled bool) error { result, err := d.userWriter.ExecContext(ctx, ` UPDATE users_devices SET device_name = $1, notify_enabled = $2 - WHERE user_id = $3 AND id = $4`, - name, IsNotificationsEnabled, userId, pairedDeviceId) + WHERE id = $3`, + name, IsNotificationsEnabled, pairedDeviceId) if err != nil { return err } - - // TODO: This can be deleted when the API layer has an improved check for the device id rowsAffected, err := result.RowsAffected() if err != nil { return err } if rowsAffected == 0 { - return fmt.Errorf("device with id %v to update notification settings not found", pairedDeviceId) + return fmt.Errorf("%w, paired device with id %v not found", ErrNotFound, pairedDeviceId) } return nil } -func (d *DataAccessService) DeleteNotificationSettingsPairedDevice(ctx context.Context, userId uint64, pairedDeviceId uint64) error { + +func (d *DataAccessService) DeleteNotificationSettingsPairedDevice(ctx context.Context, pairedDeviceId uint64) error { result, err := d.userWriter.ExecContext(ctx, ` - DELETE FROM users_devices - WHERE user_id = $1 AND id = $2`, - userId, pairedDeviceId) + DELETE FROM users_devices + WHERE id = $1`, + pairedDeviceId) if err != nil { return err } - - // TODO: This can be deleted when the API layer has an improved check for the device id rowsAffected, err := result.RowsAffected() if err != nil { return err } if rowsAffected == 0 { - return fmt.Errorf("device with id %v to delete not found", pairedDeviceId) + return fmt.Errorf("%w, paired device with id %v not found", ErrNotFound, pairedDeviceId) } return nil } + func (d *DataAccessService) UpdateNotificationSettingsClients(ctx context.Context, userId uint64, clientId uint64, IsSubscribed bool) (*t.NotificationSettingsClient, error) { result := &t.NotificationSettingsClient{Id: clientId, IsSubscribed: IsSubscribed} diff --git a/backend/pkg/api/handlers/public.go b/backend/pkg/api/handlers/public.go index 974060972..72813b142 100644 --- a/backend/pkg/api/handlers/public.go +++ b/backend/pkg/api/handlers/public.go @@ -2353,7 +2353,16 @@ func (h *HandlerService) PublicPutUserNotificationSettingsPairedDevices(w http.R handleErr(w, r, v) return } - err = h.getDataAccessor(r).UpdateNotificationSettingsPairedDevice(r.Context(), userId, pairedDeviceId, name, req.IsNotificationsEnabled) + pairedDeviceUserId, err := h.getDataAccessor(r).GetPairedDeviceUserId(r.Context(), pairedDeviceId) + if err != nil { + handleErr(w, r, err) + return + } + if userId != pairedDeviceUserId { + returnNotFound(w, r, fmt.Errorf("not found: paired device with id %d not found", pairedDeviceId)) // return 404 to not leak information + return + } + err = h.getDataAccessor(r).UpdateNotificationSettingsPairedDevice(r.Context(), pairedDeviceId, name, req.IsNotificationsEnabled) if err != nil { handleErr(w, r, err) return @@ -2387,13 +2396,21 @@ func (h *HandlerService) PublicDeleteUserNotificationSettingsPairedDevices(w htt handleErr(w, r, err) return } - // TODO use a better way to validate the paired device id pairedDeviceId := v.checkUint(mux.Vars(r)["paired_device_id"], "paired_device_id") if v.hasErrors() { handleErr(w, r, v) return } - err = h.getDataAccessor(r).DeleteNotificationSettingsPairedDevice(r.Context(), userId, pairedDeviceId) + pairedDeviceUserId, err := h.getDataAccessor(r).GetPairedDeviceUserId(r.Context(), pairedDeviceId) + if err != nil { + handleErr(w, r, err) + return + } + if userId != pairedDeviceUserId { + returnNotFound(w, r, fmt.Errorf("not found: paired device with id %d not found", pairedDeviceId)) // return 404 to not leak information + return + } + err = h.getDataAccessor(r).DeleteNotificationSettingsPairedDevice(r.Context(), pairedDeviceId) if err != nil { handleErr(w, r, err) return From 21d9cde3bf64f0ba5dd426a3cffb3aba5e321e96 Mon Sep 17 00:00:00 2001 From: benji-bitfly Date: Thu, 24 Oct 2024 14:49:20 +0200 Subject: [PATCH 116/124] feat(NotificationsOverview): add `empty fields` state See: BEDS-860 --- .../notifications/NotificationsOverview.vue | 54 ++++++++++++------- 1 file changed, 34 insertions(+), 20 deletions(-) diff --git a/frontend/components/notifications/NotificationsOverview.vue b/frontend/components/notifications/NotificationsOverview.vue index 1db754293..f6a261cea 100644 --- a/frontend/components/notifications/NotificationsOverview.vue +++ b/frontend/components/notifications/NotificationsOverview.vue @@ -100,31 +100,41 @@ const emit = defineEmits<{ {{ $t('notifications.overview.headers.most_notifications_30d') }}
- - {{ $t('notifications.overview.headers.validator_groups') }} - -
    -
  1. - - - {{ group }} - -
    2. -
- +
- {{ $t('notifications.overview.headers.account_groups') }} + {{ $t('notifications.overview.headers.validator_groups') }} -
    -
  1. - +
      +
    1. + - {{ group }} + {{ index + 1 }}. {{ group || '-' }}
    +
+ +
+ + {{ $t('notifications.overview.headers.account_groups') }} + +
    +
  1. + + + {{ index + 1 }}. {{ group || '-' }} + +
    2. +
+
@@ -203,7 +213,11 @@ a:hover { } .lists-container { display: flex; - gap: 20px; + gap: 1.25rem; +} +.lists-container-column { + flex: 1; + min-width: 0; } .icon-list { min-width: 0; From 5af248b504536e123ed38c864ece62260136ec2c Mon Sep 17 00:00:00 2001 From: benji-bitfly Date: Thu, 24 Oct 2024 14:57:30 +0200 Subject: [PATCH 117/124] refactor(NotificationsOverview): convert to `rem` for `a11y` --- .../notifications/NotificationsOverview.vue | 27 +++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/frontend/components/notifications/NotificationsOverview.vue b/frontend/components/notifications/NotificationsOverview.vue index f6a261cea..6de7c17b7 100644 --- a/frontend/components/notifications/NotificationsOverview.vue +++ b/frontend/components/notifications/NotificationsOverview.vue @@ -159,25 +159,25 @@ const emit = defineEmits<{ .container { @include main.container; - padding: 17px 20px; + padding: 1.0625rem 1.25rem; position: relative; } .info-section, .action-section { display: flex; align-items: center; justify-content: center; - gap: 10px; + gap: .625rem; } .icon { - font-size: 24px; + font-size: 1.5rem; } .text { - font-size: 18px; + font-size: 1.125rem; font-weight: 500; } .list-item { display: flex; - gap: 10px; + gap: .625rem; .list-text { @include utils.truncate-text; } @@ -201,12 +201,12 @@ const emit = defineEmits<{ .box-item { display: flex; flex-direction: column; - gap: 10px; + gap: .625rem; } .inline-items { display: flex; align-items: center; - gap: 10px; + gap: .625rem; } a:hover { color: var(--light-blue); @@ -221,15 +221,14 @@ a:hover { } .icon-list { min-width: 0; - list-style-type: none; padding: 0; margin: 0; display: flex; flex-direction: column; - gap: 10px; + gap: .625rem; } .icon { - font-size: 16px; + font-size: 1rem; } .inline-link, .gem { @@ -238,21 +237,21 @@ a:hover { .premium-invitation { display: flex; align-items: center; - gap: 5px; /* Adjust the gap as needed */ + gap: .3125rem; } .push-invitation { display: flex; align-items: center; - gap: 5px; /* Adjust the gap as needed */ + gap: .3125rem; flex-wrap: wrap; } @media (max-width: 600px) { .box { flex-direction: row; - gap: 20px; + gap: 1.25rem; } .box-item { - min-width: 250px; /* Adjust based on content width */ + min-width: 15.625rem; } } From b39b06c91f5de6ca79871618ebe85cf807893da5 Mon Sep 17 00:00:00 2001 From: peter <1674920+peterbitfly@users.noreply.github.com> Date: Thu, 17 Oct 2024 13:33:08 +0000 Subject: [PATCH 118/124] feat(BcFormatPercent): change threshold for efficiency See: BEDS-622 Co-authored-by: marcel-bitfly <174338434+marcel-bitfly@users.noreply.github.com> --- frontend/components/bc/format/FormatPercent.vue | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frontend/components/bc/format/FormatPercent.vue b/frontend/components/bc/format/FormatPercent.vue index 4f9b41036..c6dfbc025 100644 --- a/frontend/components/bc/format/FormatPercent.vue +++ b/frontend/components/bc/format/FormatPercent.vue @@ -52,7 +52,8 @@ const data = computed(() => { } label = formatPercent(percent, config) if (props.comparePercent !== undefined) { - if (Math.abs(props.comparePercent - percent) <= 0.5) { + const thresholdToDifferenciateUnderperformerAndOverperformer = 0.25 + if (Math.abs(props.comparePercent - percent) <= thresholdToDifferenciateUnderperformerAndOverperformer) { className = 'text-equal' leadingIcon = faArrowsLeftRight compareResult = 'equal' From fcdea3682269fbbd98b4e0a2849695438c1478e0 Mon Sep 17 00:00:00 2001 From: remoterami <142154971+remoterami@users.noreply.github.com> Date: Mon, 28 Oct 2024 10:51:47 +0100 Subject: [PATCH 119/124] CR feedback --- backend/pkg/api/data_access/vdb_blocks.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/pkg/api/data_access/vdb_blocks.go b/backend/pkg/api/data_access/vdb_blocks.go index b8c1ba352..47bbd651f 100644 --- a/backend/pkg/api/data_access/vdb_blocks.go +++ b/backend/pkg/api/data_access/vdb_blocks.go @@ -46,7 +46,7 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das searchGroup := regexp.MustCompile(`^[a-zA-Z0-9_\-.\ ]+$`).MatchString(search) searchIndex := regexp.MustCompile(`^[0-9]+$`).MatchString(search) - validators := goqu.T("validators") + validators := goqu.T("validators") // could adapt data type to make handling as table/alias less confusing blocks := goqu.T("blocks") groups := goqu.T("groups") @@ -215,6 +215,8 @@ func (d *DataAccessService) GetValidatorDashboardBlocks(ctx context.Context, das if err == nil { if dashboardId.Validators == nil { // fetch filtered validators if not done yet + filteredValidatorsDs = filteredValidatorsDs. + SelectAppend(groupIdQ) validatorsQuery, validatorsArgs, err := filteredValidatorsDs.Prepared(true).ToSQL() if err != nil { return nil, nil, err From 7192f4a1aa31302ccaf15aacb34a8ed1dc899202 Mon Sep 17 00:00:00 2001 From: benji-bitfly Date: Mon, 28 Oct 2024 13:13:12 +0100 Subject: [PATCH 120/124] feat(DashboardTableValidators): change representation of `validator indexes` In `DashboardTableSummary` more then `3 indexes` should not be shown. See: BEDS-649 --- frontend/.vscode/settings.json | 1 + .../dashboard/table/DashboardTableValidators.vue | 15 +++++++++------ 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/frontend/.vscode/settings.json b/frontend/.vscode/settings.json index b162c363d..3657d47ec 100644 --- a/frontend/.vscode/settings.json +++ b/frontend/.vscode/settings.json @@ -7,6 +7,7 @@ "BcToggle", "DashboardChartSummaryChartFilter", "DashboardGroupManagementModal", + "DashboardTableValidators", "DashboardValidatorManagmentModal", "NotificationMachinesTable", "NotificationsClientsTable", diff --git a/frontend/components/dashboard/table/DashboardTableValidators.vue b/frontend/components/dashboard/table/DashboardTableValidators.vue index 0428f5658..477c162c7 100644 --- a/frontend/components/dashboard/table/DashboardTableValidators.vue +++ b/frontend/components/dashboard/table/DashboardTableValidators.vue @@ -58,21 +58,24 @@ const cappedValidators = computed(() =>