Merge pull request #1020 from gobitfly/staging

merge staging->main

.github/workflows/backend-converted-types-check.yml

@@ -0,0 +1,47 @@
+name: Backend-Converted-Types-Check
+on:
+  push:
+    paths:
+      - 'backend/pkg/api/types/**'
+      - 'frontend/types/api/**'
+    branches:
+      - main
+      - staging
+  pull_request:
+    paths:
+      - 'backend/pkg/api/types/**'
+      - 'frontend/types/api/**'
+    branches:
+      - '*'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+  pull-requests: read
+  checks: write
+
+jobs:
+  build:
+    name: converted-types-check
+    runs-on: self-hosted
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v4
+        with:
+          go-version-file: 'backend/go.mod'
+          cache-dependency-path: 'backend/go.sum'
+      - name: Check if all backend-types have been converted to frontend-types
+        working-directory: backend
+        run: |
+          currHash=$(find ../frontend/types/api -type f -print0 | sort -z | xargs -0 sha1sum | sha256sum | head -c 64)
+          make frontend-types
+          newHash=$(find ../frontend/types/api -type f -print0 | sort -z | xargs -0 sha1sum | sha256sum | head -c 64)
+          if [ "$currHash" != "$newHash" ]; then
+            echo "frontend-types have changed, please commit the changes"
+            git diff --stat
+            exit 1
+          fi
+

backend/cmd/exporter/main.go

@@ -193,7 +193,19 @@ func Run() {
 		go services.StartHistoricPriceService()
 	}
 
-	go modules.StartAll(context)
+	usedModules := []modules.ModuleInterface{}
+
+	if cfg.JustV2 {
+		usedModules = append(usedModules, modules.NewDashboardDataModule(context))
+	} else {
+		usedModules = append(usedModules,
+			modules.NewSlotExporter(context),
+			modules.NewExecutionDepositsExporter(context),
+			modules.NewExecutionPayloadsExporter(context),
+		)
+	}
+
+	go modules.StartAll(context, usedModules, cfg.JustV2)
 
 	// Keep the program alive until Ctrl+C is pressed
 	utils.WaitForCtrlC()

backend/cmd/misc/main.go

@@ -247,27 +247,27 @@ func Run() {
 	}
 
 	// clickhouse
-	// db.ClickHouseWriter, db.ClickHouseReader = db.MustInitDB(&types.DatabaseConfig{
-	// 	Username:     cfg.ClickHouse.WriterDatabase.Username,
-	// 	Password:     cfg.ClickHouse.WriterDatabase.Password,
-	// 	Name:         cfg.ClickHouse.WriterDatabase.Name,
-	// 	Host:         cfg.ClickHouse.WriterDatabase.Host,
-	// 	Port:         cfg.ClickHouse.WriterDatabase.Port,
-	// 	MaxOpenConns: cfg.ClickHouse.WriterDatabase.MaxOpenConns,
-	// 	SSL:          true,
-	// 	MaxIdleConns: cfg.ClickHouse.WriterDatabase.MaxIdleConns,
-	// }, &types.DatabaseConfig{
-	// 	Username:     cfg.ClickHouse.ReaderDatabase.Username,
-	// 	Password:     cfg.ClickHouse.ReaderDatabase.Password,
-	// 	Name:         cfg.ClickHouse.ReaderDatabase.Name,
-	// 	Host:         cfg.ClickHouse.ReaderDatabase.Host,
-	// 	Port:         cfg.ClickHouse.ReaderDatabase.Port,
-	// 	MaxOpenConns: cfg.ClickHouse.ReaderDatabase.MaxOpenConns,
-	// 	SSL:          true,
-	// 	MaxIdleConns: cfg.ClickHouse.ReaderDatabase.MaxIdleConns,
-	// }, "clickhouse", "clickhouse")
-	// defer db.ClickHouseReader.Close()
-	// defer db.ClickHouseWriter.Close()
+	db.ClickHouseWriter, db.ClickHouseReader = db.MustInitDB(&types.DatabaseConfig{
+		Username:     cfg.ClickHouse.WriterDatabase.Username,
+		Password:     cfg.ClickHouse.WriterDatabase.Password,
+		Name:         cfg.ClickHouse.WriterDatabase.Name,
+		Host:         cfg.ClickHouse.WriterDatabase.Host,
+		Port:         cfg.ClickHouse.WriterDatabase.Port,
+		MaxOpenConns: cfg.ClickHouse.WriterDatabase.MaxOpenConns,
+		SSL:          true,
+		MaxIdleConns: cfg.ClickHouse.WriterDatabase.MaxIdleConns,
+	}, &types.DatabaseConfig{
+		Username:     cfg.ClickHouse.ReaderDatabase.Username,
+		Password:     cfg.ClickHouse.ReaderDatabase.Password,
+		Name:         cfg.ClickHouse.ReaderDatabase.Name,
+		Host:         cfg.ClickHouse.ReaderDatabase.Host,
+		Port:         cfg.ClickHouse.ReaderDatabase.Port,
+		MaxOpenConns: cfg.ClickHouse.ReaderDatabase.MaxOpenConns,
+		SSL:          true,
+		MaxIdleConns: cfg.ClickHouse.ReaderDatabase.MaxIdleConns,
+	}, "clickhouse", "clickhouse")
+	defer db.ClickHouseReader.Close()
+	defer db.ClickHouseWriter.Close()
 
 	// Initialize the persistent redis client
 	if requires.Redis {
@@ -564,6 +564,40 @@ func collectNotifications(startEpoch uint64) error {
 	if len(notifications[0]) > 0 {
 		spew.Dump(notifications[0])
 	}
+
+	emails, err := notification.RenderEmailsForUserEvents(0, notifications)
+	if err != nil {
+		return err
+	}
+
+	for _, email := range emails {
+		// if email.Address == "" {
+		log.Infof("to: %v", email.Address)
+		log.Infof("subject: %v", email.Subject)
+		log.Infof("body: %v", email.Email.Body)
+		log.Info("-----")
+		// }
+	}
+
+	// pushMessages, err := notification.RenderPushMessagesForUserEvents(0, notifications)
+	// if err != nil {
+	// 	return err
+	// }
+
+	// for _, pushMessage := range pushMessages {
+	// 	message := pushMessage.Messages[0]
+	// 	log.Infof("title: %v body: %v", message.Notification.Title, message.Notification.Body)
+
+	// 	if message.Token == "" {
+	// 		log.Info("sending test message")
+
+	// 		err = notification.SendPushBatch(pushMessage.UserId, []*messaging.Message{message}, false)
+	// 		if err != nil {
+	// 			log.Error(err, "error sending firebase batch job", 0)
+	// 		}
+	// 	}
+	// }
+
 	return nil
 }
 
@@ -606,7 +640,7 @@ func collectUserDbNotifications(startEpoch uint64) error {
 		if message.Token == "" {
 			log.Info("sending test message")
 
-			err = notification.SendPushBatch(pushMessage.UserId, []*messaging.Message{message}, false)
+			err = notification.SendPushBatch(pushMessage.UserId, []*messaging.Message{message}, true)
 			if err != nil {
 				log.Error(err, "error sending firebase batch job", 0)
 			}

backend/cmd/notification_collector/main.go

@@ -150,6 +150,31 @@ func Run() {
 		}, "pgx", "postgres")
 	}()
 
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		// clickhouse
+		db.ClickHouseWriter, db.ClickHouseReader = db.MustInitDB(&types.DatabaseConfig{
+			Username:     cfg.ClickHouse.WriterDatabase.Username,
+			Password:     cfg.ClickHouse.WriterDatabase.Password,
+			Name:         cfg.ClickHouse.WriterDatabase.Name,
+			Host:         cfg.ClickHouse.WriterDatabase.Host,
+			Port:         cfg.ClickHouse.WriterDatabase.Port,
+			MaxOpenConns: cfg.ClickHouse.WriterDatabase.MaxOpenConns,
+			SSL:          true,
+			MaxIdleConns: cfg.ClickHouse.WriterDatabase.MaxIdleConns,
+		}, &types.DatabaseConfig{
+			Username:     cfg.ClickHouse.ReaderDatabase.Username,
+			Password:     cfg.ClickHouse.ReaderDatabase.Password,
+			Name:         cfg.ClickHouse.ReaderDatabase.Name,
+			Host:         cfg.ClickHouse.ReaderDatabase.Host,
+			Port:         cfg.ClickHouse.ReaderDatabase.Port,
+			MaxOpenConns: cfg.ClickHouse.ReaderDatabase.MaxOpenConns,
+			SSL:          true,
+			MaxIdleConns: cfg.ClickHouse.ReaderDatabase.MaxIdleConns,
+		}, "clickhouse", "clickhouse")
+	}()
+
 	wg.Add(1)
 	go func() {
 		defer wg.Done()
@@ -184,6 +209,8 @@ func Run() {
 	defer db.FrontendWriterDB.Close()
 	defer db.AlloyReader.Close()
 	defer db.AlloyWriter.Close()
+	defer db.ClickHouseReader.Close()
+	defer db.ClickHouseWriter.Close()
 	defer db.BigtableClient.Close()
 
 	log.Infof("database connection established")

backend/cmd/typescript_converter/main.go

@@ -24,7 +24,7 @@ const (
 )
 
 // Files that should not be converted to TypeScript
-var ignoredFiles = []string{"data_access", "search_types", "archiver"}
+var ignoredFiles = []string{"data_access", "search_types", "archiver", "rocketpool"}
 
 var typeMappings = map[string]string{
 	"decimal.Decimal": "string /* decimal.Decimal */",

backend/go.mod

@@ -64,7 +64,7 @@ require (
 	github.com/prysmaticlabs/go-ssz v0.0.0-20210121151755-f6208871c388
 	github.com/rocket-pool/rocketpool-go v1.8.3-0.20240618173422-783b8668f5b4
 	github.com/rocket-pool/smartnode v1.13.6
-	github.com/shopspring/decimal v1.3.1
+	github.com/shopspring/decimal v1.4.0
 	github.com/sirupsen/logrus v1.9.3
 	github.com/stretchr/testify v1.9.0
 	github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d

backend/go.sum

@@ -873,6 +873,8 @@ github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9Nz
 github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
 github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8=
 github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
+github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k=
+github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=

backend/pkg/api/auth.go

@@ -13,8 +13,10 @@ import (
 	"github.com/gorilla/csrf"
 )
 
+var day time.Duration = time.Hour * 24
+var sessionDuration time.Duration = day * 365
+
 func newSessionManager(cfg *types.Config) *scs.SessionManager {
-	// TODO: replace redis with user db down the line (or replace sessions with oauth2)
 	pool := &redis.Pool{
 		MaxIdle: 10,
 		Dial: func() (redis.Conn, error) {
@@ -23,7 +25,7 @@ func newSessionManager(cfg *types.Config) *scs.SessionManager {
 	}
 
 	scs := scs.New()
-	scs.Lifetime = time.Hour * 24 * 7
+	scs.Lifetime = sessionDuration
 	scs.Cookie.Name = "session_id"
 	scs.Cookie.HttpOnly = true
 	scs.Cookie.Persist = true
@@ -42,6 +44,19 @@ func newSessionManager(cfg *types.Config) *scs.SessionManager {
 	return scs
 }
 
+// returns a middleware that extends the session expiration if the session is older than 1 day
+func getSlidingSessionExpirationMiddleware(scs *scs.SessionManager) func(http.Handler) http.Handler {
+	return func(next http.Handler) http.Handler {
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			deadline := scs.Deadline(r.Context()) // unauthenticated requests have deadline set to now+sessionDuration
+			if time.Until(deadline) < sessionDuration-day {
+				scs.SetDeadline(r.Context(), time.Now().Add(sessionDuration).UTC()) // setting to utc because library also does that internally
+			}
+			next.ServeHTTP(w, r)
+		})
+	}
+}
+
 // returns goriila/csrf middleware with the given config settings
 func getCsrfProtectionMiddleware(cfg *types.Config) func(http.Handler) http.Handler {
 	csrfBytes, err := hex.DecodeString(cfg.Frontend.CsrfAuthKey)