From b4a274f93f0fa2c95f60c2bd36d82873b63ef00a Mon Sep 17 00:00:00 2001
From: peter <1674920+peterbitfly@users.noreply.github.com>
Date: Tue, 3 Sep 2024 09:22:41 +0000
Subject: [PATCH 01/28] feat(dashboard): integration test scaffolding

---
 backend/pkg/api/handlers/handlers_test.go | 139 ++++++++++++++++++++++
 1 file changed, 139 insertions(+)
 create mode 100644 backend/pkg/api/handlers/handlers_test.go

diff --git a/backend/pkg/api/handlers/handlers_test.go b/backend/pkg/api/handlers/handlers_test.go
new file mode 100644
index 000000000..5cd7128cc
--- /dev/null
+++ b/backend/pkg/api/handlers/handlers_test.go
@@ -0,0 +1,139 @@
+package handlers_test
+
+import (
+	"encoding/json"
+	"flag"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"testing"
+
+	"github.com/gobitfly/beaconchain/pkg/api"
+	dataaccess "github.com/gobitfly/beaconchain/pkg/api/data_access"
+	api_types "github.com/gobitfly/beaconchain/pkg/api/types"
+	"github.com/gobitfly/beaconchain/pkg/commons/log"
+	"github.com/gobitfly/beaconchain/pkg/commons/types"
+	"github.com/gobitfly/beaconchain/pkg/commons/utils"
+	"github.com/gobitfly/beaconchain/pkg/commons/version"
+	"github.com/gorilla/mux"
+)
+
+var router *mux.Router
+var dataAccessor dataaccess.DataAccessor
+
+func TestMain(m *testing.M) {
+	setup()
+	defer teardown()
+
+	// wait till service initialization is completed (TODO: find a better way to do this)
+	// time.Sleep(30 * time.Second)
+
+	os.Exit(m.Run())
+}
+
+func teardown() {
+	dataAccessor.Close()
+}
+
+func setup() {
+	configPath := flag.String("config", "", "Path to the config file, if empty string defaults will be used")
+
+	flag.Parse()
+
+	cfg := &types.Config{}
+	err := utils.ReadConfig(cfg, *configPath)
+	if err != nil {
+		log.Fatal(err, "error reading config file", 0)
+	}
+	utils.Config = cfg
+
+	log.InfoWithFields(log.Fields{"config": *configPath, "version": version.Version, "commit": version.GitCommit, "chainName": utils.Config.Chain.ClConfig.ConfigName}, "starting")
+
+	dataAccessor = dataaccess.NewDataAccessService(cfg)
+	router = api.NewApiRouter(dataAccessor, cfg)
+}
+
+func TestInternalGetProductSummaryHandler(t *testing.T) {
+	w := httptest.NewRecorder()
+	req := httptest.NewRequest(http.MethodGet, "/api/i/product-summary", nil)
+
+	router.ServeHTTP(w, req)
+
+	resp := w.Result()
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("expected status code 200, got %d", resp.StatusCode)
+	}
+
+	data, err := io.ReadAll(resp.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	respData := api_types.InternalGetProductSummaryResponse{}
+	err = json.Unmarshal(data, &respData)
+	if err != nil {
+		log.Infof("%s", string(data))
+		t.Fatal(err)
+	}
+
+	if respData.Data.ValidatorsPerDashboardLimit == 0 {
+		t.Fatal("ValidatorsPerDashboardLimit is 0")
+	}
+
+	if len(respData.Data.ApiProducts) == 0 {
+		t.Fatal("ApiProducts length is 0")
+	}
+}
+
+func TestInternalGetLatestStateHandler(t *testing.T) {
+	req := httptest.NewRequest(http.MethodGet, "/api/i/latest-state", nil)
+
+	w := httptest.NewRecorder()
+
+	router.ServeHTTP(w, req)
+
+	resp := w.Result()
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("expected status code 200, got %d", resp.StatusCode)
+	}
+
+	data, err := io.ReadAll(resp.Body)
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	respData := api_types.InternalGetLatestStateResponse{}
+	err = json.Unmarshal(data, &respData)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if respData.Data.LatestSlot == 0 {
+		t.Fatal("latest slot is 0")
+	}
+
+	if respData.Data.FinalizedEpoch == 0 {
+		t.Fatal("finalized epoch is 0")
+	}
+}
+
+func TestInternalPostAdConfigurationsHandler(t *testing.T) {
+	req := httptest.NewRequest(http.MethodPost, "/api/i/ad-configurations", nil)
+
+	w := httptest.NewRecorder()
+
+	router.ServeHTTP(w, req)
+
+	resp := w.Result()
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusUnauthorized {
+		t.Fatalf("expected status code 401, got %d", resp.StatusCode)
+	}
+}

From f02c32c608c15bf41e58e5447b0d46437ef3426a Mon Sep 17 00:00:00 2001
From: peter <1674920+peterbitfly@users.noreply.github.com>
Date: Tue, 3 Sep 2024 11:02:53 +0000
Subject: [PATCH 02/28] feat(api): make writing tests more convenient

---
 backend/go.mod                                |   2 +
 backend/go.sum                                |   7 +-
 backend/pkg/api/handlers/handlers_test.go     | 163 +++++++++++-------
 backend/pkg/api/services/service.go           |  14 +-
 .../service_average_network_efficiency.go     |   6 +-
 .../pkg/api/services/service_email_sender.go  |   7 +-
 backend/pkg/api/services/service_slot_viz.go  |   6 +-
 .../api/services/service_validator_mapping.go |   7 +-
 8 files changed, 140 insertions(+), 72 deletions(-)

diff --git a/backend/go.mod b/backend/go.mod
index b82b96ef1..3b622725a 100644
--- a/backend/go.mod
+++ b/backend/go.mod
@@ -61,6 +61,7 @@ require (
 	github.com/rocket-pool/smartnode v1.13.6
 	github.com/shopspring/decimal v1.3.1
 	github.com/sirupsen/logrus v1.9.3
+	github.com/stretchr/testify v1.9.0
 	github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d
 	github.com/wealdtech/go-ens/v3 v3.6.0
 	github.com/wealdtech/go-eth2-types/v2 v2.8.2
@@ -194,6 +195,7 @@ require (
 	github.com/paulmach/orb v0.10.0 // indirect
 	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
 	github.com/pierrec/lz4/v4 v4.1.18 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/polydawn/refmt v0.89.0 // indirect
 	github.com/prometheus/client_model v0.6.0 // indirect
 	github.com/prometheus/common v0.47.0 // indirect
diff --git a/backend/go.sum b/backend/go.sum
index c41a5f688..f16d0bc61 100644
--- a/backend/go.sum
+++ b/backend/go.sum
@@ -840,8 +840,9 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
-github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
+github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
@@ -852,8 +853,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
-github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/supranational/blst v0.3.11 h1:LyU6FolezeWAhvQk0k6O/d49jqgO52MSDDfYgbeoEm4=
 github.com/supranational/blst v0.3.11/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw=
 github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d h1:vfofYNRScrDdvS342BElfbETmL1Aiz3i2t0zfRj16Hs=
diff --git a/backend/pkg/api/handlers/handlers_test.go b/backend/pkg/api/handlers/handlers_test.go
index 5cd7128cc..6e942d3af 100644
--- a/backend/pkg/api/handlers/handlers_test.go
+++ b/backend/pkg/api/handlers/handlers_test.go
@@ -1,10 +1,12 @@
 package handlers_test
 
 import (
+	"bytes"
 	"encoding/json"
 	"flag"
 	"io"
 	"net/http"
+	"net/http/cookiejar"
 	"net/http/httptest"
 	"os"
 	"testing"
@@ -16,12 +18,55 @@ import (
 	"github.com/gobitfly/beaconchain/pkg/commons/types"
 	"github.com/gobitfly/beaconchain/pkg/commons/utils"
 	"github.com/gobitfly/beaconchain/pkg/commons/version"
-	"github.com/gorilla/mux"
+	"github.com/stretchr/testify/assert"
 )
 
-var router *mux.Router
+var ts *testServer
 var dataAccessor dataaccess.DataAccessor
 
+type testServer struct {
+	*httptest.Server
+}
+
+// Implement a get() method on our custom testServer type. This makes a GET
+// request to a given url path using the test server client, and returns the
+// response status code, headers and body.
+func (ts *testServer) get(t *testing.T, urlPath string) (int, http.Header, string) {
+	rs, err := ts.Client().Get(ts.URL + urlPath)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer rs.Body.Close()
+	body, err := io.ReadAll(rs.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	bytes.TrimSpace(body)
+	return rs.StatusCode, rs.Header, string(body)
+}
+
+func (ts *testServer) post(t *testing.T, urlPath string, data io.Reader) (int, http.Header, string) {
+	rs, err := ts.Client().Post(ts.URL+urlPath, "application/json", data)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer rs.Body.Close()
+	body, err := io.ReadAll(rs.Body)
+	if err != nil {
+		t.Fatal(err)
+	}
+	bytes.TrimSpace(body)
+	return rs.StatusCode, rs.Header, string(body)
+}
+
+func (ts *testServer) parseErrorResonse(t *testing.T, body string) api_types.ApiErrorResponse {
+	resp := api_types.ApiErrorResponse{}
+	if err := json.Unmarshal([]byte(body), &resp); err != nil {
+		t.Fatal(err)
+	}
+	return resp
+}
+
 func TestMain(m *testing.M) {
 	setup()
 	defer teardown()
@@ -34,6 +79,7 @@ func TestMain(m *testing.M) {
 
 func teardown() {
 	dataAccessor.Close()
+	ts.Close()
 }
 
 func setup() {
@@ -51,89 +97,84 @@ func setup() {
 	log.InfoWithFields(log.Fields{"config": *configPath, "version": version.Version, "commit": version.GitCommit, "chainName": utils.Config.Chain.ClConfig.ConfigName}, "starting")
 
 	dataAccessor = dataaccess.NewDataAccessService(cfg)
-	router = api.NewApiRouter(dataAccessor, cfg)
-}
-
-func TestInternalGetProductSummaryHandler(t *testing.T) {
-	w := httptest.NewRecorder()
-	req := httptest.NewRequest(http.MethodGet, "/api/i/product-summary", nil)
+	router := api.NewApiRouter(dataAccessor, cfg)
 
-	router.ServeHTTP(w, req)
+	ts = &testServer{httptest.NewTLSServer(router)}
 
-	resp := w.Result()
-	defer resp.Body.Close()
+	jar, _ := cookiejar.New(nil)
+	ts.Server.Client().Jar = jar
+}
 
-	if resp.StatusCode != http.StatusOK {
-		t.Fatalf("expected status code 200, got %d", resp.StatusCode)
-	}
+func TestInternalGetProductSummaryHandler(t *testing.T) {
+	code, _, body := ts.get(t, "/api/i/product-summary")
 
-	data, err := io.ReadAll(resp.Body)
-	if err != nil {
-		t.Fatal(err)
-	}
+	assert.Equal(t, http.StatusOK, code)
 
 	respData := api_types.InternalGetProductSummaryResponse{}
-	err = json.Unmarshal(data, &respData)
+	err := json.Unmarshal([]byte(body), &respData)
 	if err != nil {
-		log.Infof("%s", string(data))
+		log.Infof("%s", body)
 		t.Fatal(err)
 	}
 
-	if respData.Data.ValidatorsPerDashboardLimit == 0 {
-		t.Fatal("ValidatorsPerDashboardLimit is 0")
-	}
-
-	if len(respData.Data.ApiProducts) == 0 {
-		t.Fatal("ApiProducts length is 0")
-	}
+	assert.NotEqual(t, 0, respData.Data.ValidatorsPerDashboardLimit, "ValidatorsPerDashboardLimit should not be 0")
+	assert.NotEqual(t, 0, len(respData.Data.ApiProducts), "ApiProducts should not be empty")
+	assert.NotEqual(t, 0, len(respData.Data.ExtraDashboardValidatorsPremiumAddon), "ExtraDashboardValidatorsPremiumAddon should not be empty")
+	assert.NotEqual(t, 0, len(respData.Data.PremiumProducts), "PremiumProducts should not be empty")
 }
 
 func TestInternalGetLatestStateHandler(t *testing.T) {
-	req := httptest.NewRequest(http.MethodGet, "/api/i/latest-state", nil)
-
-	w := httptest.NewRecorder()
-
-	router.ServeHTTP(w, req)
-
-	resp := w.Result()
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		t.Fatalf("expected status code 200, got %d", resp.StatusCode)
-	}
-
-	data, err := io.ReadAll(resp.Body)
-
-	if err != nil {
-		t.Fatal(err)
-	}
+	code, _, body := ts.get(t, "//api/i/latest-state")
+	assert.Equal(t, http.StatusOK, code)
 
 	respData := api_types.InternalGetLatestStateResponse{}
-	err = json.Unmarshal(data, &respData)
-	if err != nil {
+	if err := json.Unmarshal([]byte(body), &respData); err != nil {
 		t.Fatal(err)
 	}
 
-	if respData.Data.LatestSlot == 0 {
-		t.Fatal("latest slot is 0")
-	}
-
-	if respData.Data.FinalizedEpoch == 0 {
-		t.Fatal("finalized epoch is 0")
-	}
+	assert.NotEqual(t, uint64(0), respData.Data.LatestSlot, "latest slot should not be 0")
+	assert.NotEqual(t, uint64(0), respData.Data.FinalizedEpoch, "finalized epoch should not be 0")
 }
 
 func TestInternalPostAdConfigurationsHandler(t *testing.T) {
-	req := httptest.NewRequest(http.MethodPost, "/api/i/ad-configurations", nil)
+	code, _, body := ts.get(t, "/api/i/ad-configurations")
+	assert.Equal(t, http.StatusUnauthorized, code)
+
+	resp := ts.parseErrorResonse(t, body)
+	assert.Equal(t, "unauthorized: not authenticated", resp.Error)
 
-	w := httptest.NewRecorder()
+	// login
+	code, _, body = ts.post(t, "/api/i/login", bytes.NewBuffer([]byte(`{"email": "admin@admin.com", "password": "admin"}`)))
+	assert.Equal(t, http.StatusNotFound, code)
+	resp = ts.parseErrorResonse(t, body)
+	assert.Equal(t, "not found: user not found", resp.Error)
+}
 
-	router.ServeHTTP(w, req)
+func TestInternalLoginHandler(t *testing.T) {
+	// login with email in wrong format
+	code, _, body := ts.post(t, "/api/i/login", bytes.NewBuffer([]byte(`{"email": "admin", "password": "admin"}`)))
+	assert.Equal(t, http.StatusBadRequest, code)
+	resp := ts.parseErrorResonse(t, body)
+	assert.Equal(t, "email: given value 'admin' has incorrect format", resp.Error, "unexpected error message")
+
+	// login with wrong user
+	code, _, body = ts.post(t, "/api/i/login", bytes.NewBufferString(`{"email": "admin@admin.com", "password": "admin"}`))
+	assert.Equal(t, http.StatusNotFound, code)
+	resp = ts.parseErrorResonse(t, body)
+	assert.Equal(t, "not found: user not found", resp.Error, "unexpected error message") // TODO: this should not return the same error as a user with a wrong password
+}
 
-	resp := w.Result()
-	defer resp.Body.Close()
+func TestInternalSearchHandler(t *testing.T) {
+	// search for validator with index 5
+	code, _, body := ts.post(t, "/api/i/search", bytes.NewBufferString(`{"input":"5","networks":[17000],"types":["validators_by_deposit_ens_name","validators_by_deposit_address","validators_by_withdrawal_ens_name","validators_by_withdrawal_address","validators_by_withdrawal_credential","validator_by_index","validator_by_public_key","validators_by_graffiti"]}`))
+	assert.Equal(t, 200, code)
 
-	if resp.StatusCode != http.StatusUnauthorized {
-		t.Fatalf("expected status code 401, got %d", resp.StatusCode)
+	resp := api_types.InternalPostSearchResponse{}
+	if err := json.Unmarshal([]byte(body), &resp); err != nil {
+		t.Fatal(err)
 	}
+
+	assert.NotEqual(t, 0, len(resp.Data), "response data should not be empty")
+	assert.NotNil(t, resp.Data[0].NumValue, "validator index should not be nil")
+	assert.Equal(t, uint64(5), *resp.Data[0].NumValue, "validator index should be 5")
 }
diff --git a/backend/pkg/api/services/service.go b/backend/pkg/api/services/service.go
index 2e009e85c..872e3391b 100644
--- a/backend/pkg/api/services/service.go
+++ b/backend/pkg/api/services/service.go
@@ -1,6 +1,8 @@
 package services
 
 import (
+	"sync"
+
 	"github.com/go-redis/redis/v8"
 	"github.com/gobitfly/beaconchain/pkg/commons/db"
 	"github.com/gobitfly/beaconchain/pkg/commons/log"
@@ -35,11 +37,15 @@ func NewServices(readerDb, writerDb, alloyReader, alloyWriter, clickhouseReader
 }
 
 func (s *Services) InitServices() {
-	go s.startSlotVizDataService()
-	go s.startIndexMappingService()
-	go s.startEfficiencyDataService()
-	go s.startEmailSenderService()
+	wg := &sync.WaitGroup{}
+	log.Infof("initializing services...")
+	wg.Add(4)
+	go s.startSlotVizDataService(wg)
+	go s.startIndexMappingService(wg)
+	go s.startEfficiencyDataService(wg)
+	go s.startEmailSenderService(wg)
 
+	wg.Wait()
 	log.Infof("initializing prices...")
 	price.Init(utils.Config.Chain.ClConfig.DepositChainID, utils.Config.Eth1ErigonEndpoint, utils.Config.Frontend.ClCurrency, utils.Config.Frontend.ElCurrency)
 	log.Infof("...prices initialized")
diff --git a/backend/pkg/api/services/service_average_network_efficiency.go b/backend/pkg/api/services/service_average_network_efficiency.go
index 725ff0511..083b06dc2 100644
--- a/backend/pkg/api/services/service_average_network_efficiency.go
+++ b/backend/pkg/api/services/service_average_network_efficiency.go
@@ -22,7 +22,8 @@ import (
 
 var currentEfficiencyInfo atomic.Pointer[EfficiencyData]
 
-func (s *Services) startEfficiencyDataService() {
+func (s *Services) startEfficiencyDataService(wg *sync.WaitGroup) {
+	o := sync.Once{}
 	for {
 		startTime := time.Now()
 		delay := time.Duration(utils.Config.Chain.ClConfig.SlotsPerEpoch*utils.Config.Chain.ClConfig.SecondsPerSlot) * time.Second
@@ -36,6 +37,9 @@ func (s *Services) startEfficiencyDataService() {
 		} else {
 			log.Infof("=== average network efficiency data updated in %s", time.Since(startTime))
 			r(constants.Success, map[string]string{"took": time.Since(startTime).String()})
+			o.Do(func() {
+				wg.Done()
+			})
 		}
 		utils.ConstantTimeDelay(startTime, delay)
 	}
diff --git a/backend/pkg/api/services/service_email_sender.go b/backend/pkg/api/services/service_email_sender.go
index 4bf0bee69..fa8cf61e2 100644
--- a/backend/pkg/api/services/service_email_sender.go
+++ b/backend/pkg/api/services/service_email_sender.go
@@ -1,6 +1,7 @@
 package services
 
 import (
+	"sync"
 	"time"
 
 	"github.com/gobitfly/beaconchain/pkg/commons/log"
@@ -29,7 +30,8 @@ var Queue []QueuedEmail
 // collects & queues mails, sends in batches regularly (possibly aggregating multiple messasages to the same user to avoid spam?)
 // TODO ratelimiting
 // TODO send via SMTP/mailgun/others?
-func (s *Services) startEmailSenderService() {
+func (s *Services) startEmailSenderService(wg *sync.WaitGroup) {
+	o := sync.Once{}
 	for {
 		startTime := time.Now()
 		// lock mutex
@@ -44,6 +46,9 @@ func (s *Services) startEmailSenderService() {
 			}*/
 		}
 		log.Infof("=== message sending done in %s", time.Since(startTime))
+		o.Do(func() {
+			wg.Done()
+		})
 		utils.ConstantTimeDelay(startTime, 30*time.Second)
 	}
 }
diff --git a/backend/pkg/api/services/service_slot_viz.go b/backend/pkg/api/services/service_slot_viz.go
index e26145445..9b7af5031 100644
--- a/backend/pkg/api/services/service_slot_viz.go
+++ b/backend/pkg/api/services/service_slot_viz.go
@@ -27,7 +27,8 @@ import (
 
 var currentDutiesInfo atomic.Pointer[SyncData]
 
-func (s *Services) startSlotVizDataService() {
+func (s *Services) startSlotVizDataService(wg *sync.WaitGroup) {
+	o := sync.Once{}
 	for {
 		startTime := time.Now()
 		delay := time.Duration(utils.Config.Chain.ClConfig.SecondsPerSlot) * time.Second
@@ -40,6 +41,9 @@ func (s *Services) startSlotVizDataService() {
 		}
 		log.Infof("=== slotviz data updated in %s", time.Since(startTime))
 		r(constants.Success, map[string]string{"took": time.Since(startTime).String()})
+		o.Do(func() {
+			wg.Done()
+		})
 		utils.ConstantTimeDelay(startTime, delay)
 	}
 }
diff --git a/backend/pkg/api/services/service_validator_mapping.go b/backend/pkg/api/services/service_validator_mapping.go
index 374c6906a..187c41a14 100644
--- a/backend/pkg/api/services/service_validator_mapping.go
+++ b/backend/pkg/api/services/service_validator_mapping.go
@@ -5,6 +5,7 @@ import (
 	"context"
 	"encoding/gob"
 	"fmt"
+	"sync"
 	"sync/atomic"
 	"time"
 
@@ -33,8 +34,9 @@ var _cachedRedisValidatorMapping = new(types.RedisCachedValidatorsMapping)
 
 var lastEpochUpdate = uint64(0)
 
-func (s *Services) startIndexMappingService() {
+func (s *Services) startIndexMappingService(wg *sync.WaitGroup) {
 	var err error
+	o := sync.Once{}
 	for {
 		startTime := time.Now()
 		delay := time.Duration(utils.Config.Chain.ClConfig.SecondsPerSlot) * time.Second
@@ -53,6 +55,9 @@ func (s *Services) startIndexMappingService() {
 			log.Infof("=== validator mapping updated in %s", time.Since(startTime))
 			r(constants.Success, map[string]string{"took": time.Since(startTime).String(), "latest_epoch": fmt.Sprintf("%d", lastEpochUpdate)})
 			lastEpochUpdate = latestEpoch
+			o.Do(func() {
+				wg.Done()
+			})
 		}
 		utils.ConstantTimeDelay(startTime, delay)
 	}

From dd3e084d52f7f08cc5596415b4e825f31b0d0e01 Mon Sep 17 00:00:00 2001
From: peter <1674920+peterbitfly@users.noreply.github.com>
Date: Tue, 3 Sep 2024 13:20:12 +0000
Subject: [PATCH 03/28] feat(api): improve testing scaffolding

---
 backend/go.mod                            |   4 +-
 backend/go.sum                            |   4 +
 backend/pkg/api/auth.go                   |   1 +
 backend/pkg/api/handlers/handlers_test.go | 133 ++++++++++++++++++----
 4 files changed, 117 insertions(+), 25 deletions(-)

diff --git a/backend/go.mod b/backend/go.mod
index 3b622725a..8e2ae391c 100644
--- a/backend/go.mod
+++ b/backend/go.mod
@@ -21,6 +21,7 @@ require (
 	github.com/donovanhide/eventsource v0.0.0-20210830082556-c59027999da0
 	github.com/doug-martin/goqu/v9 v9.19.0
 	github.com/ethereum/go-ethereum v1.13.12
+	github.com/fergusstrange/embedded-postgres v1.29.0
 	github.com/go-faker/faker/v4 v4.3.0
 	github.com/go-redis/redis/v8 v8.11.5
 	github.com/gobitfly/eth-rewards v0.1.2-0.20230403064929-411ddc40a5f7
@@ -69,6 +70,7 @@ require (
 	github.com/xeipuuv/gojsonschema v1.2.0
 	golang.org/x/crypto v0.19.0
 	golang.org/x/exp v0.0.0-20240213143201-ec583247a57a
+	golang.org/x/net v0.21.0
 	golang.org/x/sync v0.6.0
 	golang.org/x/text v0.14.0
 	golang.org/x/time v0.5.0
@@ -222,6 +224,7 @@ require (
 	github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect
 	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
 	github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
+	github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
 	github.com/yusufpapurcu/wmi v1.2.3 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.47.0 // indirect
@@ -233,7 +236,6 @@ require (
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
 	golang.org/x/mod v0.15.0 // indirect
-	golang.org/x/net v0.21.0 // indirect
 	golang.org/x/oauth2 v0.17.0 // indirect
 	golang.org/x/sys v0.17.0 // indirect
 	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
diff --git a/backend/go.sum b/backend/go.sum
index f16d0bc61..f3e0980f2 100644
--- a/backend/go.sum
+++ b/backend/go.sum
@@ -237,6 +237,8 @@ github.com/fatih/structtag v1.2.0 h1:/OdNE99OxoI/PqaW/SuSK9uxxT3f/tcSZgon/ssNSx4
 github.com/fatih/structtag v1.2.0/go.mod h1:mBJUNpUnHmRKrKlQQlmCrh5PuhftFbNv8Ys4/aAZl94=
 github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
 github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
+github.com/fergusstrange/embedded-postgres v1.29.0 h1:Uv8hdhoiaNMuH0w8UuGXDHr60VoAQPFdgx7Qf3bzXJM=
+github.com/fergusstrange/embedded-postgres v1.29.0/go.mod h1:t/MLs0h9ukYM6FSt99R7InCHs1nW0ordoVCcnzmpTYw=
 github.com/ferranbt/fastssz v0.1.3 h1:ZI+z3JH05h4kgmFXdHuR1aWYsgrg7o+Fw7/NCzM16Mo=
 github.com/ferranbt/fastssz v0.1.3/go.mod h1:0Y9TEd/9XuFlh7mskMPfXiI2Dkw4Ddg9EyXt1W7MRvE=
 github.com/fjl/memsize v0.0.2 h1:27txuSD9or+NZlnOWdKUxeBzTAUkWCVh+4Gf2dWFOzA=
@@ -915,6 +917,8 @@ github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHo
 github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
 github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=
 github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
+github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo=
+github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos=
 github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU=
 github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8=
 github.com/ydb-platform/ydb-go-genproto v0.0.0-20240126124512-dbb0e1720dbf h1:ckwNHVo4bv2tqNkgx3W3HANh3ta1j6TR5qw08J1A7Tw=
diff --git a/backend/pkg/api/auth.go b/backend/pkg/api/auth.go
index dc8a62814..048c90865 100644
--- a/backend/pkg/api/auth.go
+++ b/backend/pkg/api/auth.go
@@ -35,6 +35,7 @@ func newSessionManager(cfg *types.Config) *scs.SessionManager {
 		secure = true
 	}
 	scs.Cookie.Secure = secure
+	log.Info("Session cookie secure:", secure)
 	scs.Cookie.SameSite = sameSite
 
 	scs.Store = redisstore.New(pool)
diff --git a/backend/pkg/api/handlers/handlers_test.go b/backend/pkg/api/handlers/handlers_test.go
index 6e942d3af..84f826ec3 100644
--- a/backend/pkg/api/handlers/handlers_test.go
+++ b/backend/pkg/api/handlers/handlers_test.go
@@ -9,8 +9,11 @@ import (
 	"net/http/cookiejar"
 	"net/http/httptest"
 	"os"
+	"os/exec"
 	"testing"
+	"time"
 
+	embeddedpostgres "github.com/fergusstrange/embedded-postgres"
 	"github.com/gobitfly/beaconchain/pkg/api"
 	dataaccess "github.com/gobitfly/beaconchain/pkg/api/data_access"
 	api_types "github.com/gobitfly/beaconchain/pkg/api/types"
@@ -18,11 +21,15 @@ import (
 	"github.com/gobitfly/beaconchain/pkg/commons/types"
 	"github.com/gobitfly/beaconchain/pkg/commons/utils"
 	"github.com/gobitfly/beaconchain/pkg/commons/version"
+	"github.com/jmoiron/sqlx"
+	"github.com/pressly/goose/v3"
 	"github.com/stretchr/testify/assert"
+	"golang.org/x/crypto/bcrypt"
 )
 
 var ts *testServer
 var dataAccessor dataaccess.DataAccessor
+var postgres *embeddedpostgres.EmbeddedPostgres
 
 type testServer struct {
 	*httptest.Server
@@ -69,17 +76,19 @@ func (ts *testServer) parseErrorResonse(t *testing.T, body string) api_types.Api
 
 func TestMain(m *testing.M) {
 	setup()
-	defer teardown()
+	code := m.Run()
+	teardown()
 
 	// wait till service initialization is completed (TODO: find a better way to do this)
 	// time.Sleep(30 * time.Second)
 
-	os.Exit(m.Run())
+	os.Exit(code)
 }
 
 func teardown() {
 	dataAccessor.Close()
 	ts.Close()
+	postgres.Stop()
 }
 
 func setup() {
@@ -87,11 +96,50 @@ func setup() {
 
 	flag.Parse()
 
+	_ = exec.Command("pkill", "-9", "postgres").Run()
+	postgres = embeddedpostgres.NewDatabase(embeddedpostgres.DefaultConfig().Username("postgres"))
+	err := postgres.Start()
+	if err != nil {
+		log.Fatal(err, "error starting embedded postgres", 0)
+	}
+
+	tempDb, err := sqlx.Connect("postgres", "host=localhost port=5432 user=postgres password=postgres dbname=postgres sslmode=disable")
+	if err != nil {
+		log.Fatal(err, "error connection to test db", 0)
+	}
+
+	if err := goose.Up(tempDb.DB, "../../../pkg/commons/db/migrations/postgres"); err != nil {
+		log.Fatal(err, "error running migrations", 0)
+	}
+
+	// insert dummy user for testing (email: admin@admin, password: admin)
+	pHash, _ := bcrypt.GenerateFromPassword([]byte("admin"), 10)
+	_, err = tempDb.Exec(`
+      INSERT INTO users (password, email, register_ts, api_key, email_confirmed)
+      VALUES ($1, $2, TO_TIMESTAMP($3), $4, $5)`,
+		string(pHash), "admin@admin.com", time.Now().Unix(), "admin", true,
+	)
+	if err != nil {
+		log.Fatal(err, "error inserting user", 0)
+	}
+
 	cfg := &types.Config{}
-	err := utils.ReadConfig(cfg, *configPath)
+	err = utils.ReadConfig(cfg, *configPath)
 	if err != nil {
 		log.Fatal(err, "error reading config file", 0)
 	}
+	cfg.Frontend.ReaderDatabase.Host = "localhost"
+	cfg.Frontend.ReaderDatabase.Port = "5432"
+	cfg.Frontend.ReaderDatabase.Name = "postgres"
+	cfg.Frontend.ReaderDatabase.Password = "postgres"
+	cfg.Frontend.ReaderDatabase.Username = "postgres"
+
+	cfg.Frontend.WriterDatabase.Host = "localhost"
+	cfg.Frontend.WriterDatabase.Port = "5432"
+	cfg.Frontend.WriterDatabase.Name = "postgres"
+	cfg.Frontend.WriterDatabase.Password = "postgres"
+	cfg.Frontend.WriterDatabase.Username = "postgres"
+
 	utils.Config = cfg
 
 	log.InfoWithFields(log.Fields{"config": *configPath, "version": version.Version, "commit": version.GitCommit, "chainName": utils.Config.Chain.ClConfig.ConfigName}, "starting")
@@ -100,8 +148,11 @@ func setup() {
 	router := api.NewApiRouter(dataAccessor, cfg)
 
 	ts = &testServer{httptest.NewTLSServer(router)}
-
-	jar, _ := cookiejar.New(nil)
+	log.Info(ts.URL)
+	jar, err := cookiejar.New(nil)
+	if err != nil {
+		log.Fatal(err, "error creating cookie jar", 0)
+	}
 	ts.Server.Client().Jar = jar
 }
 
@@ -124,7 +175,7 @@ func TestInternalGetProductSummaryHandler(t *testing.T) {
 }
 
 func TestInternalGetLatestStateHandler(t *testing.T) {
-	code, _, body := ts.get(t, "//api/i/latest-state")
+	code, _, body := ts.get(t, "/api/i/latest-state")
 	assert.Equal(t, http.StatusOK, code)
 
 	respData := api_types.InternalGetLatestStateResponse{}
@@ -136,20 +187,6 @@ func TestInternalGetLatestStateHandler(t *testing.T) {
 	assert.NotEqual(t, uint64(0), respData.Data.FinalizedEpoch, "finalized epoch should not be 0")
 }
 
-func TestInternalPostAdConfigurationsHandler(t *testing.T) {
-	code, _, body := ts.get(t, "/api/i/ad-configurations")
-	assert.Equal(t, http.StatusUnauthorized, code)
-
-	resp := ts.parseErrorResonse(t, body)
-	assert.Equal(t, "unauthorized: not authenticated", resp.Error)
-
-	// login
-	code, _, body = ts.post(t, "/api/i/login", bytes.NewBuffer([]byte(`{"email": "admin@admin.com", "password": "admin"}`)))
-	assert.Equal(t, http.StatusNotFound, code)
-	resp = ts.parseErrorResonse(t, body)
-	assert.Equal(t, "not found: user not found", resp.Error)
-}
-
 func TestInternalLoginHandler(t *testing.T) {
 	// login with email in wrong format
 	code, _, body := ts.post(t, "/api/i/login", bytes.NewBuffer([]byte(`{"email": "admin", "password": "admin"}`)))
@@ -157,11 +194,33 @@ func TestInternalLoginHandler(t *testing.T) {
 	resp := ts.parseErrorResonse(t, body)
 	assert.Equal(t, "email: given value 'admin' has incorrect format", resp.Error, "unexpected error message")
 
-	// login with wrong user
-	code, _, body = ts.post(t, "/api/i/login", bytes.NewBufferString(`{"email": "admin@admin.com", "password": "admin"}`))
-	assert.Equal(t, http.StatusNotFound, code)
+	// login with correct user and wrong password
+	code, _, body = ts.post(t, "/api/i/login", bytes.NewBufferString(`{"email": "admin@admin.com", "password": "wrong"}`))
+	assert.Equal(t, http.StatusUnauthorized, code, "login should not be successful")
 	resp = ts.parseErrorResonse(t, body)
-	assert.Equal(t, "not found: user not found", resp.Error, "unexpected error message") // TODO: this should not return the same error as a user with a wrong password
+	assert.Equal(t, "unauthorized: invalid email or password", resp.Error, "unexpected error message")
+
+	// login with correct user and password
+	code, _, _ = ts.post(t, "/api/i/login", bytes.NewBufferString(`{"email": "admin@admin.com", "password": "admin"}`))
+	assert.Equal(t, http.StatusOK, code, "login should be successful")
+
+	// check if user is logged in and has a valid session
+	code, _, body = ts.get(t, "/api/i/users/me")
+	assert.Equal(t, http.StatusOK, code, "call to users/me should be successful")
+	meResponse := &api_types.InternalGetUserInfoResponse{}
+	if err := json.Unmarshal([]byte(body), meResponse); err != nil {
+		t.Fatal(err)
+	}
+	// check if email is censored
+	assert.Equal(t, meResponse.Data.Email, "a***n@a***n.com", "email should be a***n@a***n.com")
+
+	// check if logout works
+	code, _, _ = ts.post(t, "/api/i/logout", bytes.NewBufferString(``))
+	assert.Equal(t, http.StatusOK, code, "logout should be successful")
+
+	// check if user is logged out
+	code, _, _ = ts.get(t, "/api/i/users/me")
+	assert.Equal(t, http.StatusUnauthorized, code, "call to users/me should be unauthorized")
 }
 
 func TestInternalSearchHandler(t *testing.T) {
@@ -177,4 +236,30 @@ func TestInternalSearchHandler(t *testing.T) {
 	assert.NotEqual(t, 0, len(resp.Data), "response data should not be empty")
 	assert.NotNil(t, resp.Data[0].NumValue, "validator index should not be nil")
 	assert.Equal(t, uint64(5), *resp.Data[0].NumValue, "validator index should be 5")
+
+	// search for validator by pubkey
+	code, _, body = ts.post(t, "/api/i/search", bytes.NewBufferString(`{"input":"0x9699af2bad9826694a480cb523cbe545dc41db955356b3b0d4871f1cf3e4924ae4132fa8c374a0505ae2076d3d65b3e0","networks":[17000],"types":["validators_by_deposit_ens_name","validators_by_deposit_address","validators_by_withdrawal_ens_name","validators_by_withdrawal_address","validators_by_withdrawal_credential","validator_by_index","validator_by_public_key","validators_by_graffiti"]}`))
+	assert.Equal(t, 200, code)
+
+	resp = api_types.InternalPostSearchResponse{}
+	if err := json.Unmarshal([]byte(body), &resp); err != nil {
+		t.Fatal(err)
+	}
+
+	assert.NotEqual(t, 0, len(resp.Data), "response data should not be empty")
+	assert.NotNil(t, resp.Data[0].NumValue, "validator index should not be nil")
+	assert.Equal(t, uint64(5), *resp.Data[0].NumValue, "validator index should be 5")
+
+	// search for validator by withdawal address
+	code, _, body = ts.post(t, "/api/i/search", bytes.NewBufferString(`{"input":"0x0e5dda855eb1de2a212cd1f62b2a3ee49d20c444","networks":[17000],"types":["validators_by_deposit_ens_name","validators_by_deposit_address","validators_by_withdrawal_ens_name","validators_by_withdrawal_address","validators_by_withdrawal_credential","validator_by_index","validator_by_public_key","validators_by_graffiti"]}`))
+	assert.Equal(t, 200, code)
+
+	resp = api_types.InternalPostSearchResponse{}
+	if err := json.Unmarshal([]byte(body), &resp); err != nil {
+		t.Fatal(err)
+	}
+
+	assert.NotEqual(t, 0, len(resp.Data), "response data should not be empty")
+	assert.NotNil(t, resp.Data[0].NumValue, "validator index should not be nil")
+	assert.Greater(t, *resp.Data[0].NumValue, uint64(0), "returned number of validators should be greater than 0")
 }

From e481b07adc4914170e762faf5c352b8260d18926 Mon Sep 17 00:00:00 2001
From: peter <1674920+peterbitfly@users.noreply.github.com>
Date: Tue, 3 Sep 2024 13:24:53 +0000
Subject: [PATCH 04/28] feat(api): simplify json parsing in tests

---
 backend/pkg/api/handlers/handlers_test.go | 38 +++++++++++------------
 1 file changed, 18 insertions(+), 20 deletions(-)

diff --git a/backend/pkg/api/handlers/handlers_test.go b/backend/pkg/api/handlers/handlers_test.go
index 84f826ec3..10249c6a9 100644
--- a/backend/pkg/api/handlers/handlers_test.go
+++ b/backend/pkg/api/handlers/handlers_test.go
@@ -96,13 +96,17 @@ func setup() {
 
 	flag.Parse()
 
+	// terminate any currently running postgres instances
 	_ = exec.Command("pkill", "-9", "postgres").Run()
+
+	// start embedded postgres
 	postgres = embeddedpostgres.NewDatabase(embeddedpostgres.DefaultConfig().Username("postgres"))
 	err := postgres.Start()
 	if err != nil {
 		log.Fatal(err, "error starting embedded postgres", 0)
 	}
 
+	// connection the the embedded db and run migrations
 	tempDb, err := sqlx.Connect("postgres", "host=localhost port=5432 user=postgres password=postgres dbname=postgres sslmode=disable")
 	if err != nil {
 		log.Fatal(err, "error connection to test db", 0)
@@ -128,6 +132,8 @@ func setup() {
 	if err != nil {
 		log.Fatal(err, "error reading config file", 0)
 	}
+
+	// hardcode db connection details for testing
 	cfg.Frontend.ReaderDatabase.Host = "localhost"
 	cfg.Frontend.ReaderDatabase.Port = "5432"
 	cfg.Frontend.ReaderDatabase.Name = "postgres"
@@ -148,7 +154,6 @@ func setup() {
 	router := api.NewApiRouter(dataAccessor, cfg)
 
 	ts = &testServer{httptest.NewTLSServer(router)}
-	log.Info(ts.URL)
 	jar, err := cookiejar.New(nil)
 	if err != nil {
 		log.Fatal(err, "error creating cookie jar", 0)
@@ -163,10 +168,7 @@ func TestInternalGetProductSummaryHandler(t *testing.T) {
 
 	respData := api_types.InternalGetProductSummaryResponse{}
 	err := json.Unmarshal([]byte(body), &respData)
-	if err != nil {
-		log.Infof("%s", body)
-		t.Fatal(err)
-	}
+	assert.Nil(t, err, "error unmarshalling response")
 
 	assert.NotEqual(t, 0, respData.Data.ValidatorsPerDashboardLimit, "ValidatorsPerDashboardLimit should not be 0")
 	assert.NotEqual(t, 0, len(respData.Data.ApiProducts), "ApiProducts should not be empty")
@@ -179,9 +181,8 @@ func TestInternalGetLatestStateHandler(t *testing.T) {
 	assert.Equal(t, http.StatusOK, code)
 
 	respData := api_types.InternalGetLatestStateResponse{}
-	if err := json.Unmarshal([]byte(body), &respData); err != nil {
-		t.Fatal(err)
-	}
+	err := json.Unmarshal([]byte(body), &respData)
+	assert.Nil(t, err, "error unmarshalling response")
 
 	assert.NotEqual(t, uint64(0), respData.Data.LatestSlot, "latest slot should not be 0")
 	assert.NotEqual(t, uint64(0), respData.Data.FinalizedEpoch, "finalized epoch should not be 0")
@@ -208,9 +209,9 @@ func TestInternalLoginHandler(t *testing.T) {
 	code, _, body = ts.get(t, "/api/i/users/me")
 	assert.Equal(t, http.StatusOK, code, "call to users/me should be successful")
 	meResponse := &api_types.InternalGetUserInfoResponse{}
-	if err := json.Unmarshal([]byte(body), meResponse); err != nil {
-		t.Fatal(err)
-	}
+	err := json.Unmarshal([]byte(body), meResponse)
+	assert.Nil(t, err, "error unmarshalling response")
+
 	// check if email is censored
 	assert.Equal(t, meResponse.Data.Email, "a***n@a***n.com", "email should be a***n@a***n.com")
 
@@ -229,9 +230,8 @@ func TestInternalSearchHandler(t *testing.T) {
 	assert.Equal(t, 200, code)
 
 	resp := api_types.InternalPostSearchResponse{}
-	if err := json.Unmarshal([]byte(body), &resp); err != nil {
-		t.Fatal(err)
-	}
+	err := json.Unmarshal([]byte(body), &resp)
+	assert.Nil(t, err, "error unmarshalling response")
 
 	assert.NotEqual(t, 0, len(resp.Data), "response data should not be empty")
 	assert.NotNil(t, resp.Data[0].NumValue, "validator index should not be nil")
@@ -242,9 +242,8 @@ func TestInternalSearchHandler(t *testing.T) {
 	assert.Equal(t, 200, code)
 
 	resp = api_types.InternalPostSearchResponse{}
-	if err := json.Unmarshal([]byte(body), &resp); err != nil {
-		t.Fatal(err)
-	}
+	err = json.Unmarshal([]byte(body), &resp)
+	assert.Nil(t, err, "error unmarshalling response")
 
 	assert.NotEqual(t, 0, len(resp.Data), "response data should not be empty")
 	assert.NotNil(t, resp.Data[0].NumValue, "validator index should not be nil")
@@ -255,9 +254,8 @@ func TestInternalSearchHandler(t *testing.T) {
 	assert.Equal(t, 200, code)
 
 	resp = api_types.InternalPostSearchResponse{}
-	if err := json.Unmarshal([]byte(body), &resp); err != nil {
-		t.Fatal(err)
-	}
+	err = json.Unmarshal([]byte(body), &resp)
+	assert.Nil(t, err, "error unmarshalling response")
 
 	assert.NotEqual(t, 0, len(resp.Data), "response data should not be empty")
 	assert.NotNil(t, resp.Data[0].NumValue, "validator index should not be nil")

From 6d07c0c04c2c5d2bad1c651991d0fe76f50708bd Mon Sep 17 00:00:00 2001
From: peter <1674920+peterbitfly@users.noreply.github.com>
Date: Tue, 3 Sep 2024 13:26:49 +0000
Subject: [PATCH 05/28] chore(api): improve test formatting

---
 backend/pkg/api/handlers/handlers_test.go | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/backend/pkg/api/handlers/handlers_test.go b/backend/pkg/api/handlers/handlers_test.go
index 10249c6a9..bfa37d5ce 100644
--- a/backend/pkg/api/handlers/handlers_test.go
+++ b/backend/pkg/api/handlers/handlers_test.go
@@ -169,7 +169,6 @@ func TestInternalGetProductSummaryHandler(t *testing.T) {
 	respData := api_types.InternalGetProductSummaryResponse{}
 	err := json.Unmarshal([]byte(body), &respData)
 	assert.Nil(t, err, "error unmarshalling response")
-
 	assert.NotEqual(t, 0, respData.Data.ValidatorsPerDashboardLimit, "ValidatorsPerDashboardLimit should not be 0")
 	assert.NotEqual(t, 0, len(respData.Data.ApiProducts), "ApiProducts should not be empty")
 	assert.NotEqual(t, 0, len(respData.Data.ExtraDashboardValidatorsPremiumAddon), "ExtraDashboardValidatorsPremiumAddon should not be empty")
@@ -183,7 +182,6 @@ func TestInternalGetLatestStateHandler(t *testing.T) {
 	respData := api_types.InternalGetLatestStateResponse{}
 	err := json.Unmarshal([]byte(body), &respData)
 	assert.Nil(t, err, "error unmarshalling response")
-
 	assert.NotEqual(t, uint64(0), respData.Data.LatestSlot, "latest slot should not be 0")
 	assert.NotEqual(t, uint64(0), respData.Data.FinalizedEpoch, "finalized epoch should not be 0")
 }
@@ -208,10 +206,10 @@ func TestInternalLoginHandler(t *testing.T) {
 	// check if user is logged in and has a valid session
 	code, _, body = ts.get(t, "/api/i/users/me")
 	assert.Equal(t, http.StatusOK, code, "call to users/me should be successful")
+
 	meResponse := &api_types.InternalGetUserInfoResponse{}
 	err := json.Unmarshal([]byte(body), meResponse)
 	assert.Nil(t, err, "error unmarshalling response")
-
 	// check if email is censored
 	assert.Equal(t, meResponse.Data.Email, "a***n@a***n.com", "email should be a***n@a***n.com")
 
@@ -232,7 +230,6 @@ func TestInternalSearchHandler(t *testing.T) {
 	resp := api_types.InternalPostSearchResponse{}
 	err := json.Unmarshal([]byte(body), &resp)
 	assert.Nil(t, err, "error unmarshalling response")
-
 	assert.NotEqual(t, 0, len(resp.Data), "response data should not be empty")
 	assert.NotNil(t, resp.Data[0].NumValue, "validator index should not be nil")
 	assert.Equal(t, uint64(5), *resp.Data[0].NumValue, "validator index should be 5")
@@ -244,7 +241,6 @@ func TestInternalSearchHandler(t *testing.T) {
 	resp = api_types.InternalPostSearchResponse{}
 	err = json.Unmarshal([]byte(body), &resp)
 	assert.Nil(t, err, "error unmarshalling response")
-
 	assert.NotEqual(t, 0, len(resp.Data), "response data should not be empty")
 	assert.NotNil(t, resp.Data[0].NumValue, "validator index should not be nil")
 	assert.Equal(t, uint64(5), *resp.Data[0].NumValue, "validator index should be 5")
@@ -256,7 +252,6 @@ func TestInternalSearchHandler(t *testing.T) {
 	resp = api_types.InternalPostSearchResponse{}
 	err = json.Unmarshal([]byte(body), &resp)
 	assert.Nil(t, err, "error unmarshalling response")
-
 	assert.NotEqual(t, 0, len(resp.Data), "response data should not be empty")
 	assert.NotNil(t, resp.Data[0].NumValue, "validator index should not be nil")
 	assert.Greater(t, *resp.Data[0].NumValue, uint64(0), "returned number of validators should be greater than 0")

From d81e50694f5bf7cff2953ca4e64878e197d94242 Mon Sep 17 00:00:00 2001
From: peter <1674920+peterbitfly@users.noreply.github.com>
Date: Tue, 3 Sep 2024 13:46:39 +0000
Subject: [PATCH 06/28] chore(api): remove debug logging

---
 backend/pkg/api/auth.go                   |  1 -
 backend/pkg/api/handlers/handlers_test.go | 36 ++++++++++++++++++++---
 2 files changed, 32 insertions(+), 5 deletions(-)

diff --git a/backend/pkg/api/auth.go b/backend/pkg/api/auth.go
index 048c90865..dc8a62814 100644
--- a/backend/pkg/api/auth.go
+++ b/backend/pkg/api/auth.go
@@ -35,7 +35,6 @@ func newSessionManager(cfg *types.Config) *scs.SessionManager {
 		secure = true
 	}
 	scs.Cookie.Secure = secure
-	log.Info("Session cookie secure:", secure)
 	scs.Cookie.SameSite = sameSite
 
 	scs.Store = redisstore.New(pool)
diff --git a/backend/pkg/api/handlers/handlers_test.go b/backend/pkg/api/handlers/handlers_test.go
index bfa37d5ce..c11c9031a 100644
--- a/backend/pkg/api/handlers/handlers_test.go
+++ b/backend/pkg/api/handlers/handlers_test.go
@@ -163,7 +163,6 @@ func setup() {
 
 func TestInternalGetProductSummaryHandler(t *testing.T) {
 	code, _, body := ts.get(t, "/api/i/product-summary")
-
 	assert.Equal(t, http.StatusOK, code)
 
 	respData := api_types.InternalGetProductSummaryResponse{}
@@ -225,7 +224,7 @@ func TestInternalLoginHandler(t *testing.T) {
 func TestInternalSearchHandler(t *testing.T) {
 	// search for validator with index 5
 	code, _, body := ts.post(t, "/api/i/search", bytes.NewBufferString(`{"input":"5","networks":[17000],"types":["validators_by_deposit_ens_name","validators_by_deposit_address","validators_by_withdrawal_ens_name","validators_by_withdrawal_address","validators_by_withdrawal_credential","validator_by_index","validator_by_public_key","validators_by_graffiti"]}`))
-	assert.Equal(t, 200, code)
+	assert.Equal(t, http.StatusOK, code)
 
 	resp := api_types.InternalPostSearchResponse{}
 	err := json.Unmarshal([]byte(body), &resp)
@@ -236,7 +235,7 @@ func TestInternalSearchHandler(t *testing.T) {
 
 	// search for validator by pubkey
 	code, _, body = ts.post(t, "/api/i/search", bytes.NewBufferString(`{"input":"0x9699af2bad9826694a480cb523cbe545dc41db955356b3b0d4871f1cf3e4924ae4132fa8c374a0505ae2076d3d65b3e0","networks":[17000],"types":["validators_by_deposit_ens_name","validators_by_deposit_address","validators_by_withdrawal_ens_name","validators_by_withdrawal_address","validators_by_withdrawal_credential","validator_by_index","validator_by_public_key","validators_by_graffiti"]}`))
-	assert.Equal(t, 200, code)
+	assert.Equal(t, http.StatusOK, code)
 
 	resp = api_types.InternalPostSearchResponse{}
 	err = json.Unmarshal([]byte(body), &resp)
@@ -247,7 +246,7 @@ func TestInternalSearchHandler(t *testing.T) {
 
 	// search for validator by withdawal address
 	code, _, body = ts.post(t, "/api/i/search", bytes.NewBufferString(`{"input":"0x0e5dda855eb1de2a212cd1f62b2a3ee49d20c444","networks":[17000],"types":["validators_by_deposit_ens_name","validators_by_deposit_address","validators_by_withdrawal_ens_name","validators_by_withdrawal_address","validators_by_withdrawal_credential","validator_by_index","validator_by_public_key","validators_by_graffiti"]}`))
-	assert.Equal(t, 200, code)
+	assert.Equal(t, http.StatusOK, code)
 
 	resp = api_types.InternalPostSearchResponse{}
 	err = json.Unmarshal([]byte(body), &resp)
@@ -256,3 +255,32 @@ func TestInternalSearchHandler(t *testing.T) {
 	assert.NotNil(t, resp.Data[0].NumValue, "validator index should not be nil")
 	assert.Greater(t, *resp.Data[0].NumValue, uint64(0), "returned number of validators should be greater than 0")
 }
+
+func TestSlotVizHandler(t *testing.T) {
+	code, _, body := ts.get(t, "/api/i/validator-dashboards/NQ/slot-viz")
+	assert.Equal(t, http.StatusOK, code)
+
+	resp := api_types.GetValidatorDashboardSlotVizResponse{}
+	err := json.Unmarshal([]byte(body), &resp)
+	assert.Nil(t, err, "error unmarshalling response")
+	assert.Equal(t, 4, len(resp.Data), "response data should contain the last 4 epochs")
+
+	headStateCount := 0
+	for _, epoch := range resp.Data {
+
+		if epoch.State == "head" { // count the amount of head epochs returned, should be exactly 1
+			headStateCount++
+		}
+		attestationAssignments := 0
+		assert.Equal(t, 32, len(epoch.Slots), "each epoch should contain 32 slots")
+
+		for _, slot := range epoch.Slots {
+			if slot.Attestations != nil { // count the amount of attestation assignments for each epoch, should be exactly 1
+				attestationAssignments++
+			}
+		}
+
+		assert.Equal(t, attestationAssignments, 1, "epoch should have exactly one attestation assignment")
+	}
+	assert.Equal(t, 1, headStateCount, "one of the last 4 epochs should be in head state")
+}

From 5c715ed14e8f1c8de2d8499af2b4764364cd1bd0 Mon Sep 17 00:00:00 2001
From: peter <1674920+peterbitfly@users.noreply.github.com>
Date: Tue, 3 Sep 2024 15:54:02 +0200
Subject: [PATCH 07/28] chore(api): please linter

---
 backend/pkg/api/handlers/handlers_test.go | 38 ++++++++++++-----------
 1 file changed, 20 insertions(+), 18 deletions(-)

diff --git a/backend/pkg/api/handlers/handlers_test.go b/backend/pkg/api/handlers/handlers_test.go
index c11c9031a..42e928257 100644
--- a/backend/pkg/api/handlers/handlers_test.go
+++ b/backend/pkg/api/handlers/handlers_test.go
@@ -38,7 +38,7 @@ type testServer struct {
 // Implement a get() method on our custom testServer type. This makes a GET
 // request to a given url path using the test server client, and returns the
 // response status code, headers and body.
-func (ts *testServer) get(t *testing.T, urlPath string) (int, http.Header, string) {
+func (ts *testServer) get(t *testing.T, urlPath string) (int, string) {
 	rs, err := ts.Client().Get(ts.URL + urlPath)
 	if err != nil {
 		t.Fatal(err)
@@ -49,10 +49,10 @@ func (ts *testServer) get(t *testing.T, urlPath string) (int, http.Header, strin
 		t.Fatal(err)
 	}
 	bytes.TrimSpace(body)
-	return rs.StatusCode, rs.Header, string(body)
+	return rs.StatusCode, string(body)
 }
 
-func (ts *testServer) post(t *testing.T, urlPath string, data io.Reader) (int, http.Header, string) {
+func (ts *testServer) post(t *testing.T, urlPath string, data io.Reader) (int, string) {
 	rs, err := ts.Client().Post(ts.URL+urlPath, "application/json", data)
 	if err != nil {
 		t.Fatal(err)
@@ -63,7 +63,7 @@ func (ts *testServer) post(t *testing.T, urlPath string, data io.Reader) (int, h
 		t.Fatal(err)
 	}
 	bytes.TrimSpace(body)
-	return rs.StatusCode, rs.Header, string(body)
+	return rs.StatusCode, string(body)
 }
 
 func (ts *testServer) parseErrorResonse(t *testing.T, body string) api_types.ApiErrorResponse {
@@ -88,7 +88,10 @@ func TestMain(m *testing.M) {
 func teardown() {
 	dataAccessor.Close()
 	ts.Close()
-	postgres.Stop()
+	err := postgres.Stop()
+	if err != nil {
+		log.Error(err, "error stopping embedded postgres", 0)
+	}
 }
 
 func setup() {
@@ -162,7 +165,7 @@ func setup() {
 }
 
 func TestInternalGetProductSummaryHandler(t *testing.T) {
-	code, _, body := ts.get(t, "/api/i/product-summary")
+	code, body := ts.get(t, "/api/i/product-summary")
 	assert.Equal(t, http.StatusOK, code)
 
 	respData := api_types.InternalGetProductSummaryResponse{}
@@ -175,7 +178,7 @@ func TestInternalGetProductSummaryHandler(t *testing.T) {
 }
 
 func TestInternalGetLatestStateHandler(t *testing.T) {
-	code, _, body := ts.get(t, "/api/i/latest-state")
+	code, body := ts.get(t, "/api/i/latest-state")
 	assert.Equal(t, http.StatusOK, code)
 
 	respData := api_types.InternalGetLatestStateResponse{}
@@ -187,23 +190,23 @@ func TestInternalGetLatestStateHandler(t *testing.T) {
 
 func TestInternalLoginHandler(t *testing.T) {
 	// login with email in wrong format
-	code, _, body := ts.post(t, "/api/i/login", bytes.NewBuffer([]byte(`{"email": "admin", "password": "admin"}`)))
+	code, body := ts.post(t, "/api/i/login", bytes.NewBuffer([]byte(`{"email": "admin", "password": "admin"}`)))
 	assert.Equal(t, http.StatusBadRequest, code)
 	resp := ts.parseErrorResonse(t, body)
 	assert.Equal(t, "email: given value 'admin' has incorrect format", resp.Error, "unexpected error message")
 
 	// login with correct user and wrong password
-	code, _, body = ts.post(t, "/api/i/login", bytes.NewBufferString(`{"email": "admin@admin.com", "password": "wrong"}`))
+	code, body = ts.post(t, "/api/i/login", bytes.NewBufferString(`{"email": "admin@admin.com", "password": "wrong"}`))
 	assert.Equal(t, http.StatusUnauthorized, code, "login should not be successful")
 	resp = ts.parseErrorResonse(t, body)
 	assert.Equal(t, "unauthorized: invalid email or password", resp.Error, "unexpected error message")
 
 	// login with correct user and password
-	code, _, _ = ts.post(t, "/api/i/login", bytes.NewBufferString(`{"email": "admin@admin.com", "password": "admin"}`))
+	code, _ = ts.post(t, "/api/i/login", bytes.NewBufferString(`{"email": "admin@admin.com", "password": "admin"}`))
 	assert.Equal(t, http.StatusOK, code, "login should be successful")
 
 	// check if user is logged in and has a valid session
-	code, _, body = ts.get(t, "/api/i/users/me")
+	code, body = ts.get(t, "/api/i/users/me")
 	assert.Equal(t, http.StatusOK, code, "call to users/me should be successful")
 
 	meResponse := &api_types.InternalGetUserInfoResponse{}
@@ -213,17 +216,17 @@ func TestInternalLoginHandler(t *testing.T) {
 	assert.Equal(t, meResponse.Data.Email, "a***n@a***n.com", "email should be a***n@a***n.com")
 
 	// check if logout works
-	code, _, _ = ts.post(t, "/api/i/logout", bytes.NewBufferString(``))
+	code, _ = ts.post(t, "/api/i/logout", bytes.NewBufferString(``))
 	assert.Equal(t, http.StatusOK, code, "logout should be successful")
 
 	// check if user is logged out
-	code, _, _ = ts.get(t, "/api/i/users/me")
+	code, _ = ts.get(t, "/api/i/users/me")
 	assert.Equal(t, http.StatusUnauthorized, code, "call to users/me should be unauthorized")
 }
 
 func TestInternalSearchHandler(t *testing.T) {
 	// search for validator with index 5
-	code, _, body := ts.post(t, "/api/i/search", bytes.NewBufferString(`{"input":"5","networks":[17000],"types":["validators_by_deposit_ens_name","validators_by_deposit_address","validators_by_withdrawal_ens_name","validators_by_withdrawal_address","validators_by_withdrawal_credential","validator_by_index","validator_by_public_key","validators_by_graffiti"]}`))
+	code, body := ts.post(t, "/api/i/search", bytes.NewBufferString(`{"input":"5","networks":[17000],"types":["validators_by_deposit_ens_name","validators_by_deposit_address","validators_by_withdrawal_ens_name","validators_by_withdrawal_address","validators_by_withdrawal_credential","validator_by_index","validator_by_public_key","validators_by_graffiti"]}`))
 	assert.Equal(t, http.StatusOK, code)
 
 	resp := api_types.InternalPostSearchResponse{}
@@ -234,7 +237,7 @@ func TestInternalSearchHandler(t *testing.T) {
 	assert.Equal(t, uint64(5), *resp.Data[0].NumValue, "validator index should be 5")
 
 	// search for validator by pubkey
-	code, _, body = ts.post(t, "/api/i/search", bytes.NewBufferString(`{"input":"0x9699af2bad9826694a480cb523cbe545dc41db955356b3b0d4871f1cf3e4924ae4132fa8c374a0505ae2076d3d65b3e0","networks":[17000],"types":["validators_by_deposit_ens_name","validators_by_deposit_address","validators_by_withdrawal_ens_name","validators_by_withdrawal_address","validators_by_withdrawal_credential","validator_by_index","validator_by_public_key","validators_by_graffiti"]}`))
+	code, body = ts.post(t, "/api/i/search", bytes.NewBufferString(`{"input":"0x9699af2bad9826694a480cb523cbe545dc41db955356b3b0d4871f1cf3e4924ae4132fa8c374a0505ae2076d3d65b3e0","networks":[17000],"types":["validators_by_deposit_ens_name","validators_by_deposit_address","validators_by_withdrawal_ens_name","validators_by_withdrawal_address","validators_by_withdrawal_credential","validator_by_index","validator_by_public_key","validators_by_graffiti"]}`))
 	assert.Equal(t, http.StatusOK, code)
 
 	resp = api_types.InternalPostSearchResponse{}
@@ -245,7 +248,7 @@ func TestInternalSearchHandler(t *testing.T) {
 	assert.Equal(t, uint64(5), *resp.Data[0].NumValue, "validator index should be 5")
 
 	// search for validator by withdawal address
-	code, _, body = ts.post(t, "/api/i/search", bytes.NewBufferString(`{"input":"0x0e5dda855eb1de2a212cd1f62b2a3ee49d20c444","networks":[17000],"types":["validators_by_deposit_ens_name","validators_by_deposit_address","validators_by_withdrawal_ens_name","validators_by_withdrawal_address","validators_by_withdrawal_credential","validator_by_index","validator_by_public_key","validators_by_graffiti"]}`))
+	code, body = ts.post(t, "/api/i/search", bytes.NewBufferString(`{"input":"0x0e5dda855eb1de2a212cd1f62b2a3ee49d20c444","networks":[17000],"types":["validators_by_deposit_ens_name","validators_by_deposit_address","validators_by_withdrawal_ens_name","validators_by_withdrawal_address","validators_by_withdrawal_credential","validator_by_index","validator_by_public_key","validators_by_graffiti"]}`))
 	assert.Equal(t, http.StatusOK, code)
 
 	resp = api_types.InternalPostSearchResponse{}
@@ -257,7 +260,7 @@ func TestInternalSearchHandler(t *testing.T) {
 }
 
 func TestSlotVizHandler(t *testing.T) {
-	code, _, body := ts.get(t, "/api/i/validator-dashboards/NQ/slot-viz")
+	code, body := ts.get(t, "/api/i/validator-dashboards/NQ/slot-viz")
 	assert.Equal(t, http.StatusOK, code)
 
 	resp := api_types.GetValidatorDashboardSlotVizResponse{}
@@ -267,7 +270,6 @@ func TestSlotVizHandler(t *testing.T) {
 
 	headStateCount := 0
 	for _, epoch := range resp.Data {
-
 		if epoch.State == "head" { // count the amount of head epochs returned, should be exactly 1
 			headStateCount++
 		}

From 8933f7c8eace93bdbd6eeeaec6b4f3ea6359f8c5 Mon Sep 17 00:00:00 2001
From: peter <1674920+peterbitfly@users.noreply.github.com>
Date: Wed, 4 Sep 2024 09:41:53 +0000
Subject: [PATCH 08/28] feat(api): add integration test workflow

---
 .../workflows/backend-integration-test.yml    | 44 +++++++++++++++++++
 1 file changed, 44 insertions(+)
 create mode 100644 .github/workflows/backend-integration-test.yml

diff --git a/.github/workflows/backend-integration-test.yml b/.github/workflows/backend-integration-test.yml
new file mode 100644
index 000000000..12da9f685
--- /dev/null
+++ b/.github/workflows/backend-integration-test.yml
@@ -0,0 +1,44 @@
+name: Backend-Linter
+
+on:
+  push:
+    paths:
+      - 'backend/**'
+    branches:
+      - main
+      - staging
+      - BEDS-401/api_integration_tests
+  pull_request:
+    paths:
+      - 'backend/**'
+    branches:
+      - '*'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+  pull-requests: read
+  checks: write
+
+jobs:
+  build:
+    name: lint
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-go@v4
+        with:
+          go-version: '1.23'
+          cache: false
+      - name: golangci-lint
+        uses: golangci/golangci-lint-action@v3
+        with:
+          version: v1.60.1
+          working-directory: backend
+          args: --timeout=5m
+
+
+

From 89fae485a5ec482db7280a55f6aaf89fdd7e55c9 Mon Sep 17 00:00:00 2001
From: peter <1674920+peterbitfly@users.noreply.github.com>
Date: Wed, 4 Sep 2024 09:45:02 +0000
Subject: [PATCH 09/28] feat(api): update test workflow

---
 .github/workflows/backend-integration-test.yml | 16 ++++++----------
 1 file changed, 6 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/backend-integration-test.yml b/.github/workflows/backend-integration-test.yml
index 12da9f685..1a40f54b5 100644
--- a/.github/workflows/backend-integration-test.yml
+++ b/.github/workflows/backend-integration-test.yml
@@ -1,9 +1,9 @@
-name: Backend-Linter
+name: Backend-Integration-Test
 
 on:
   push:
-    paths:
-      - 'backend/**'
+    # paths:
+    #   - 'backend/**'
     branches:
       - main
       - staging
@@ -25,7 +25,7 @@ permissions:
 
 jobs:
   build:
-    name: lint
+    name: integration-test
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
@@ -33,12 +33,8 @@ jobs:
         with:
           go-version: '1.23'
           cache: false
-      - name: golangci-lint
-        uses: golangci/golangci-lint-action@v3
-        with:
-          version: v1.60.1
-          working-directory: backend
-          args: --timeout=5m
+      - name: Display Go version
+        run: go version
 
 
 

From a5ee230c08b23de6bdc0c97d2ec7ceeebfa06f83 Mon Sep 17 00:00:00 2001
From: peter <1674920+peterbitfly@users.noreply.github.com>
Date: Wed, 4 Sep 2024 09:47:38 +0000
Subject: [PATCH 10/28] chore(api): update test workflow

---
 .github/workflows/backend-integration-test.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/backend-integration-test.yml b/.github/workflows/backend-integration-test.yml
index 1a40f54b5..02f893a43 100644
--- a/.github/workflows/backend-integration-test.yml
+++ b/.github/workflows/backend-integration-test.yml
@@ -33,8 +33,8 @@ jobs:
         with:
           go-version: '1.23'
           cache: false
-      - name: Display Go version
-        run: go version
+      - name: Test with the Go CLI
+        run: go test -v ./backend/pkg/api/handlers/*
 
 
 

From f4d224d5affc96137cb4030f6a0202e09f60aabb Mon Sep 17 00:00:00 2001
From: peter <1674920+peterbitfly@users.noreply.github.com>
Date: Wed, 4 Sep 2024 09:49:54 +0000
Subject: [PATCH 11/28] chore(api): update test workflow

---
 .github/workflows/backend-integration-test.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/backend-integration-test.yml b/.github/workflows/backend-integration-test.yml
index 02f893a43..e6d509251 100644
--- a/.github/workflows/backend-integration-test.yml
+++ b/.github/workflows/backend-integration-test.yml
@@ -32,8 +32,9 @@ jobs:
       - uses: actions/setup-go@v4
         with:
           go-version: '1.23'
-          cache: false
+          go-version-file: 'backend/go.mod'
       - name: Test with the Go CLI
+        working-directory: backend
         run: go test -v ./backend/pkg/api/handlers/*
 
 

From ecba3dd3770e165dcf5a0948b26d2f1ec52b0ebb Mon Sep 17 00:00:00 2001
From: peter <1674920+peterbitfly@users.noreply.github.com>
Date: Wed, 4 Sep 2024 09:50:40 +0000
Subject: [PATCH 12/28] chore: update test workflow

---
 .github/workflows/backend-integration-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/backend-integration-test.yml b/.github/workflows/backend-integration-test.yml
index e6d509251..41c3428cd 100644
--- a/.github/workflows/backend-integration-test.yml
+++ b/.github/workflows/backend-integration-test.yml
@@ -35,7 +35,7 @@ jobs:
           go-version-file: 'backend/go.mod'
       - name: Test with the Go CLI
         working-directory: backend
-        run: go test -v ./backend/pkg/api/handlers/*
+        run: go test -v ./pkg/api/handlers/*
 
 
 

From 6e281708afdcc47465c3c0e72566ef345520f4c5 Mon Sep 17 00:00:00 2001
From: peter <1674920+peterbitfly@users.noreply.github.com>
Date: Wed, 4 Sep 2024 09:53:49 +0000
Subject: [PATCH 13/28] chore: update test workflow

---
 .github/workflows/backend-integration-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/backend-integration-test.yml b/.github/workflows/backend-integration-test.yml
index 41c3428cd..1c894b12c 100644
--- a/.github/workflows/backend-integration-test.yml
+++ b/.github/workflows/backend-integration-test.yml
@@ -35,7 +35,7 @@ jobs:
           go-version-file: 'backend/go.mod'
       - name: Test with the Go CLI
         working-directory: backend
-        run: go test -v ./pkg/api/handlers/*
+        run: go test -v ./pkg/api/handlers/* .config "${{ secrets.CI_CONFIG_PATH }}"
 
 
 

From 1c4cc5a885fa4426f13f5f6f8b54b77fd57faff3 Mon Sep 17 00:00:00 2001
From: peter <1674920+peterbitfly@users.noreply.github.com>
Date: Wed, 4 Sep 2024 09:55:23 +0000
Subject: [PATCH 14/28] chore(api): update test workflow

---
 .github/workflows/backend-integration-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/backend-integration-test.yml b/.github/workflows/backend-integration-test.yml
index 1c894b12c..501468ce9 100644
--- a/.github/workflows/backend-integration-test.yml
+++ b/.github/workflows/backend-integration-test.yml
@@ -35,7 +35,7 @@ jobs:
           go-version-file: 'backend/go.mod'
       - name: Test with the Go CLI
         working-directory: backend
-        run: go test -v ./pkg/api/handlers/* .config "${{ secrets.CI_CONFIG_PATH }}"
+        run: go test -v ./pkg/api/handlers/* -config "${{ secrets.CI_CONFIG_PATH }}"
 
 
 

From f5a3cdcfbea44cfb7238843bc06e8c0a41009595 Mon Sep 17 00:00:00 2001
From: peter <1674920+peterbitfly@users.noreply.github.com>
Date: Wed, 4 Sep 2024 10:07:39 +0000
Subject: [PATCH 15/28] chore: update test workflow

---
 .github/workflows/backend-integration-test.yml | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/.github/workflows/backend-integration-test.yml b/.github/workflows/backend-integration-test.yml
index 501468ce9..c717a7cb6 100644
--- a/.github/workflows/backend-integration-test.yml
+++ b/.github/workflows/backend-integration-test.yml
@@ -1,5 +1,4 @@
 name: Backend-Integration-Test
-
 on:
   push:
     # paths:
@@ -26,7 +25,7 @@ permissions:
 jobs:
   build:
     name: integration-test
-    runs-on: ubuntu-latest
+    runs-on: self-hosted
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v4

From 8e127a71732336e797bb91e69a0d1fa14cc637d5 Mon Sep 17 00:00:00 2001
From: peter <1674920+peterbitfly@users.noreply.github.com>
Date: Wed, 4 Sep 2024 10:29:56 +0000
Subject: [PATCH 16/28] chore: enable backend integration tests on push to main
 and staging branches

---
 .github/workflows/backend-integration-test.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/backend-integration-test.yml b/.github/workflows/backend-integration-test.yml
index c717a7cb6..a4505308a 100644
--- a/.github/workflows/backend-integration-test.yml
+++ b/.github/workflows/backend-integration-test.yml
@@ -1,8 +1,8 @@
 name: Backend-Integration-Test
 on:
   push:
-    # paths:
-    #   - 'backend/**'
+    paths:
+      - 'backend/**'
     branches:
       - main
       - staging

From 97a53920fa4cf2ad94f91163a3c846778819247e Mon Sep 17 00:00:00 2001
From: peter <1674920+peterbitfly@users.noreply.github.com>
Date: Wed, 4 Sep 2024 12:38:24 +0200
Subject: [PATCH 17/28] refactor: change database connection log level to debug

---
 backend/pkg/commons/db/db.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/backend/pkg/commons/db/db.go b/backend/pkg/commons/db/db.go
index ee046b57d..bc1489514 100644
--- a/backend/pkg/commons/db/db.go
+++ b/backend/pkg/commons/db/db.go
@@ -117,7 +117,7 @@ func MustInitDB(writer *types.DatabaseConfig, reader *types.DatabaseConfig, driv
 		}
 	}
 
-	log.Infof("connecting to %s database %s:%s/%s as writer with %d/%d max open/idle connections", databaseBrand, writer.Host, writer.Port, writer.Name, writer.MaxOpenConns, writer.MaxIdleConns)
+	log.Debugf("connecting to %s database %s:%s/%s as writer with %d/%d max open/idle connections", databaseBrand, writer.Host, writer.Port, writer.Name, writer.MaxOpenConns, writer.MaxIdleConns)
 	dbConnWriter, err := sqlx.Open(driverName, fmt.Sprintf("%s://%s:%s@%s/%s?%s", databaseBrand, writer.Username, writer.Password, net.JoinHostPort(writer.Host, writer.Port), writer.Name, sslParam))
 	if err != nil {
 		log.Fatal(err, "error getting Connection Writer database", 0)
@@ -147,7 +147,7 @@ func MustInitDB(writer *types.DatabaseConfig, reader *types.DatabaseConfig, driv
 		}
 	}
 
-	log.Infof("connecting to %s database %s:%s/%s as reader with %d/%d max open/idle connections", databaseBrand, reader.Host, reader.Port, reader.Name, reader.MaxOpenConns, reader.MaxIdleConns)
+	log.Debugf("connecting to %s database %s:%s/%s as reader with %d/%d max open/idle connections", databaseBrand, reader.Host, reader.Port, reader.Name, reader.MaxOpenConns, reader.MaxIdleConns)
 	dbConnReader, err := sqlx.Open(driverName, fmt.Sprintf("%s://%s:%s@%s/%s?%s", databaseBrand, reader.Username, reader.Password, net.JoinHostPort(reader.Host, reader.Port), reader.Name, sslParam))
 	if err != nil {
 		log.Fatal(err, "error getting Connection Reader database", 0)

From 603ecc6cc11a316e79721c0532b9480649a12543 Mon Sep 17 00:00:00 2001
From: peter <1674920+peterbitfly@users.noreply.github.com>
Date: Wed, 4 Sep 2024 12:06:36 +0000
Subject: [PATCH 18/28] chore(api): code review comments

---
 .../workflows/backend-integration-test.yml    |   2 +-
 backend/go.mod                                |   2 +-
 .../handlers_test.go => api_test.go}          | 153 ++++++++++++------
 3 files changed, 102 insertions(+), 55 deletions(-)
 rename backend/pkg/api/{handlers/handlers_test.go => api_test.go} (69%)

diff --git a/.github/workflows/backend-integration-test.yml b/.github/workflows/backend-integration-test.yml
index a4505308a..4b4d68beb 100644
--- a/.github/workflows/backend-integration-test.yml
+++ b/.github/workflows/backend-integration-test.yml
@@ -34,7 +34,7 @@ jobs:
           go-version-file: 'backend/go.mod'
       - name: Test with the Go CLI
         working-directory: backend
-        run: go test -v ./pkg/api/handlers/* -config "${{ secrets.CI_CONFIG_PATH }}"
+        run: go test ./pkg/api/... -config "${{ secrets.CI_CONFIG_PATH }}"
 
 
 
diff --git a/backend/go.mod b/backend/go.mod
index 8e2ae391c..fe53e1c8e 100644
--- a/backend/go.mod
+++ b/backend/go.mod
@@ -70,7 +70,6 @@ require (
 	github.com/xeipuuv/gojsonschema v1.2.0
 	golang.org/x/crypto v0.19.0
 	golang.org/x/exp v0.0.0-20240213143201-ec583247a57a
-	golang.org/x/net v0.21.0
 	golang.org/x/sync v0.6.0
 	golang.org/x/text v0.14.0
 	golang.org/x/time v0.5.0
@@ -236,6 +235,7 @@ require (
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.27.0 // indirect
 	golang.org/x/mod v0.15.0 // indirect
+	golang.org/x/net v0.21.0 // indirect
 	golang.org/x/oauth2 v0.17.0 // indirect
 	golang.org/x/sys v0.17.0 // indirect
 	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
diff --git a/backend/pkg/api/handlers/handlers_test.go b/backend/pkg/api/api_test.go
similarity index 69%
rename from backend/pkg/api/handlers/handlers_test.go
rename to backend/pkg/api/api_test.go
index 42e928257..af4619491 100644
--- a/backend/pkg/api/handlers/handlers_test.go
+++ b/backend/pkg/api/api_test.go
@@ -1,4 +1,4 @@
-package handlers_test
+package api_test
 
 import (
 	"bytes"
@@ -35,25 +35,14 @@ type testServer struct {
 	*httptest.Server
 }
 
-// Implement a get() method on our custom testServer type. This makes a GET
-// request to a given url path using the test server client, and returns the
-// response status code, headers and body.
-func (ts *testServer) get(t *testing.T, urlPath string) (int, string) {
-	rs, err := ts.Client().Get(ts.URL + urlPath)
+func (ts *testServer) request(t *testing.T, method, urlPath string, data io.Reader) (int, string) {
+	req, err := http.NewRequest(method, ts.URL+urlPath, data)
 	if err != nil {
 		t.Fatal(err)
 	}
-	defer rs.Body.Close()
-	body, err := io.ReadAll(rs.Body)
-	if err != nil {
-		t.Fatal(err)
-	}
-	bytes.TrimSpace(body)
-	return rs.StatusCode, string(body)
-}
+	req.Header.Set("Content-Type", "application/json")
 
-func (ts *testServer) post(t *testing.T, urlPath string, data io.Reader) (int, string) {
-	rs, err := ts.Client().Post(ts.URL+urlPath, "application/json", data)
+	rs, err := ts.Client().Do(req)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -65,6 +54,12 @@ func (ts *testServer) post(t *testing.T, urlPath string, data io.Reader) (int, s
 	bytes.TrimSpace(body)
 	return rs.StatusCode, string(body)
 }
+func (ts *testServer) get(t *testing.T, urlPath string) (int, string) {
+	return ts.request(t, http.MethodGet, urlPath, nil)
+}
+func (ts *testServer) post(t *testing.T, urlPath string, data io.Reader) (int, string) {
+	return ts.request(t, http.MethodPost, urlPath, data)
+}
 
 func (ts *testServer) parseErrorResonse(t *testing.T, body string) api_types.ApiErrorResponse {
 	resp := api_types.ApiErrorResponse{}
@@ -115,7 +110,7 @@ func setup() {
 		log.Fatal(err, "error connection to test db", 0)
 	}
 
-	if err := goose.Up(tempDb.DB, "../../../pkg/commons/db/migrations/postgres"); err != nil {
+	if err := goose.Up(tempDb.DB, "../../pkg/commons/db/migrations/postgres"); err != nil {
 		log.Fatal(err, "error running migrations", 0)
 	}
 
@@ -189,44 +184,64 @@ func TestInternalGetLatestStateHandler(t *testing.T) {
 }
 
 func TestInternalLoginHandler(t *testing.T) {
-	// login with email in wrong format
-	code, body := ts.post(t, "/api/i/login", bytes.NewBuffer([]byte(`{"email": "admin", "password": "admin"}`)))
-	assert.Equal(t, http.StatusBadRequest, code)
-	resp := ts.parseErrorResonse(t, body)
-	assert.Equal(t, "email: given value 'admin' has incorrect format", resp.Error, "unexpected error message")
-
-	// login with correct user and wrong password
-	code, body = ts.post(t, "/api/i/login", bytes.NewBufferString(`{"email": "admin@admin.com", "password": "wrong"}`))
-	assert.Equal(t, http.StatusUnauthorized, code, "login should not be successful")
-	resp = ts.parseErrorResonse(t, body)
-	assert.Equal(t, "unauthorized: invalid email or password", resp.Error, "unexpected error message")
-
-	// login with correct user and password
-	code, _ = ts.post(t, "/api/i/login", bytes.NewBufferString(`{"email": "admin@admin.com", "password": "admin"}`))
-	assert.Equal(t, http.StatusOK, code, "login should be successful")
-
-	// check if user is logged in and has a valid session
-	code, body = ts.get(t, "/api/i/users/me")
-	assert.Equal(t, http.StatusOK, code, "call to users/me should be successful")
-
-	meResponse := &api_types.InternalGetUserInfoResponse{}
-	err := json.Unmarshal([]byte(body), meResponse)
-	assert.Nil(t, err, "error unmarshalling response")
-	// check if email is censored
-	assert.Equal(t, meResponse.Data.Email, "a***n@a***n.com", "email should be a***n@a***n.com")
-
-	// check if logout works
-	code, _ = ts.post(t, "/api/i/logout", bytes.NewBufferString(``))
-	assert.Equal(t, http.StatusOK, code, "logout should be successful")
-
-	// check if user is logged out
-	code, _ = ts.get(t, "/api/i/users/me")
-	assert.Equal(t, http.StatusUnauthorized, code, "call to users/me should be unauthorized")
+	t.Run("login with email in wrong format", func(t *testing.T) {
+		code, body := ts.post(t, "/api/i/login", bytes.NewBuffer([]byte(`{"email": "admin", "password": "admin"}`)))
+		assert.Equal(t, http.StatusBadRequest, code)
+		resp := ts.parseErrorResonse(t, body)
+		assert.Equal(t, "email: given value 'admin' has incorrect format", resp.Error, "unexpected error message")
+	})
+	t.Run("login with correct user and wrong password", func(t *testing.T) {
+		code, body := ts.post(t, "/api/i/login", bytes.NewBufferString(`{"email": "admin@admin.com", "password": "wrong"}`))
+		assert.Equal(t, http.StatusUnauthorized, code, "login should not be successful")
+		resp := ts.parseErrorResonse(t, body)
+		assert.Equal(t, "unauthorized: invalid email or password", resp.Error, "unexpected error message")
+	})
+
+	t.Run("login with correct user and password", func(t *testing.T) {
+		code, _ := ts.post(t, "/api/i/login", bytes.NewBufferString(`{"email": "admin@admin.com", "password": "admin"}`))
+		assert.Equal(t, http.StatusOK, code, "login should be successful")
+	})
+
+	t.Run("check if user is logged in and has a valid session", func(t *testing.T) {
+		code, body := ts.get(t, "/api/i/users/me")
+		assert.Equal(t, http.StatusOK, code, "call to users/me should be successful")
+
+		meResponse := &api_types.InternalGetUserInfoResponse{}
+		err := json.Unmarshal([]byte(body), meResponse)
+		assert.Nil(t, err, "error unmarshalling response")
+		// check if email is censored
+		assert.Equal(t, meResponse.Data.Email, "a***n@a***n.com", "email should be a***n@a***n.com")
+	})
+
+	t.Run("check if logout works", func(t *testing.T) {
+		code, _ := ts.post(t, "/api/i/logout", bytes.NewBufferString(``))
+		assert.Equal(t, http.StatusOK, code, "logout should be successful")
+	})
+	t.Run("// check if user is logged out", func(t *testing.T) {
+		code, _ := ts.get(t, "/api/i/users/me")
+		assert.Equal(t, http.StatusUnauthorized, code, "call to users/me should be unauthorized")
+	})
 }
 
 func TestInternalSearchHandler(t *testing.T) {
 	// search for validator with index 5
-	code, body := ts.post(t, "/api/i/search", bytes.NewBufferString(`{"input":"5","networks":[17000],"types":["validators_by_deposit_ens_name","validators_by_deposit_address","validators_by_withdrawal_ens_name","validators_by_withdrawal_address","validators_by_withdrawal_credential","validator_by_index","validator_by_public_key","validators_by_graffiti"]}`))
+	code, body := ts.post(t, "/api/i/search", bytes.NewBufferString(`
+	{
+		"input":"5",
+		"networks":[
+			17000
+		],
+		"types":[
+			"validators_by_deposit_ens_name",
+			"validators_by_deposit_address",
+			"validators_by_withdrawal_ens_name",
+			"validators_by_withdrawal_address",
+			"validators_by_withdrawal_credential",
+			"validator_by_index",
+			"validator_by_public_key",
+			"validators_by_graffiti"
+		]
+	}`))
 	assert.Equal(t, http.StatusOK, code)
 
 	resp := api_types.InternalPostSearchResponse{}
@@ -237,7 +252,23 @@ func TestInternalSearchHandler(t *testing.T) {
 	assert.Equal(t, uint64(5), *resp.Data[0].NumValue, "validator index should be 5")
 
 	// search for validator by pubkey
-	code, body = ts.post(t, "/api/i/search", bytes.NewBufferString(`{"input":"0x9699af2bad9826694a480cb523cbe545dc41db955356b3b0d4871f1cf3e4924ae4132fa8c374a0505ae2076d3d65b3e0","networks":[17000],"types":["validators_by_deposit_ens_name","validators_by_deposit_address","validators_by_withdrawal_ens_name","validators_by_withdrawal_address","validators_by_withdrawal_credential","validator_by_index","validator_by_public_key","validators_by_graffiti"]}`))
+	code, body = ts.post(t, "/api/i/search", bytes.NewBufferString(`
+	{
+		"input":"0x9699af2bad9826694a480cb523cbe545dc41db955356b3b0d4871f1cf3e4924ae4132fa8c374a0505ae2076d3d65b3e0",
+		"networks":[
+			17000
+		],
+		"types":[
+			"validators_by_deposit_ens_name",
+			"validators_by_deposit_address",
+			"validators_by_withdrawal_ens_name",
+			"validators_by_withdrawal_address",
+			"validators_by_withdrawal_credential",
+			"validator_by_index",
+			"validator_by_public_key",
+			"validators_by_graffiti"
+		]
+	}`))
 	assert.Equal(t, http.StatusOK, code)
 
 	resp = api_types.InternalPostSearchResponse{}
@@ -248,7 +279,23 @@ func TestInternalSearchHandler(t *testing.T) {
 	assert.Equal(t, uint64(5), *resp.Data[0].NumValue, "validator index should be 5")
 
 	// search for validator by withdawal address
-	code, body = ts.post(t, "/api/i/search", bytes.NewBufferString(`{"input":"0x0e5dda855eb1de2a212cd1f62b2a3ee49d20c444","networks":[17000],"types":["validators_by_deposit_ens_name","validators_by_deposit_address","validators_by_withdrawal_ens_name","validators_by_withdrawal_address","validators_by_withdrawal_credential","validator_by_index","validator_by_public_key","validators_by_graffiti"]}`))
+	code, body = ts.post(t, "/api/i/search", bytes.NewBufferString(`
+	{
+		"input":"0x0e5dda855eb1de2a212cd1f62b2a3ee49d20c444",
+		"networks":[
+			17000
+		],
+		"types":[
+			"validators_by_deposit_ens_name",
+			"validators_by_deposit_address",
+			"validators_by_withdrawal_ens_name",
+			"validators_by_withdrawal_address",
+			"validators_by_withdrawal_credential",
+			"validator_by_index",
+			"validator_by_public_key",
+			"validators_by_graffiti"
+		]
+	}`))
 	assert.Equal(t, http.StatusOK, code)
 
 	resp = api_types.InternalPostSearchResponse{}

From 73c2653048e16095808b96c6ae4eab99dc44bed2 Mon Sep 17 00:00:00 2001
From: peter <1674920+peterbitfly@users.noreply.github.com>
Date: Wed, 4 Sep 2024 12:11:14 +0000
Subject: [PATCH 19/28] chore: Update backend integration test workflow

---
 .github/workflows/backend-integration-test.yml | 2 --
 1 file changed, 2 deletions(-)

diff --git a/.github/workflows/backend-integration-test.yml b/.github/workflows/backend-integration-test.yml
index 4b4d68beb..89b393b44 100644
--- a/.github/workflows/backend-integration-test.yml
+++ b/.github/workflows/backend-integration-test.yml
@@ -6,7 +6,6 @@ on:
     branches:
       - main
       - staging
-      - BEDS-401/api_integration_tests
   pull_request:
     paths:
       - 'backend/**'
@@ -30,7 +29,6 @@ jobs:
       - uses: actions/checkout@v3
       - uses: actions/setup-go@v4
         with:
-          go-version: '1.23'
           go-version-file: 'backend/go.mod'
       - name: Test with the Go CLI
         working-directory: backend

From d127dbe2ff13d01d2c83405b6fcde882f57e4a7a Mon Sep 17 00:00:00 2001
From: peter <1674920+peterbitfly@users.noreply.github.com>
Date: Wed, 4 Sep 2024 12:14:13 +0000
Subject: [PATCH 20/28] chore: Update backend integration test workflow

---
 .github/workflows/backend-integration-test.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/backend-integration-test.yml b/.github/workflows/backend-integration-test.yml
index 89b393b44..e05ef5c93 100644
--- a/.github/workflows/backend-integration-test.yml
+++ b/.github/workflows/backend-integration-test.yml
@@ -30,6 +30,7 @@ jobs:
       - uses: actions/setup-go@v4
         with:
           go-version-file: 'backend/go.mod'
+          cache-dependency-path: 'backend/go.sum'
       - name: Test with the Go CLI
         working-directory: backend
         run: go test ./pkg/api/... -config "${{ secrets.CI_CONFIG_PATH }}"

From 79632946b2c6388840975aa99da038f4a4c18471 Mon Sep 17 00:00:00 2001
From: peter <1674920+peterbitfly@users.noreply.github.com>
Date: Wed, 4 Sep 2024 12:16:38 +0000
Subject: [PATCH 21/28] chore: Initialize services and update logging in
 service.go

---
 backend/pkg/api/services/service.go | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/backend/pkg/api/services/service.go b/backend/pkg/api/services/service.go
index 872e3391b..8108d9687 100644
--- a/backend/pkg/api/services/service.go
+++ b/backend/pkg/api/services/service.go
@@ -45,8 +45,10 @@ func (s *Services) InitServices() {
 	go s.startEfficiencyDataService(wg)
 	go s.startEmailSenderService(wg)
 
-	wg.Wait()
 	log.Infof("initializing prices...")
 	price.Init(utils.Config.Chain.ClConfig.DepositChainID, utils.Config.Eth1ErigonEndpoint, utils.Config.Frontend.ClCurrency, utils.Config.Frontend.ElCurrency)
 	log.Infof("...prices initialized")
+
+	wg.Wait()
+	log.Infof("...services initialized")
 }

From bf1d9d2c9c2c163458a1d4f070fb058e2a165b93 Mon Sep 17 00:00:00 2001
From: peter <1674920+peterbitfly@users.noreply.github.com>
Date: Wed, 4 Sep 2024 12:17:52 +0000
Subject: [PATCH 22/28] chore: Update backend integration test workflow

---
 .github/workflows/backend-integration-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/backend-integration-test.yml b/.github/workflows/backend-integration-test.yml
index e05ef5c93..162085860 100644
--- a/.github/workflows/backend-integration-test.yml
+++ b/.github/workflows/backend-integration-test.yml
@@ -33,7 +33,7 @@ jobs:
           cache-dependency-path: 'backend/go.sum'
       - name: Test with the Go CLI
         working-directory: backend
-        run: go test ./pkg/api/... -config "${{ secrets.CI_CONFIG_PATH }}"
+        run: go test -v ./pkg/api/... -config "${{ secrets.CI_CONFIG_PATH }}"
 
 
 

From 76a0184e24aec4aa46fa708ef65c817f7130449d Mon Sep 17 00:00:00 2001
From: peter <1674920+peterbitfly@users.noreply.github.com>
Date: Wed, 4 Sep 2024 12:19:58 +0000
Subject: [PATCH 23/28] chore: Add log statement for test setup completion

---
 backend/pkg/api/api_test.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/backend/pkg/api/api_test.go b/backend/pkg/api/api_test.go
index af4619491..baca02684 100644
--- a/backend/pkg/api/api_test.go
+++ b/backend/pkg/api/api_test.go
@@ -71,6 +71,7 @@ func (ts *testServer) parseErrorResonse(t *testing.T, body string) api_types.Api
 
 func TestMain(m *testing.M) {
 	setup()
+	log.Info("test stup completed")
 	code := m.Run()
 	teardown()
 

From 0a71110eda95f09f85ec33c80b5845aea251c1fa Mon Sep 17 00:00:00 2001
From: peter <1674920+peterbitfly@users.noreply.github.com>
Date: Wed, 4 Sep 2024 12:21:28 +0000
Subject: [PATCH 24/28] chore: Update test setup completion log statement in
 api_test.go

---
 backend/pkg/api/api_test.go | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/backend/pkg/api/api_test.go b/backend/pkg/api/api_test.go
index baca02684..dc0df1b87 100644
--- a/backend/pkg/api/api_test.go
+++ b/backend/pkg/api/api_test.go
@@ -71,7 +71,7 @@ func (ts *testServer) parseErrorResonse(t *testing.T, body string) api_types.Api
 
 func TestMain(m *testing.M) {
 	setup()
-	log.Info("test stup completed")
+	log.Info("test setup completed")
 	code := m.Run()
 	teardown()
 
@@ -149,7 +149,9 @@ func setup() {
 
 	log.InfoWithFields(log.Fields{"config": *configPath, "version": version.Version, "commit": version.GitCommit, "chainName": utils.Config.Chain.ClConfig.ConfigName}, "starting")
 
+	log.Info("initializing data access service")
 	dataAccessor = dataaccess.NewDataAccessService(cfg)
+	log.Info("initializing api router")
 	router := api.NewApiRouter(dataAccessor, cfg)
 
 	ts = &testServer{httptest.NewTLSServer(router)}

From f574b4011a2a986d7bfbdc19900201af8ea1a22d Mon Sep 17 00:00:00 2001
From: peter <1674920+peterbitfly@users.noreply.github.com>
Date: Wed, 4 Sep 2024 12:22:48 +0000
Subject: [PATCH 25/28] chore: Initialize services and update logging in
 service.go

---
 backend/pkg/api/data_access/data_access.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/backend/pkg/api/data_access/data_access.go b/backend/pkg/api/data_access/data_access.go
index aa8adead6..d3a5d8ad1 100644
--- a/backend/pkg/api/data_access/data_access.go
+++ b/backend/pkg/api/data_access/data_access.go
@@ -80,9 +80,11 @@ func NewDataAccessService(cfg *types.Config) *DataAccessService {
 	db.BigtableClient = das.bigtable
 	db.PersistentRedisDbClient = das.persistentRedisDbClient
 
+	log.Info("DataAccessService initialized")
 	// Create the services
 	das.services = services.NewServices(das.readerDb, das.writerDb, das.alloyReader, das.alloyWriter, das.clickhouseReader, das.bigtable, das.persistentRedisDbClient)
 
+	log.Info("Services created")
 	// Initialize the services
 	das.services.InitServices()
 

From 6ae9373e528ff20fcce670652856209362dee583 Mon Sep 17 00:00:00 2001
From: peter <1674920+peterbitfly@users.noreply.github.com>
Date: Wed, 4 Sep 2024 12:24:21 +0000
Subject: [PATCH 26/28] chore: Update logging in service.go and initialize
 services

---
 backend/pkg/api/data_access/data_access.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/backend/pkg/api/data_access/data_access.go b/backend/pkg/api/data_access/data_access.go
index d3a5d8ad1..1e1c17ab9 100644
--- a/backend/pkg/api/data_access/data_access.go
+++ b/backend/pkg/api/data_access/data_access.go
@@ -84,7 +84,7 @@ func NewDataAccessService(cfg *types.Config) *DataAccessService {
 	// Create the services
 	das.services = services.NewServices(das.readerDb, das.writerDb, das.alloyReader, das.alloyWriter, das.clickhouseReader, das.bigtable, das.persistentRedisDbClient)
 
-	log.Info("Services created")
+	log.Info("Services created_")
 	// Initialize the services
 	das.services.InitServices()
 

From 2254143e9d093882e938ccccf630544f0397c0a7 Mon Sep 17 00:00:00 2001
From: peter <1674920+peterbitfly@users.noreply.github.com>
Date: Wed, 4 Sep 2024 12:29:58 +0000
Subject: [PATCH 27/28] chore: Start data access services in api_test.go
 setup()

---
 backend/pkg/api/api_test.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/backend/pkg/api/api_test.go b/backend/pkg/api/api_test.go
index dc0df1b87..8a9d2c5e0 100644
--- a/backend/pkg/api/api_test.go
+++ b/backend/pkg/api/api_test.go
@@ -151,6 +151,8 @@ func setup() {
 
 	log.Info("initializing data access service")
 	dataAccessor = dataaccess.NewDataAccessService(cfg)
+	dataAccessor.StartDataAccessServices()
+
 	log.Info("initializing api router")
 	router := api.NewApiRouter(dataAccessor, cfg)
 

From fbd9716c9d6fc1cc55c467f43586d793e0bc4f7a Mon Sep 17 00:00:00 2001
From: peter <1674920+peterbitfly@users.noreply.github.com>
Date: Wed, 4 Sep 2024 12:41:38 +0000
Subject: [PATCH 28/28] chore: Refactor backend integration test workflow and
 update test setup in api_test.go

---
 .github/workflows/backend-integration-test.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/backend-integration-test.yml b/.github/workflows/backend-integration-test.yml
index 162085860..e05ef5c93 100644
--- a/.github/workflows/backend-integration-test.yml
+++ b/.github/workflows/backend-integration-test.yml
@@ -33,7 +33,7 @@ jobs:
           cache-dependency-path: 'backend/go.sum'
       - name: Test with the Go CLI
         working-directory: backend
-        run: go test -v ./pkg/api/... -config "${{ secrets.CI_CONFIG_PATH }}"
+        run: go test ./pkg/api/... -config "${{ secrets.CI_CONFIG_PATH }}"