feat(test): mainnet datastreamer test tweaks (#25)

* feat(test): mainnet datastreamer test tweaks

* sequance check, build testimage

---------

Co-authored-by: Anton <[email protected]>

.github/workflows/test-datastream.yml

@@ -0,0 +1,39 @@
+name: Build and Push datastream test
+
+on:
+  workflow_dispatch:
+
+env:
+  DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }}
+  DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+  IMAGE_NAME: gatewayfm/cdk-erigon-datastream-test
+jobs:
+  build-and-push:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v2
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ env.DOCKERHUB_USERNAME }}
+          password: ${{ env.DOCKERHUB_TOKEN }}
+
+      - name: Build and Push Docker Image
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: Dockerfile.test
+          platforms: linux/amd64,linux/arm64
+          push: true
+          tags: ${{ env.IMAGE_NAME }}:${{ github.sha }}
+
+      - name: Verify Image
+        run: docker run --rm ${{ env.IMAGE_NAME }}:${{ github.sha }} --version

Dockerfile.test

@@ -0,0 +1,16 @@
+FROM golang:1.19 AS base
+
+RUN apt-get update \
+&& apt-get -y install make openssh-client ca-certificates && update-ca-certificates
+
+FROM base AS test
+
+WORKDIR /app
+
+ADD go.mod go.mod
+ADD go.sum go.sum
+
+RUN go mod download
+ADD . .
+
+ENTRYPOINT ["go", "run", "/app/zk/datastream/test/test.go"]

zk/datastream/test/test.go

@@ -5,17 +5,18 @@ import (
 	"strconv"
 
 	"github.com/ledgerwatch/erigon-lib/common"
-	"github.com/ledgerwatch/erigon/zk/datastream"
 	"github.com/ledgerwatch/erigon/zk/datastream/client"
 	"github.com/ledgerwatch/erigon/zk/datastream/test/utils"
 	"github.com/ledgerwatch/erigon/zk/datastream/types"
 	"github.com/ledgerwatch/erigon/zkevm/log"
 )
 
+const dataStreamUrl = "stream.zkevm-rpc.com:6900"
+
 // This code downloads headers and blocks from a datastream server.
 func main() {
 	// Create client
-	c := client.NewClient(datastream.TestDatastreamUrl)
+	c := client.NewClient(dataStreamUrl)
 
 	// Start client (connect to the server)
 	defer c.Stop()
@@ -27,30 +28,35 @@ func main() {
 	bookmark := types.NewL2BlockBookmark(0)
 
 	// Read all entries from server
-	blocksRead, _, _, entriesReadAmount, err := c.ReadEntries(bookmark, 3000)
+	blocksRead, _, _, entriesReadAmount, err := c.ReadEntries(bookmark, 1000000)
 	if err != nil {
 		panic(err)
 	}
 	fmt.Println("Entries read amount: ", entriesReadAmount)
 	fmt.Println("Blocks read amount: ", len(*blocksRead))
 
 	lastGer := common.Hash{}
+	expectedDsBlock := uint64(1)
+
+	var missingBlocks []uint64
+
 	for i, dsBlock := range *blocksRead {
 		if i == 0 {
 			continue
 		}
-		rpcBlock, err := utils.GetBlockByHash(dsBlock.L2Blockhash.String())
-		if err != nil {
-			panic(err)
-		}
 
+		if dsBlock.L2BlockNumber != expectedDsBlock {
+				missingBlocks = append(missingBlocks, dsBlock.L2BlockNumber)
+				log.Error("Missing blocks: %v", missingBlocks)
+			}
+			expectedDsBlock++
+
+		rpcBlock, _ := utils.GetBlockByHash(dsBlock.L2Blockhash.String())
 		match := matchBlocks(dsBlock, rpcBlock, lastGer)
 		if !match {
 			log.Error("Blocks don't match")
-			return
 		}
 		if lastGer.Hex() != dsBlock.GlobalExitRoot.Hex() {
-			fmt.Println("New ger!")
 			lastGer = dsBlock.GlobalExitRoot
 		}
 	}
@@ -98,6 +104,7 @@ func matchBlocks(dsBlock types.FullL2Block, rpcBlock utils.Result, lastGer commo
 			return false
 		}
 	}
+
 	// for i, tx := range dsBlock.L2Txs {
 	// 	if tx..String() != rpcBlock.Transactions[i] {
 	// 		log.Error("Block txs don't match", "blockNumber", dsBlock.L2BlockNumber, "dsBlockTx", tx.String(), "rpcBlockTx", rpcBlock.Transactions[i])

zk/datastream/test/utils/utils.go

@@ -46,11 +46,13 @@ func GetBlockByHash(blockHash string) (Result, error) {
 		return Result{}, err
 	}
 
-	req, err := http.NewRequest("POST", "https://rpc.internal.zkevm-test.net", bytes.NewBuffer(jsonPayload))
+	req, err := http.NewRequest("POST", "https://zkevm-rpc.com", bytes.NewBuffer(jsonPayload))
 	if err != nil {
 		return Result{}, err
 	}
 
+	req.Header.Set("Content-Type", "application/json")
+
 	client := &http.Client{}
 	resp, err := client.Do(req)
 	if err != nil {
@@ -107,7 +109,7 @@ func CompareValuesString(vlockNum, ts string, ger common.Hash) error {
 		return err
 	}
 
-	resp, err := http.Post("https://rpc.internal.zkevm-test.net", "application/json", bytes.NewBuffer(jsonPayload))
+	resp, err := http.Post("https://zkevm-rpc.com", "application/json", bytes.NewBuffer(jsonPayload))
 	if err != nil {
 		fmt.Println("Error sending request:", err)
 		return err