diff --git a/.github/workflows/build-sdks.yml b/.github/workflows/build-sdks.yml index 2069e90b1..91d194090 100644 --- a/.github/workflows/build-sdks.yml +++ b/.github/workflows/build-sdks.yml @@ -27,7 +27,7 @@ jobs: - name: Set up Go 1.x uses: actions/setup-go@v3 with: - go-version: 1.20.4 + go-version: 1.21.5 - name: Clean build run: make clean-mobilesdk @@ -99,7 +99,7 @@ jobs: - name: Set up Go 1.x uses: actions/setup-go@v3 with: - go-version: 1.20.4 + go-version: 1.21.5 - name: Install deps run: | @@ -202,7 +202,7 @@ jobs: - name: Set up Go 1.x uses: actions/setup-go@v3 with: - go-version: 1.20.4 + go-version: 1.21.5 - name: Clean build run: make clean-mobilesdk @@ -274,7 +274,7 @@ jobs: - name: Set up Go 1.x uses: actions/setup-go@v3 with: - go-version: 1.20.4 + go-version: 1.21.5 - name: Install deps run: | @@ -335,10 +335,10 @@ jobs: name: Build-wasm runs-on: [self-hosted, arc-runner] steps: - - name: Set up Go 1.20 - uses: actions/setup-go@v2 + - name: Set up Go 1.x + uses: actions/setup-go@v3 with: - go-version: ^1.20 + go-version: 1.21.5 - name: Checkout uses: actions/checkout@v2 @@ -349,7 +349,7 @@ jobs: sudo apt-get -y install build-essential nghttp2 libnghttp2-dev libssl-dev wget - name: Build - run: docker run --rm -v $PWD:/gosdk -w /gosdk golang:1.20 make wasm-build + run: docker run --rm -v $PWD:/gosdk -w /gosdk golang:1.21.5 make wasm-build - name: 'Upload Artifact' uses: actions/upload-artifact@v3 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 2c25a29f4..233217c20 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -23,25 +23,17 @@ jobs: sudo apt update -y sudo apt -y install build-essential nghttp2 libnghttp2-dev libssl-dev wget - - name: "Setup Go" - shell: 'script --return --quiet --command "bash {0}"' - run: | - [ -f ./https://go.dev/dl/go1.20.3.linux-amd64.tar.gz ] || wget https://go.dev/dl/go1.20.3.linux-amd64.tar.gz - [ -d /usr/local/go ] && sudo rm -rf /usr/local/go - [ -f /usr/local/bin/go ] && sudo rm -rf /usr/local/bin/go - sudo tar -C /usr/local -xzf ./go1.20.3.linux-amd64.tar.gz - - echo "PATH=$PATH:/usr/local/go/bin" >> $GITHUB_ENV - export PATH=$PATH:/usr/local/go/bin - which go - go env + - name: Set up Go 1.x + uses: actions/setup-go@v3 + with: + go-version: 1.21.5 - uses: actions/checkout@v3 - name: golangci-lint uses: golangci/golangci-lint-action@v3 with: - version: v1.52.2 + version: v1.57.1 skip-build-cache: true skip-pkg-cache: true only-new-issues: true @@ -56,18 +48,10 @@ jobs: sudo apt update -y sudo apt -y install build-essential nghttp2 libnghttp2-dev libssl-dev wget - - name: "Setup Go" - shell: 'script --return --quiet --command "bash {0}"' - run: | - [ -f ./https://go.dev/dl/go1.20.3.linux-amd64.tar.gz ] || wget https://go.dev/dl/go1.20.3.linux-amd64.tar.gz - [ -d /usr/local/go ] && sudo rm -rf /usr/local/go - [ -f /usr/local/bin/go ] && sudo rm -rf /usr/local/bin/go - sudo tar -C /usr/local -xzf ./go1.20.3.linux-amd64.tar.gz - - echo "PATH=$PATH:/usr/local/go/bin" >> $GITHUB_ENV - export PATH=$PATH:/usr/local/go/bin - which go - go env + - name: Set up Go 1.x + uses: actions/setup-go@v3 + with: + go-version: 1.21.5 - name: Install deps run: | @@ -248,14 +232,14 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Setup go 1.20 - uses: actions/setup-go@v2 + - name: Set up Go 1.x + uses: actions/setup-go@v3 with: - go-version: '1.20' # The Go version to download (if necessary) and use. + go-version: 1.21.5 - uses: actions/setup-node@v2 with: - node-version: '14' + node-version: 20.11.1 - name: Setup PATH for wasm run: echo "${{env.GOROOT}}/misc/wasm" >> $GITHUB_PATH diff --git a/core/sys/fs_mem.go b/core/sys/fs_mem.go index 369504a45..3d9f01823 100644 --- a/core/sys/fs_mem.go +++ b/core/sys/fs_mem.go @@ -310,7 +310,7 @@ func (f *MemChanFile) Write(p []byte) (n int, err error) { f.Buffer <- data } else { if cap(f.data) == 0 { - f.data = make([]byte, 0, f.ChunkWriteSize) + f.data = make([]byte, 0, len(p)) } f.data = append(f.data, p...) } @@ -326,7 +326,7 @@ func (f *MemChanFile) Sync() error { } f.Buffer <- f.data[current:end] } - f.data = make([]byte, 0, f.ChunkWriteSize) + f.data = nil return nil } func (f *MemChanFile) Seek(offset int64, whence int) (ret int64, err error) { diff --git a/go.mod b/go.mod index 28e7e6799..c1280b8db 100644 --- a/go.mod +++ b/go.mod @@ -2,8 +2,6 @@ module github.com/0chain/gosdk go 1.21 -toolchain go1.21.5 - require ( github.com/0chain/common v0.0.6-0.20230127095721-8df4d1d72565 github.com/0chain/errors v1.0.3 @@ -15,7 +13,7 @@ require ( github.com/google/uuid v1.3.0 github.com/gorilla/handlers v1.5.1 github.com/gorilla/mux v1.8.0 - github.com/h2non/filetype v1.1.3 + github.com/h2non/filetype v1.1.4-0.20231228185113-6469358c2bcb github.com/hashicorp/go-retryablehttp v0.7.2 github.com/hashicorp/golang-lru/v2 v2.0.1 github.com/herumi/bls-go-binary v1.31.0 diff --git a/go.sum b/go.sum index 616bf4e1c..4e8909d54 100644 --- a/go.sum +++ b/go.sum @@ -283,8 +283,8 @@ github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg= -github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY= +github.com/h2non/filetype v1.1.4-0.20231228185113-6469358c2bcb h1:GlQyMv2C48qmfPItvAXFoyN341Swxp9JNVeUZxnmbJw= +github.com/h2non/filetype v1.1.4-0.20231228185113-6469358c2bcb/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY= github.com/hashicorp/go-bexpr v0.1.10 h1:9kuI5PFotCboP3dkDYFr/wi0gg0QVbSNz5oFRpxn4uE= github.com/hashicorp/go-bexpr v0.1.10/go.mod h1:oxlubA2vC/gFVfX1A6JGp7ls7uCDlfJn732ehYYg+g0= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= diff --git a/wasmsdk/bridge.go b/wasmsdk/bridge.go index fa4daab1e..b50652216 100644 --- a/wasmsdk/bridge.go +++ b/wasmsdk/bridge.go @@ -2,6 +2,7 @@ package main import ( "context" + "encoding/base64" "encoding/json" "path" "strconv" @@ -9,6 +10,7 @@ import ( "github.com/0chain/gosdk/zcnbridge" "github.com/0chain/gosdk/zcnbridge/errors" + "github.com/0chain/gosdk/zcnbridge/log" "github.com/0chain/gosdk/zcnbridge/transaction" "github.com/0chain/gosdk/zcnbridge/wallet" "github.com/0chain/gosdk/zcncore" @@ -124,6 +126,7 @@ func getNotProcessedWZCNBurnEvents() string { return errors.New("getNotProcessedWZCNBurnEvents", "failed to retreive last ZCN processed mint nonce").Error() } + log.Logger.Debug("MintNonce = " + strconv.Itoa(int(mintNonce))) burnEvents, err := bridge.QueryEthereumBurnEvents(strconv.Itoa(int(mintNonce))) if err != nil { return errors.Wrap("getNotProcessedWZCNBurnEvents", "failed to retreive WZCN burn events", err).Error() @@ -171,10 +174,60 @@ func getNotProcessedZCNBurnTickets() string { return string(result) } +// estimateBurnWZCNGasAmount performs gas amount estimation for the given burn wzcn transaction. +func estimateBurnWZCNGasAmount(from, to string, amountTokens int) string { // nolint:golint,unused + estimateBurnWZCNGasAmountResponse, err := bridge.EstimateBurnWZCNGasAmount( + context.Background(), from, to, amountTokens) + if err != nil { + return errors.Wrap("estimateBurnWZCNGasAmount", "failed to estimate gas amount", err).Error() + } + + var result []byte + result, err = json.Marshal(estimateBurnWZCNGasAmountResponse) + if err != nil { + return errors.Wrap("estimateBurnWZCNGasAmount", "failed to marshal gas amount estimation result", err).Error() + } + + return string(result) +} + +// estimateMintWZCNGasAmount performs gas amount estimation for the given mint wzcn transaction. +func estimateMintWZCNGasAmount(from, to, zcnTransaction string, amountToken, nonce int64, signaturesRaw []string) string { // nolint:golint,unused + var signaturesBytes [][]byte + + var ( + signatureBytes []byte + err error + ) + + for _, signature := range signaturesRaw { + signatureBytes, err = base64.StdEncoding.DecodeString(signature) + if err != nil { + return errors.Wrap("estimateMintWZCNGasAmount", "failed to convert raw signature into bytes", err).Error() + } + + signaturesBytes = append(signaturesBytes, signatureBytes) + } + + estimateMintWZCNGasAmountResponse, err := bridge.EstimateMintWZCNGasAmount( + context.Background(), from, to, zcnTransaction, amountToken, nonce, signaturesBytes) + if err != nil { + return errors.Wrap("estimateMintWZCNGasAmount", "failed to estimate gas amount", err).Error() + } + + var result []byte + result, err = json.Marshal(estimateMintWZCNGasAmountResponse) + if err != nil { + return errors.Wrap("estimateMintWZCNGasAmount", "failed to marshal gas amount estimation result", err).Error() + } + + return string(result) +} + // estimateGasPrice performs gas estimation for the given transaction using Alchemy enhanced API returning // approximate final gas fee. -func estimateGasPrice(from, to string, value int64) string { // nolint:golint,unused - estimateGasPriceResponse, err := bridge.EstimateGasPrice(context.Background(), from, to, value) +func estimateGasPrice() string { // nolint:golint,unused + estimateGasPriceResponse, err := bridge.EstimateGasPrice(context.Background()) if err != nil { return errors.Wrap("estimateGasPrice", "failed to estimate gas price", err).Error() } diff --git a/wasmsdk/proxy.go b/wasmsdk/proxy.go index af35ab095..301183784 100644 --- a/wasmsdk/proxy.go +++ b/wasmsdk/proxy.go @@ -160,6 +160,7 @@ func main() { "showLogs": showLogs, "getUSDRate": getUSDRate, "isWalletID": isWalletID, + "getVersion": getVersion, "getLookupHash": getLookupHash, "createThumbnail": createThumbnail, "makeSCRestAPICall": makeSCRestAPICall, @@ -236,6 +237,8 @@ func main() { "getMintWZCNPayload": getMintWZCNPayload, "getNotProcessedWZCNBurnEvents": getNotProcessedWZCNBurnEvents, "getNotProcessedZCNBurnTickets": getNotProcessedZCNBurnTickets, + "estimateBurnWZCNGasAmount": estimateBurnWZCNGasAmount, + "estimateMintWZCNGasAmount": estimateMintWZCNGasAmount, "estimateGasPrice": estimateGasPrice, //zcn diff --git a/wasmsdk/sdk.go b/wasmsdk/sdk.go index b616d48ee..0521c8857 100644 --- a/wasmsdk/sdk.go +++ b/wasmsdk/sdk.go @@ -49,6 +49,10 @@ func initSDKs(chainID, blockWorker, signatureScheme string, return nil } +func getVersion() string { + return sdk.GetVersion() +} + var sdkLogger *logger.Logger var zcnLogger *logger.Logger var logEnabled = false diff --git a/wasmsdk/statusbar.go b/wasmsdk/statusbar.go index 47455a456..d07ad5e19 100644 --- a/wasmsdk/statusbar.go +++ b/wasmsdk/statusbar.go @@ -68,7 +68,7 @@ func (s *StatusBar) Completed(allocationID, filePath string, filename string, mi if s.localPath != "" { fs, _ := sys.Files.Open(s.localPath) mf, _ := fs.(*sys.MemFile) - s.objURL = CreateObjectURL(mf.Buffer, "application/octet-stream") + s.objURL = CreateObjectURL(mf.Buffer, mimetype) } if s.callback != nil { jsCallbackMutex.Lock() diff --git a/zboxcore/sdk/allocation.go b/zboxcore/sdk/allocation.go index 24a2458b1..5e30d2cdb 100644 --- a/zboxcore/sdk/allocation.go +++ b/zboxcore/sdk/allocation.go @@ -402,11 +402,13 @@ func (a *Allocation) RepairFile(file sys.File, remotepath string, statusCallback WithEncrypt(true), WithStatusCallback(statusCallback), WithEncryptedPoint(ref.EncryptedKeyPoint), + WithChunkNumber(100), } } else { opts = []ChunkedUploadOption{ WithMask(mask), WithStatusCallback(statusCallback), + WithChunkNumber(100), } } op := &OperationRequest{ @@ -842,6 +844,11 @@ func (a *Allocation) DoMultiOperation(operations []OperationRequest, opts ...Mul wg.Wait() // Check consensus if mo.operationMask.CountOnes() < mo.consensusThresh { + l.Logger.Error("Multioperation: create connection failed. Required consensus not met", + zap.Int("consensusThresh", mo.consensusThresh), + zap.Int("operationMask", mo.operationMask.CountOnes()), + zap.Any("connectionErrors", connectionErrors)) + majorErr := zboxutil.MajorError(connectionErrors) if majorErr != nil { return errors.New("consensus_not_met", @@ -1046,7 +1053,9 @@ func (a *Allocation) DownloadThumbnail(localPath string, remotePath string, veri } err = a.addAndGenerateDownloadRequest(f, remotePath, DOWNLOAD_CONTENT_THUMB, 1, 0, - numBlockDownloads, verifyDownload, status, isFinal, localFilePath) + numBlockDownloads, verifyDownload, status, isFinal, localFilePath, WithFileCallback(func() { + f.Close() //nolint: errcheck + })) if err != nil { if !toKeep { os.Remove(localFilePath) //nolint: errcheck @@ -1106,6 +1115,9 @@ func (a *Allocation) generateDownloadRequest( downloadReq.contentMode = contentMode downloadReq.connectionID = connectionID downloadReq.downloadQueue = make(downloadQueue, len(a.Blobbers)) + for i := 0; i < len(a.Blobbers); i++ { + downloadReq.downloadQueue[i].timeTaken = 1000000 + } return downloadReq, nil } @@ -2219,6 +2231,9 @@ func (a *Allocation) downloadFromAuthTicket(fileHandler sys.File, authTicket str downloadReq.fullconsensus = a.fullconsensus downloadReq.consensusThresh = a.consensusThreshold downloadReq.downloadQueue = make(downloadQueue, len(a.Blobbers)) + for i := 0; i < len(a.Blobbers); i++ { + downloadReq.downloadQueue[i].timeTaken = 1000000 + } downloadReq.connectionID = zboxutil.NewConnectionId() downloadReq.completedCallback = func(remotepath string, remotepathHash string) { a.mutex.Lock() diff --git a/zboxcore/sdk/blockdownloadworker.go b/zboxcore/sdk/blockdownloadworker.go index 24a98c017..6605941cc 100644 --- a/zboxcore/sdk/blockdownloadworker.go +++ b/zboxcore/sdk/blockdownloadworker.go @@ -157,7 +157,7 @@ func (req *BlockDownloadRequest) downloadBlobberBlock(hostClient *fasthttp.HostC err = func() error { now := time.Now() - statuscode, respBuf, err := hostClient.GetWithRequestTimeout(httpreq, req.respBuf, 30*time.Second) + statuscode, respBuf, err := hostClient.GetWithRequest(httpreq, req.respBuf) fasthttp.ReleaseRequest(httpreq) timeTaken := time.Since(now).Milliseconds() if err != nil { @@ -263,10 +263,12 @@ func AddBlockDownloadReq(ctx context.Context, req *BlockDownloadRequest, rb zbox if rb != nil { reqCtx, cncl := context.WithTimeout(ctx, (time.Second * 10)) defer cncl() - req.respBuf = rb.RequestChunk(reqCtx, int(req.blockNum/req.numBlocks)) + req.respBuf = rb.RequestChunk(reqCtx, int(req.blockNum)) if len(req.respBuf) == 0 { req.respBuf = make([]byte, int(req.numBlocks)*effectiveBlockSize) } + } else { + req.respBuf = make([]byte, int(req.numBlocks)*effectiveBlockSize) } downloadBlockChan[req.blobber.ID] <- req } diff --git a/zboxcore/sdk/chunked_upload.go b/zboxcore/sdk/chunked_upload.go index a8d906074..d7ad05229 100644 --- a/zboxcore/sdk/chunked_upload.go +++ b/zboxcore/sdk/chunked_upload.go @@ -33,7 +33,7 @@ import ( ) const ( - DefaultUploadTimeOut = 45 * time.Second + DefaultUploadTimeOut = 120 * time.Second ) var ( @@ -297,6 +297,11 @@ func CreateChunkedUpload( } func calculateWorkersAndRequests(dataShards, totalShards, chunknumber int) (uploadWorkers int, uploadRequests int) { + if IsWasm { + uploadWorkers = 1 + uploadRequests = 2 + return + } if totalShards < 4 { uploadWorkers = 4 } else { diff --git a/zboxcore/sdk/chunked_upload_blobber.go b/zboxcore/sdk/chunked_upload_blobber.go index 8a4fde376..2ce411094 100644 --- a/zboxcore/sdk/chunked_upload_blobber.go +++ b/zboxcore/sdk/chunked_upload_blobber.go @@ -103,11 +103,6 @@ func (sb *ChunkedUploadBlobber) sendUploadRequest( } respbody := resp.Body() - if err != nil { - logger.Logger.Error("Error: Resp ", err) - return fmt.Errorf("Error while reading body. Error %s", err), false - } - if resp.StatusCode() == http.StatusTooManyRequests { logger.Logger.Error("Got too many request error") var r int diff --git a/zboxcore/sdk/chunked_upload_form_builder.go b/zboxcore/sdk/chunked_upload_form_builder.go index f3ef02265..94cb956f5 100644 --- a/zboxcore/sdk/chunked_upload_form_builder.go +++ b/zboxcore/sdk/chunked_upload_form_builder.go @@ -92,7 +92,15 @@ func (b *chunkedUploadFormBuilder) Build( for i := 0; i < numBodies; i++ { - body := new(bytes.Buffer) + startRange := i * MAX_BLOCKS + endRange := startRange + MAX_BLOCKS + if endRange > len(fileChunksData) { + endRange = len(fileChunksData) + } + + bodyBuf := make([]byte, 0, (CHUNK_SIZE*(endRange-startRange))+1024) + + body := bytes.NewBuffer(bodyBuf) formWriter := multipart.NewWriter(body) defer formWriter.Close() @@ -101,11 +109,6 @@ func (b *chunkedUploadFormBuilder) Build( return res, err } - startRange := i * MAX_BLOCKS - endRange := startRange + MAX_BLOCKS - if endRange > len(fileChunksData) { - endRange = len(fileChunksData) - } for _, chunkBytes := range fileChunksData[startRange:endRange] { _, err = uploadFile.Write(chunkBytes) if err != nil { diff --git a/zboxcore/sdk/dirworker.go b/zboxcore/sdk/dirworker.go index 6ec68becd..ed52ee134 100644 --- a/zboxcore/sdk/dirworker.go +++ b/zboxcore/sdk/dirworker.go @@ -195,7 +195,7 @@ func (req *DirRequest) createDirInBlobber(blobber *blockchain.StorageNode, pos u for i := 0; i < 3; i++ { err, shouldContinue = func() (err error, shouldContinue bool) { - ctx, cncl := context.WithTimeout(req.ctx, (time.Second * 30)) + ctx, cncl := context.WithTimeout(req.ctx, (time.Second * 10)) resp, err = zboxutil.Client.Do(httpreq.WithContext(ctx)) cncl() if err != nil { diff --git a/zboxcore/sdk/download_progress_storer.go b/zboxcore/sdk/download_progress_storer.go index 0ac364496..67b6f1743 100644 --- a/zboxcore/sdk/download_progress_storer.go +++ b/zboxcore/sdk/download_progress_storer.go @@ -14,7 +14,7 @@ import ( type DownloadProgressStorer interface { // Load load download progress by id - Load(id string) *DownloadProgress + Load(id string, numBlocks int) *DownloadProgress // Update download progress Update(writtenBlock int) // Remove remove download progress by id @@ -72,7 +72,7 @@ func (ds *FsDownloadProgressStorer) Start(ctx context.Context) { }() } -func (ds *FsDownloadProgressStorer) Load(progressID string) *DownloadProgress { +func (ds *FsDownloadProgressStorer) Load(progressID string, numBlocks int) *DownloadProgress { dp := &DownloadProgress{} buf, err := sys.Files.ReadFile(progressID) if err != nil { @@ -82,6 +82,8 @@ func (ds *FsDownloadProgressStorer) Load(progressID string) *DownloadProgress { return nil } ds.dp = dp + dp.numBlocks = numBlocks + ds.next = dp.LastWrittenBlock return ds.dp } diff --git a/zboxcore/sdk/downloadworker.go b/zboxcore/sdk/downloadworker.go index 2b789f644..4675ce4c8 100644 --- a/zboxcore/sdk/downloadworker.go +++ b/zboxcore/sdk/downloadworker.go @@ -105,7 +105,7 @@ type DownloadRequest struct { bufferMap map[int]zboxutil.DownloadBuffer downloadStorer DownloadProgressStorer workdir string - downloadQueue downloadQueue + downloadQueue downloadQueue // Always initialize this queue with max time taken } type downloadPriority struct { @@ -285,7 +285,7 @@ func (req *DownloadRequest) downloadBlock( if req.shouldVerify { go AddBlockDownloadReq(req.ctx, blockDownloadReq, nil, req.effectiveBlockSize) } else { - go AddBlockDownloadReq(req.ctx, blockDownloadReq, req.bufferMap[int(pos)], req.effectiveBlockSize) + go AddBlockDownloadReq(req.ctx, blockDownloadReq, req.bufferMap[blobberIdx], req.effectiveBlockSize) } } @@ -303,14 +303,21 @@ func (req *DownloadRequest) downloadBlock( var err error defer func() { if err != nil { - atomic.AddInt32(&failed, 1) - req.removeFromMask(uint64(result.maskIdx)) + totalFail := atomic.AddInt32(&failed, 1) + // if first request remove from end as we will convert the slice into heap + if timeRequest { + req.removeFromMask(uint64(activeBlobbers - int(totalFail))) + } else { + req.removeFromMask(uint64(result.maskIdx)) + } downloadErrors[i] = fmt.Sprintf("Error %s from %s", err.Error(), req.blobbers[result.idx].Baseurl) logger.Logger.Error(err) - req.bufferMap[result.idx].ReleaseChunk(int(req.startBlock / req.numBlocks)) + if req.bufferMap != nil && req.bufferMap[result.idx] != nil { + req.bufferMap[result.idx].ReleaseChunk(int(req.startBlock)) + } } else if timeRequest { - req.downloadQueue[result.idx].timeTaken = result.timeTaken + req.downloadQueue[result.maskIdx].timeTaken = result.timeTaken } wg.Done() }() @@ -532,19 +539,31 @@ func (req *DownloadRequest) processDownload() { if !req.shouldVerify { var pos uint64 req.bufferMap = make(map[int]zboxutil.DownloadBuffer) + defer func() { + l.Logger.Info("Clearing download buffers: ", len(req.bufferMap)) + for ind, rb := range req.bufferMap { + rb.ClearBuffer() + delete(req.bufferMap, ind) + } + req.bufferMap = nil + }() sz := downloadWorkerCount + EXTRA_COUNT if sz > n { sz = n } for i := req.downloadMask; !i.Equals64(0); i = i.And(zboxutil.NewUint128(1).Lsh(pos).Not()) { pos = uint64(i.TrailingZeros()) + blobberIdx := int(pos) if writerAt { - req.bufferMap[int(pos)] = zboxutil.NewDownloadBufferWithChan(sz, int(numBlocks), req.effectiveBlockSize) + req.bufferMap[blobberIdx] = zboxutil.NewDownloadBufferWithChan(sz, int(numBlocks), req.effectiveBlockSize) } else { - req.bufferMap[int(pos)] = zboxutil.NewDownloadBufferWithMask(sz, int(numBlocks), req.effectiveBlockSize) + req.bufferMap[blobberIdx] = zboxutil.NewDownloadBufferWithMask(sz, int(numBlocks), req.effectiveBlockSize) } } } + // reset mask to number of active blobbers, not it denotes index of download queue and not blobber index + activeBlobbers := req.downloadMask.CountOnes() + req.downloadMask = zboxutil.NewUint128(1).Lsh(uint64(activeBlobbers)).Sub64(1) logger.Logger.Info( fmt.Sprintf("Downloading file with size: %d from start block: %d and end block: %d. "+ @@ -600,7 +619,7 @@ func (req *DownloadRequest) processDownload() { hashWg.Wait() } for _, rb := range req.bufferMap { - rb.ReleaseChunk(i) + rb.ReleaseChunk(int(startBlock + int64(i)*numBlocks)) } downloaded = downloaded + totalWritten remainingSize -= int64(totalWritten) @@ -648,7 +667,7 @@ func (req *DownloadRequest) processDownload() { hashWg.Wait() } for _, rb := range req.bufferMap { - rb.ReleaseChunk(i) + rb.ReleaseChunk(int(startBlock + int64(i)*numBlocks)) } downloaded = downloaded + totalWritten @@ -678,7 +697,7 @@ func (req *DownloadRequest) processDownload() { var progressLock sync.Mutex firstReqWG := sync.WaitGroup{} firstReqWG.Add(1) - eg, _ := errgroup.WithContext(ctx) + eg, egCtx := errgroup.WithContext(ctx) eg.SetLimit(downloadWorkerCount + EXTRA_COUNT) for i := 0; i < n; i++ { j := i @@ -686,6 +705,11 @@ func (req *DownloadRequest) processDownload() { firstReqWG.Wait() heap.Init(&req.downloadQueue) } + select { + case <-egCtx.Done(): + goto breakDownloadLoop + default: + } eg.Go(func() error { if j == 0 { @@ -721,7 +745,7 @@ func (req *DownloadRequest) processDownload() { return errors.Wrap(err, fmt.Sprintf("WriteAt failed for block %d. ", startBlock+int64(j)*numBlocks)) } for _, rb := range req.bufferMap { - rb.ReleaseChunk(j) + rb.ReleaseChunk(int(startBlock + int64(j)*numBlocks)) } if req.downloadStorer != nil { go req.downloadStorer.Update(int(startBlock + int64(j)*numBlocks + blocksToDownload)) @@ -735,11 +759,13 @@ func (req *DownloadRequest) processDownload() { } return nil }) + breakDownloadLoop: } if err := eg.Wait(); err != nil { writeCancel() - req.errorCB(err, remotePathCB) close(blocks) + wg.Wait() + req.errorCB(err, remotePathCB) return } @@ -1025,17 +1051,17 @@ func (req *DownloadRequest) calculateShardsParams( progressID := req.progressID() var dp *DownloadProgress if info.Size() > 0 { - dp = req.downloadStorer.Load(progressID) + dp = req.downloadStorer.Load(progressID, int(req.numBlocks)) } if dp != nil { req.startBlock = int64(dp.LastWrittenBlock) } else { dp = &DownloadProgress{ - ID: progressID, + ID: progressID, + numBlocks: int(req.numBlocks), } req.downloadStorer.Save(dp) } - dp.numBlocks = int(req.numBlocks) } } @@ -1082,11 +1108,8 @@ func GetFileRefFromBlobber(allocationID, blobberId, remotePath string) (fRef *fi listReq.ctx = ctx listReq.remotefilepath = remotePath - listReq.wg = &sync.WaitGroup{} - listReq.wg.Add(1) rspCh := make(chan *fileMetaResponse, 1) go listReq.getFileMetaInfoFromBlobber(listReq.blobbers[0], 0, rspCh) - listReq.wg.Wait() resp := <-rspCh return resp.fileref, resp.err } @@ -1221,6 +1244,7 @@ func (req *DownloadRequest) getFileMetaConsensus(fMetaResp []*fileMetaResponse) return nil, fmt.Errorf("consensus_not_met") } req.downloadMask = foundMask + heap.Init(&req.downloadQueue) return selected.fileref, nil } diff --git a/zboxcore/sdk/filemetaworker.go b/zboxcore/sdk/filemetaworker.go index 4d505d05b..a57debcf3 100644 --- a/zboxcore/sdk/filemetaworker.go +++ b/zboxcore/sdk/filemetaworker.go @@ -8,7 +8,6 @@ import ( "io" "mime/multipart" "net/http" - "sync" "time" "github.com/0chain/errors" @@ -25,7 +24,6 @@ type fileMetaResponse struct { } func (req *ListRequest) getFileMetaInfoFromBlobber(blobber *blockchain.StorageNode, blobberIdx int, rspCh chan<- *fileMetaResponse) { - defer req.wg.Done() body := new(bytes.Buffer) formWriter := multipart.NewWriter(body) @@ -92,13 +90,10 @@ func (req *ListRequest) getFileMetaInfoFromBlobber(blobber *blockchain.StorageNo func (req *ListRequest) getFileMetaFromBlobbers() []*fileMetaResponse { numList := len(req.blobbers) - req.wg = &sync.WaitGroup{} - req.wg.Add(numList) rspCh := make(chan *fileMetaResponse, numList) for i := 0; i < numList; i++ { go req.getFileMetaInfoFromBlobber(req.blobbers[i], i, rspCh) } - req.wg.Wait() fileInfos := make([]*fileMetaResponse, len(req.blobbers)) for i := 0; i < numList; i++ { ch := <-rspCh diff --git a/zboxcore/sdk/filemetaworker_test.go b/zboxcore/sdk/filemetaworker_test.go index 1bc06cd71..2b37c8e95 100644 --- a/zboxcore/sdk/filemetaworker_test.go +++ b/zboxcore/sdk/filemetaworker_test.go @@ -169,12 +169,9 @@ func TestListRequest_getFileMetaInfoFromBlobber(t *testing.T) { authToken: &marker.AuthTicket{ Signature: mockSignature, }, - wg: &sync.WaitGroup{}, } rspCh := make(chan *fileMetaResponse, 1) - req.wg.Add(1) go req.getFileMetaInfoFromBlobber(blobber, 73, rspCh) - req.wg.Wait() var resp *fileMetaResponse select { @@ -280,7 +277,6 @@ func TestListRequest_getFileConsensusFromBlobbers(t *testing.T) { allocationTx: mockAllocationTxId, ctx: context.TODO(), blobbers: []*blockchain.StorageNode{}, - wg: &sync.WaitGroup{}, Consensus: tt.consensus, //nolint } for i := 0; i < tt.numBlobbers; i++ { diff --git a/zboxcore/sdk/filestatsworker.go b/zboxcore/sdk/filestatsworker.go index f75822f2c..b7b846515 100644 --- a/zboxcore/sdk/filestatsworker.go +++ b/zboxcore/sdk/filestatsworker.go @@ -8,7 +8,6 @@ import ( "mime/multipart" "net/http" "strings" - "sync" "time" "github.com/0chain/errors" @@ -45,7 +44,6 @@ type fileStatsResponse struct { } func (req *ListRequest) getFileStatsInfoFromBlobber(blobber *blockchain.StorageNode, blobberIdx int, rspCh chan<- *fileStatsResponse) { - defer req.wg.Done() body := new(bytes.Buffer) formWriter := multipart.NewWriter(body) @@ -111,13 +109,10 @@ func (req *ListRequest) getFileStatsInfoFromBlobber(blobber *blockchain.StorageN func (req *ListRequest) getFileStatsFromBlobbers() map[string]*FileStats { numList := len(req.blobbers) //fmt.Printf("%v\n", req.blobbers) - req.wg = &sync.WaitGroup{} - req.wg.Add(numList) rspCh := make(chan *fileStatsResponse, numList) for i := 0; i < numList; i++ { go req.getFileStatsInfoFromBlobber(req.blobbers[i], i, rspCh) } - req.wg.Wait() fileInfos := make(map[string]*FileStats) for i := 0; i < numList; i++ { ch := <-rspCh diff --git a/zboxcore/sdk/filestatsworker_test.go b/zboxcore/sdk/filestatsworker_test.go index e2e4938b6..60a3e17a0 100644 --- a/zboxcore/sdk/filestatsworker_test.go +++ b/zboxcore/sdk/filestatsworker_test.go @@ -11,7 +11,6 @@ import ( "net/http" "strconv" "strings" - "sync" "testing" "github.com/0chain/errors" @@ -157,12 +156,9 @@ func TestListRequest_getFileStatsInfoFromBlobber(t *testing.T) { allocationTx: mockAllocationTxId, remotefilepath: mockRemoteFilePath, ctx: context.Background(), - wg: &sync.WaitGroup{}, } rspCh := make(chan *fileStatsResponse, 1) - req.wg.Add(1) go req.getFileStatsInfoFromBlobber(&blobber, mockBlobberIndex, rspCh) - req.wg.Wait() resp := <-rspCh require.EqualValues(t, tt.wantErr, resp.err != nil) if resp.err != nil { @@ -257,7 +253,6 @@ func TestListRequest_getFileStatsFromBlobbers(t *testing.T) { allocationTx: mockAllocationTxId, ctx: context.TODO(), blobbers: []*blockchain.StorageNode{}, - wg: &sync.WaitGroup{}, } for i := 0; i < tt.numBlobbers; i++ { req.blobbers = append(req.blobbers, &blockchain.StorageNode{ diff --git a/zboxcore/sdk/listworker.go b/zboxcore/sdk/listworker.go index 55eaa2d19..611ad635b 100644 --- a/zboxcore/sdk/listworker.go +++ b/zboxcore/sdk/listworker.go @@ -30,7 +30,6 @@ type ListRequest struct { remotefilepath string authToken *marker.AuthTicket ctx context.Context - wg *sync.WaitGroup forRepair bool listOnly bool offset int @@ -91,7 +90,6 @@ func WithListRequestForRepair(forRepair bool) ListRequestOptions { } func (req *ListRequest) getListInfoFromBlobber(blobber *blockchain.StorageNode, blobberIdx int, rspCh chan<- *listResponse) { - defer req.wg.Done() //body := new(bytes.Buffer) //formWriter := multipart.NewWriter(body) @@ -130,7 +128,7 @@ func (req *ListRequest) getListInfoFromBlobber(blobber *blockchain.StorageNode, } //httpreq.Header.Add("Content-Type", formWriter.FormDataContentType()) - ctx, cncl := context.WithTimeout(req.ctx, (time.Second * 30)) + ctx, cncl := context.WithTimeout(req.ctx, (time.Second * 10)) err = zboxutil.HttpDo(ctx, cncl, httpreq, func(resp *http.Response, err error) error { if err != nil { l.Logger.Error("List : ", err) @@ -162,46 +160,45 @@ func (req *ListRequest) getListInfoFromBlobber(blobber *blockchain.StorageNode, func (req *ListRequest) getlistFromBlobbers() ([]*listResponse, error) { numList := len(req.blobbers) - req.wg = &sync.WaitGroup{} - req.wg.Add(numList) rspCh := make(chan *listResponse, numList) for i := 0; i < numList; i++ { go req.getListInfoFromBlobber(req.blobbers[i], i, rspCh) } - req.wg.Wait() listInfos := make([]*listResponse, numList) + consensusMap := make(map[string][]*blockchain.StorageNode) + var consensusHash string for i := 0; i < numList; i++ { listInfos[i] = <-rspCh + if !req.forRepair { + if listInfos[i].err != nil || listInfos[i].ref == nil { + continue + } + hash := listInfos[i].ref.FileMetaHash + consensusMap[hash] = append(consensusMap[hash], req.blobbers[listInfos[i].blobberIdx]) + if len(consensusMap[hash]) >= req.consensusThresh { + consensusHash = hash + break + } + } } if req.listOnly { return listInfos, nil } - consensusMap := make(map[string][]*blockchain.StorageNode) - var consensusHash string - for i := 0; i < numList; i++ { - if listInfos[i].err != nil || listInfos[i].ref == nil { - continue - } - hash := listInfos[i].ref.FileMetaHash - consensusMap[hash] = append(consensusMap[hash], req.blobbers[listInfos[i].blobberIdx]) - if len(consensusMap[hash]) >= req.consensusThresh { - consensusHash = hash - } - } + var err error req.listOnly = true listLen := len(consensusMap[consensusHash]) if listLen < req.consensusThresh { return listInfos, listInfos[0].err } + listInfos = listInfos[:1] + listOnlyRespCh := make(chan *listResponse, 1) for i := 0; i < listLen; i++ { var rnd = rand.New(rand.NewSource(time.Now().UnixNano())) num := rnd.Intn(listLen) randomBlobber := consensusMap[consensusHash][num] - req.wg.Add(1) - go req.getListInfoFromBlobber(randomBlobber, 0, rspCh) - req.wg.Wait() - listInfos[0] = <-rspCh + go req.getListInfoFromBlobber(randomBlobber, 0, listOnlyRespCh) + listInfos[0] = <-listOnlyRespCh if listInfos[0].err == nil { return listInfos, nil } diff --git a/zboxcore/sdk/listworker_test.go b/zboxcore/sdk/listworker_test.go index 85890520c..eacea00ea 100644 --- a/zboxcore/sdk/listworker_test.go +++ b/zboxcore/sdk/listworker_test.go @@ -185,12 +185,9 @@ func TestListRequest_getListInfoFromBlobber(t *testing.T) { authToken: &marker.AuthTicket{ Signature: mockSignature, }, - wg: &sync.WaitGroup{}, } rspCh := make(chan *listResponse, 1) - req.wg.Add(1) go req.getListInfoFromBlobber(blobber, 41, rspCh) - req.wg.Wait() resp := <-rspCh require.EqualValues(tt.wantErr, resp.err != nil) if resp.err != nil { @@ -288,7 +285,6 @@ func TestListRequest_GetListFromBlobbers(t *testing.T) { allocationTx: mockAllocationTxId, ctx: context.TODO(), blobbers: []*blockchain.StorageNode{}, - wg: &sync.WaitGroup{}, Consensus: Consensus{ consensusThresh: tt.consensusThresh, fullconsensus: tt.fullconsensus, diff --git a/zboxcore/sdk/multi_operation_worker.go b/zboxcore/sdk/multi_operation_worker.go index 7f18a26e6..88b3f8f59 100644 --- a/zboxcore/sdk/multi_operation_worker.go +++ b/zboxcore/sdk/multi_operation_worker.go @@ -26,7 +26,7 @@ import ( ) const ( - DefaultCreateConnectionTimeOut = 2 * time.Minute + DefaultCreateConnectionTimeOut = 10 * time.Second ) var BatchSize = 6 diff --git a/zboxcore/sdk/repairworker.go b/zboxcore/sdk/repairworker.go index bc8e0d90e..f271f3939 100644 --- a/zboxcore/sdk/repairworker.go +++ b/zboxcore/sdk/repairworker.go @@ -63,7 +63,7 @@ func (r *RepairRequest) processRepair(ctx context.Context, a *Allocation) { if r.checkForCancel(a) { return } - + SetNumBlockDownloads(100) r.iterateDir(a, r.listDir) if r.statusCB != nil { @@ -177,7 +177,7 @@ func (r *RepairRequest) repairFile(a *Allocation, file *ListResult) []OperationR return nil } memFile := &sys.MemChanFile{ - Buffer: make(chan []byte, 10), + Buffer: make(chan []byte, 100), ChunkWriteSize: int(a.GetChunkReadSize(ref.EncryptedKey != "")), } op = a.RepairFile(memFile, file.Path, statusCB, found, ref) diff --git a/zboxcore/sdk/rollback.go b/zboxcore/sdk/rollback.go index 06fa4afa9..c230ae906 100644 --- a/zboxcore/sdk/rollback.go +++ b/zboxcore/sdk/rollback.go @@ -55,10 +55,11 @@ func GetWritemarker(allocID, allocTx, id, baseUrl string) (*LatestPrevWriteMarke if err != nil { return nil, err } - + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() for retries := 0; retries < 3; retries++ { - resp, err := zboxutil.Client.Do(req) + resp, err := zboxutil.Client.Do(req.WithContext(ctx)) if err != nil { return nil, err } diff --git a/zboxcore/zboxutil/download_buffer.go b/zboxcore/zboxutil/download_buffer.go index 70694c694..ba4b8c692 100644 --- a/zboxcore/zboxutil/download_buffer.go +++ b/zboxcore/zboxutil/download_buffer.go @@ -9,6 +9,7 @@ import ( type DownloadBuffer interface { RequestChunk(ctx context.Context, num int) []byte ReleaseChunk(num int) + ClearBuffer() } type DownloadBufferWithChan struct { @@ -59,12 +60,21 @@ func (r *DownloadBufferWithChan) RequestChunk(ctx context.Context, num int) []by } } +func (r *DownloadBufferWithChan) ClearBuffer() { + r.buf = nil + close(r.ch) + for k := range r.mp { + delete(r.mp, k) + } + r.mp = nil +} + type DownloadBufferWithMask struct { buf []byte length int reqSize int mask uint32 - mu sync.RWMutex + mu sync.Mutex } func NewDownloadBufferWithMask(size, numBlocks, effectiveBlockSize int) *DownloadBufferWithMask { @@ -85,16 +95,15 @@ func (r *DownloadBufferWithMask) RequestChunk(ctx context.Context, num int) []by return nil default: } - r.mu.RLock() + r.mu.Lock() isSet := r.mask & (1 << num) - r.mu.RUnlock() // already assigned if isSet == 0 { + r.mu.Unlock() time.Sleep(500 * time.Millisecond) continue } // assign the chunk by clearing the bit - r.mu.Lock() r.mask &= ^(1 << num) r.mu.Unlock() return r.buf[num*r.reqSize : (num+1)*r.reqSize] @@ -107,3 +116,7 @@ func (r *DownloadBufferWithMask) ReleaseChunk(num int) { defer r.mu.Unlock() r.mask |= 1 << num } + +func (r *DownloadBufferWithMask) ClearBuffer() { + r.buf = nil +} diff --git a/zboxcore/zboxutil/http.go b/zboxcore/zboxutil/http.go index 92e02a004..771c60d90 100644 --- a/zboxcore/zboxutil/http.go +++ b/zboxcore/zboxutil/http.go @@ -189,8 +189,8 @@ func init() { Concurrency: 4096, DNSCacheDuration: time.Hour, }).Dial, - ReadTimeout: 60 * time.Second, - WriteTimeout: 60 * time.Second, + ReadTimeout: 120 * time.Second, + WriteTimeout: 120 * time.Second, } envProxy.initialize() log.Init(logger.DEBUG, "0box-sdk") diff --git a/zboxcore/zboxutil/util.go b/zboxcore/zboxutil/util.go index dbff87480..65e3bfd66 100644 --- a/zboxcore/zboxutil/util.go +++ b/zboxcore/zboxutil/util.go @@ -66,13 +66,14 @@ func (b *lazybuf) string() string { } func GetFileContentType(out io.ReadSeeker) (string, error) { - buffer := make([]byte, 261) - _, err := out.Read(buffer) + buffer := make([]byte, 10240) + n, err := out.Read(buffer) defer out.Seek(0, 0) //nolint if err != nil && err != io.EOF { return "", err } + buffer = buffer[:n] kind, _ := filetype.Match(buffer) if kind == filetype.Unknown { diff --git a/zcnbridge/bridge.go b/zcnbridge/bridge.go index eec3623bc..acb78e214 100644 --- a/zcnbridge/bridge.go +++ b/zcnbridge/bridge.go @@ -998,35 +998,44 @@ func (b *BridgeClient) prepareBridge(ctx context.Context, ethereumAddress, metho return bridgeInstance, transactOpts, nil } -// isEstimateGasPriceAvailable checks if currently selected ethereum node url can be used for gas estimation. -func (b *BridgeClient) isEstimateGasPriceAvailable() bool { - return strings.Contains(b.EthereumNodeURL, "eth-mainnet.g.alchemy.com") +// getProviderType validates the provider url and exposes pre-defined type definition. +func (b *BridgeClient) getProviderType() int { + if strings.Contains(b.EthereumNodeURL, "g.alchemy.com") { + return AlchemyProvider + } else if strings.Contains(b.EthereumNodeURL, "rpc.tenderly.co") { + return TenderlyProvider + } else { + return UnknownProvider + } } -// EstimateGasPrice performs gas estimation for the given transaction using Alchemy enhanced API returning -// approximate final gas fee. -func (b *BridgeClient) EstimateGasPrice(ctx context.Context, from, to string, value int64) (*GasPriceEstimationResult, error) { - if !b.isEstimateGasPriceAvailable() { - return nil, errors.New("used json-rpc does not allow to estimate gas price") - } +// estimateTenderlyGasAmount performs gas amount estimation for the given transaction using Tenderly provider. +func (b *BridgeClient) estimateTenderlyGasAmount(ctx context.Context, from, to string, value int64) (float64, error) { + return 8000000, nil +} +// estimateAlchemyGasAmount performs gas amount estimation for the given transaction using Alchemy provider +func (b *BridgeClient) estimateAlchemyGasAmount(ctx context.Context, from, to, data string, value int64) (float64, error) { client := jsonrpc.NewClient(b.EthereumNodeURL) valueHex := ConvertIntToHex(value) - resp, err := client.Call(ctx, "eth_estimateGas", &GasEstimationRequest{ - From: from, To: to, Value: valueHex}) + resp, err := client.Call(ctx, "eth_estimateGas", &AlchemyGasEstimationRequest{ + From: from, + To: to, + Value: valueHex, + Data: data}) if err != nil { - return nil, errors.Wrap(err, "gas price estimation failed") + return 0, errors.Wrap(err, "gas price estimation failed") } if resp.Error != nil { - return nil, errors.Wrap(errors.New(resp.Error.Error()), "gas price estimation failed") + return 0, errors.Wrap(errors.New(resp.Error.Error()), "gas price estimation failed") } gasAmountRaw, ok := resp.Result.(string) if !ok { - return nil, errors.New("failed to parse gas amount") + return 0, errors.New("failed to parse gas amount") } gasAmountInt := new(big.Float) @@ -1034,21 +1043,97 @@ func (b *BridgeClient) EstimateGasPrice(ctx context.Context, from, to string, va gasAmountFloat, _ := gasAmountInt.Float64() - fmt.Println(gasAmountFloat) + return gasAmountFloat, nil +} + +// EstimateBurnWZCNGasAmount performs gas amount estimation for the given wzcn burn transaction. +func (b *BridgeClient) EstimateBurnWZCNGasAmount(ctx context.Context, from, to string, amountTokens int) (float64, error) { + switch b.getProviderType() { + case AlchemyProvider: + abi, err := bridge.BridgeMetaData.GetAbi() + if err != nil { + return 0, errors.Wrap(err, "failed to get ABI") + } + + clientID := DefaultClientIDEncoder(zcncore.GetClientWalletID()) + + amount := new(big.Int) + amount.SetInt64(int64(amountTokens)) + + var packRaw []byte + packRaw, err = abi.Pack("burn", amount, clientID) + if err != nil { + return 0, errors.Wrap(err, "failed to pack arguments") + } + + pack := "0x" + hex.EncodeToString(packRaw) + + return b.estimateAlchemyGasAmount(ctx, from, to, pack, 0) + case TenderlyProvider: + return b.estimateTenderlyGasAmount(ctx, from, to, 0) + } + + return 0, errors.New("used json-rpc does not allow to estimate gas amount") +} + +// EstimateMintWZCNGasAmount performs gas amount estimation for the given wzcn mint transaction. +func (b *BridgeClient) EstimateMintWZCNGasAmount( + ctx context.Context, from, to, zcnTransactionRaw string, amountToken, nonceRaw int64, signaturesRaw [][]byte) (float64, error) { + switch b.getProviderType() { + case AlchemyProvider: + amount := new(big.Int) + amount.SetInt64(amountToken) + + zcnTransaction := DefaultClientIDEncoder(zcnTransactionRaw) - resp, err = client.Call(ctx, "eth_gasPrice") + nonce := new(big.Int) + nonce.SetInt64(nonceRaw) + + fromRaw := common.HexToAddress(from) + + abi, err := bridge.BridgeMetaData.GetAbi() + if err != nil { + return 0, errors.Wrap(err, "failed to get ABI") + } + + var packRaw []byte + packRaw, err = abi.Pack("mint", fromRaw, amount, zcnTransaction, nonce, signaturesRaw) + if err != nil { + return 0, errors.Wrap(err, "failed to pack arguments") + } + + pack := "0x" + hex.EncodeToString(packRaw) + + return b.estimateAlchemyGasAmount(ctx, from, to, pack, 0) + case TenderlyProvider: + return b.estimateTenderlyGasAmount(ctx, from, to, 0) + } + + return 0, errors.New("used json-rpc does not allow to estimate gas amount") +} + +// estimateTenderlyGasPrice performs gas estimation for the given transaction using Tenderly API. +func (b *BridgeClient) estimateTenderlyGasPrice(ctx context.Context) (float64, error) { + return 1, nil +} + +// estimateAlchemyGasPrice performs gas estimation for the given transaction using Alchemy enhanced API returning +// approximate final gas fee. +func (b *BridgeClient) estimateAlchemyGasPrice(ctx context.Context) (float64, error) { + client := jsonrpc.NewClient(b.EthereumNodeURL) + + resp, err := client.Call(ctx, "eth_gasPrice") if err != nil { - return nil, errors.Wrap(err, "gas price estimation failed") + return 0, errors.Wrap(err, "gas price estimation failed") } if resp.Error != nil { - return nil, errors.Wrap(errors.New(resp.Error.Error()), "gas price estimation failed") + return 0, errors.Wrap(errors.New(resp.Error.Error()), "gas price estimation failed") } - var gasPriceRaw string - gasPriceRaw, ok = resp.Result.(string) + gasPriceRaw, ok := resp.Result.(string) if !ok { - return nil, errors.New("failed to parse gas price") + return 0, errors.New("failed to parse gas price") } gasPriceInt := new(big.Float) @@ -1056,8 +1141,17 @@ func (b *BridgeClient) EstimateGasPrice(ctx context.Context, from, to string, va gasPriceFloat, _ := gasPriceInt.Float64() - fmt.Println(gasPriceFloat) + return gasPriceFloat, nil +} + +// EstimateGasPrice performs gas estimation for the given transaction. +func (b *BridgeClient) EstimateGasPrice(ctx context.Context) (float64, error) { + switch b.getProviderType() { + case AlchemyProvider: + return b.estimateAlchemyGasPrice(ctx) + case TenderlyProvider: + return b.estimateTenderlyGasPrice(ctx) + } - return &GasPriceEstimationResult{ - Value: gasPriceFloat * gasAmountFloat}, nil + return 0, errors.New("used json-rpc does not allow to estimate gas price") } diff --git a/zcnbridge/bridge_helper.go b/zcnbridge/bridge_helper.go index 08014f445..8ffb69dcc 100644 --- a/zcnbridge/bridge_helper.go +++ b/zcnbridge/bridge_helper.go @@ -10,6 +10,14 @@ import ( "github.com/pkg/errors" ) +// AlchemyGasEstimationRequest describes request used for Alchemy enhanced JSON-RPC API. +type AlchemyGasEstimationRequest struct { + From string `json:"from"` + To string `json:"to"` + Value string `json:"value"` + Data string `json:"data"` +} + // GasEstimationRequest describes request used for Alchemy enhanced JSON-RPC API. type GasEstimationRequest struct { From string `json:"from"` diff --git a/zcnbridge/bridge_test.go b/zcnbridge/bridge_test.go index 51a750dec..93cea8c0d 100644 --- a/zcnbridge/bridge_test.go +++ b/zcnbridge/bridge_test.go @@ -41,9 +41,10 @@ import ( const ( ethereumAddress = "0xD8c9156e782C68EE671C09b6b92de76C97948432" - alchemyEthereumNodeURL = "https://eth-mainnet.g.alchemy.com/v2/9VanLUbRE0pLmDHwCHGJlhs9GHosrfD9" - infuraEthereumNodeURL = "https://mainnet.infura.io/v3/7238211010344719ad14a89db874158c" - value = 1e+10 + alchemyEthereumNodeURL = "https://eth-mainnet.g.alchemy.com/v2/9VanLUbRE0pLmDHwCHGJlhs9GHosrfD9" + tenderlyEthereumNodeURL = "https://rpc.tenderly.co/fork/835ecb4e-1f60-4129-adc2-b0c741193839" + infuraEthereumNodeURL = "https://mainnet.infura.io/v3/7238211010344719ad14a89db874158c" + value = 1e+10 password = "02289b9" @@ -628,17 +629,24 @@ func Test_ZCNBridge(t *testing.T) { )) }) - t.Run("should check if gas price estimation works with correct ethereum node url", func(t *testing.T) { + t.Run("should check if gas price estimation works with correct alchemy ethereum node url", func(t *testing.T) { bridgeClient = getBridgeClient(bancorMockServerURL, alchemyEthereumNodeURL, ethereumClient, transactionProvider, keyStore) - _, err := bridgeClient.EstimateGasPrice(context.Background(), tokenAddress, bridgeAddress, value) + _, err := bridgeClient.EstimateGasPrice(context.Background()) require.Contains(t, err.Error(), "Must be authenticated!") }) + t.Run("should check if gas price estimation works with correct tenderly ethereum node url", func(t *testing.T) { + bridgeClient = getBridgeClient(bancorMockServerURL, tenderlyEthereumNodeURL, ethereumClient, transactionProvider, keyStore) + + _, err := bridgeClient.EstimateGasPrice(context.Background()) + require.NoError(t, err) + }) + t.Run("should check if gas price estimation works with incorrect ethereum node url", func(t *testing.T) { bridgeClient = getBridgeClient(bancorMockServerURL, infuraEthereumNodeURL, ethereumClient, transactionProvider, keyStore) - _, err := bridgeClient.EstimateGasPrice(context.Background(), tokenAddress, bridgeAddress, value) + _, err := bridgeClient.EstimateGasPrice(context.Background()) require.Error(t, err) }) } diff --git a/zcnbridge/config.go b/zcnbridge/config.go index 03ba097ba..f01e3e0aa 100644 --- a/zcnbridge/config.go +++ b/zcnbridge/config.go @@ -15,6 +15,12 @@ import ( "github.com/spf13/viper" ) +const ( + TenderlyProvider = iota + AlchemyProvider + UnknownProvider +) + const ( ZChainsClientConfigName = "config.yaml" ZChainWalletConfigName = "wallet.json" diff --git a/zcnbridge/wallet/status.go b/zcnbridge/wallet/status.go index 244c16cdf..48dbef449 100644 --- a/zcnbridge/wallet/status.go +++ b/zcnbridge/wallet/status.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" "fmt" + "net/http" "os" "sync" @@ -80,8 +81,16 @@ func (zcn *ZCNStatus) OnWalletCreateComplete(status int, wallet string, err stri zcn.walletString = wallet } -func (zcn *ZCNStatus) OnInfoAvailable(_ int, status int, info string, err string) { +func (zcn *ZCNStatus) OnInfoAvailable(op int, status int, info string, err string) { defer zcn.Wg.Done() + + // If status is 400 for OpGetMintNonce, mintNonce is considered as 0 + if op == zcncore.OpGetMintNonce && status == http.StatusBadRequest { + zcn.Err = nil + zcn.Success = true + return + } + if status != zcncore.StatusSuccess { zcn.Err = errors.New(err) zcn.Success = false diff --git a/zcncore/transaction_query.go b/zcncore/transaction_query.go index 797d36d6a..044d477fc 100644 --- a/zcncore/transaction_query.go +++ b/zcncore/transaction_query.go @@ -535,6 +535,11 @@ func GetInfoFromSharders(urlSuffix string, op int, cb GetInfoCallback) { qr, err := tq.GetInfo(context.TODO(), urlSuffix) if err != nil { + if qr != nil && op == OpGetMintNonce { + logging.Debug("OpGetMintNonce QueryResult error", "; Content = ", qr.Content, "; Error = ", qr.Error.Error(), "; StatusCode = ", qr.StatusCode) + cb.OnInfoAvailable(op, qr.StatusCode, "", qr.Error.Error()) + return + } cb.OnInfoAvailable(op, StatusError, "", err.Error()) return }