NEAR2Eth-Relay: Optimized Block Selection (#735)

* NEAR2Eth-Relay: implement optimized block selection (aurora-is-near/bounties#1)
* NEAR2Eth-Relay: allow submitting EIP-1559 txs (aurora-is-near/bounties#1)
* CLI: declare options `near2eth-relay-next-block-select-delay-ms`, `eth-use-eip-1559`, `log-verbose`
* Utils: extend RobustWeb3 with `maxPriorityFeePerGas` and `getFeeData` methods
* Docs: add development.md doc
* CI tests: fix `err.data.substring is not a function` error

cli/commands/danger-submit-invalid-near-block.js

@@ -9,9 +9,14 @@ class DangerSubmitInvalidNearBlock {
     ethClientArtifactPath,
     ethClientAddress,
     ethGasMultiplier,
+    ethUseEip1559,
     near2ethRelayMinDelay,
     near2ethRelayMaxDelay,
-    near2ethRelayErrorDelay
+    near2ethRelayErrorDelay,
+    near2ethRelayBlockSelectDuration,
+    near2ethRelayNextBlockSelectDelayMs,
+    near2ethRelayAfterSubmitDelayMs,
+    logVerbose
   }) {
     const relay = new Near2EthRelay()
     await relay.initialize({
@@ -27,7 +32,12 @@ class DangerSubmitInvalidNearBlock {
       near2ethRelayMinDelay,
       near2ethRelayMaxDelay,
       near2ethRelayErrorDelay,
-      ethGasMultiplier
+      near2ethRelayBlockSelectDuration,
+      near2ethRelayNextBlockSelectDelayMs,
+      near2ethRelayAfterSubmitDelayMs,
+      ethGasMultiplier,
+      ethUseEip1559,
+      logVerbose
     })
   }
 }

cli/commands/start/near2eth-relay.js

@@ -13,11 +13,15 @@ class StartNear2EthRelayCommand {
     ethClientArtifactPath,
     ethClientAddress,
     ethGasMultiplier,
+    ethUseEip1559,
     near2ethRelayMinDelay,
     near2ethRelayMaxDelay,
     near2ethRelayErrorDelay,
+    near2ethRelayBlockSelectDuration,
+    near2ethRelayNextBlockSelectDelayMs,
     near2ethRelayAfterSubmitDelayMs,
-    metricsPort
+    metricsPort,
+    logVerbose
   }) {
     if (daemon === 'true') {
       ProcessManager.connect((err) => {
@@ -42,12 +46,16 @@ class StartNear2EthRelayCommand {
             '--eth-client-artifact-path', ethClientArtifactPath,
             '--eth-client-address', ethClientAddress,
             '--eth-gas-multiplier', ethGasMultiplier,
+            '--eth-use-eip-1559', ethUseEip1559,
             '--near2eth-relay-min-delay', near2ethRelayMinDelay,
             '--near2eth-relay-max-delay', near2ethRelayMaxDelay,
             '--near2eth-relay-error-delay', near2ethRelayErrorDelay,
+            '--near2eth-relay-block-select-duration', near2ethRelayBlockSelectDuration,
+            '--near2eth-relay-next-block-select-delay-ms', near2ethRelayNextBlockSelectDelayMs,
             '--near2eth-relay-after-submit-delay-ms', near2ethRelayAfterSubmitDelayMs,
             '--daemon', 'false',
-            '--metrics-port', metricsPort
+            '--metrics-port', metricsPort,
+            '--log-verbose', logVerbose
           ],
           wait_ready: true,
           kill_timeout: 60000,
@@ -70,8 +78,12 @@ class StartNear2EthRelayCommand {
         near2ethRelayMinDelay,
         near2ethRelayMaxDelay,
         near2ethRelayErrorDelay,
+        near2ethRelayBlockSelectDuration,
+        near2ethRelayNextBlockSelectDelayMs,
         near2ethRelayAfterSubmitDelayMs,
-        ethGasMultiplier
+        ethGasMultiplier,
+        ethUseEip1559,
+        logVerbose
       })
     }
   }

cli/index.js

@@ -157,11 +157,21 @@ RainbowConfig.declareOption(
   'How many times more in Ethereum gas are we willing to overpay.',
   '1'
 )
+RainbowConfig.declareOption(
+  'eth-use-eip-1559',
+  'Allow submitting transactions using the EIP-1559 pricing mechanism.',
+  'false'
+)
 RainbowConfig.declareOption(
   'metrics-port',
   'On which port to expose metrics for corresponding relayer, if not provided no metrics exposed',
   null
 )
+RainbowConfig.declareOption(
+  'log-verbose',
+  'Log more information than the standard logging process.',
+  'false'
+)
 
 // User-specific arguments.
 RainbowConfig.declareOption(
@@ -278,6 +288,16 @@ RainbowConfig.declareOption(
   'Number of seconds to wait before retrying if there is an error.',
   '1'
 )
+RainbowConfig.declareOption(
+  'near2eth-relay-block-select-duration',
+  'Number of seconds to select the optimal block to submit.',
+  '300'
+)
+RainbowConfig.declareOption(
+  'near2eth-relay-next-block-select-delay-ms',
+  'Number of ms until the next request in the optimal block selection algorithm.',
+  '1200'
+)
 RainbowConfig.declareOption(
   'near2eth-relay-after-submit-delay-ms',
   'Number of ms to wait after successfully submitting light client block to prevent submitting the same block again.',
@@ -371,10 +391,14 @@ RainbowConfig.addOptions(
     'near2eth-relay-min-delay',
     'near2eth-relay-max-delay',
     'near2eth-relay-error-delay',
+    'near2eth-relay-block-select-duration',
+    'near2eth-relay-next-block-select-delay-ms',
     'near2eth-relay-after-submit-delay-ms',
     'eth-gas-multiplier',
+    'eth-use-eip-1559',
     'daemon',
-    'metrics-port'
+    'metrics-port',
+    'log-verbose'
   ]
 )
 
@@ -872,7 +896,12 @@ RainbowConfig.addOptions(
     'near2eth-relay-min-delay',
     'near2eth-relay-max-delay',
     'near2eth-relay-error-delay',
-    'eth-gas-multiplier'
+    'near2eth-relay-block-select-duration',
+    'near2eth-relay-next-block-select-delay-ms',
+    'near2eth-relay-after-submit-delay-ms',
+    'eth-gas-multiplier',
+    'eth-use-eip-1559',
+    'log-verbose'
   ]
 )
 

docs/development.md

@@ -0,0 +1,66 @@
+# Development
+
+The loosely coupled architecture of the Rainbow Bridge allows its parts to be developed independently of each other.
+This document describes the development workflow for each such component.
+
+## Preparation
+
+This assumes that supported versions of the `nvm`, `yarn`, `hardhat` are already installed.
+
+Switch to a supported `node` version and install dependencies:
+```bash
+nvm install 13
+yarn install
+```
+## Near2Eth Relay
+
+The service responsible for submitting the NEAR Light Client block  to Ethereum smart contracts.
+
+Compile Solidity contracts:
+```bash
+cd contracts/eth/nearbridge
+yarn install
+yarn build
+```
+
+Run local Hardhat node:
+```bash
+hardhat node
+```
+
+Deploy ED25519 Solidity contract:
+```bash
+cli/index.js init-eth-ed25519 \
+    --eth-node-url http://127.0.0.1:8545/ \
+    --eth-master-sk 0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80
+```
+Address of deployed contract: `0x5fbdb2315678afecb367f032d93f642f64180aa3`
+
+**NOTE:** The private key specified here is the key of the first account created by [Hardhat](https://hardhat.org/hardhat-network/#running-stand-alone-in-order-to-support-wallets-and-other-software) and is publicly known.
+
+Deploy and initialize EthClient contracts:
+```bash
+cli/index.js init-eth-client \
+    --eth-node-url http://127.0.0.1:8545/ \
+    --eth-master-sk 0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80 \
+    --eth-ed25519-address 0x5fbdb2315678afecb367f032d93f642f64180aa3 \
+    --eth-client-lock-duration 30 \
+    --eth-client-replace-duration 60
+```
+Address of deployed contract: `0xe7f1725e7734ce288f8367e1bb143e90bb3f0512`
+
+Start Near2EthRelay:
+```bash
+cli/index.js start near2eth-relay \
+    --eth-node-url http://127.0.0.1:8545/ \
+    --eth-master-sk 0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80 \
+    --near-node-url https://rpc.testnet.near.org/ \
+    --near-network-id testnet \
+    --eth-client-address 0xe7f1725e7734ce288f8367e1bb143e90bb3f0512 \
+    --eth-use-eip-1559 true \
+    --near2eth-relay-max-delay 10 \
+    --near2eth-relay-block-select-duration 30 \
+    --near2eth-relay-after-submit-delay-ms 1000 \
+    --log-verbose true \
+    --daemon false
+```

near2eth/near2eth-block-relay/index.js

@@ -174,8 +174,12 @@ class Near2EthRelay {
     near2ethRelayMinDelay,
     near2ethRelayMaxDelay,
     near2ethRelayErrorDelay,
+    near2ethRelayBlockSelectDuration,
+    near2ethRelayNextBlockSelectDelayMs,
     near2ethRelayAfterSubmitDelayMs,
-    ethGasMultiplier
+    ethGasMultiplier,
+    ethUseEip1559,
+    logVerbose
   }) {
     const clientContract = this.clientContract
     const robustWeb3 = this.robustWeb3
@@ -188,10 +192,56 @@ class Near2EthRelay {
     const errorDelay = Number(near2ethRelayErrorDelay)
     const afterSubmitDelayMs = Number(near2ethRelayAfterSubmitDelayMs)
 
+    const selectDurationNs = web3.utils.toBN(Number(near2ethRelayBlockSelectDuration) * 1000_000_000)
+    const nextBlockSelectDelayMs = Number(near2ethRelayNextBlockSelectDelayMs)
+
+    ethGasMultiplier = Number(ethGasMultiplier)
+    ethUseEip1559 = ethUseEip1559 === 'true'
+    logVerbose = logVerbose === 'true'
+
     const httpPrometheus = new HttpPrometheus(this.metricsPort, 'near_bridge_near2eth_')
     const clientHeightGauge = httpPrometheus.gauge('client_height', 'amount of block client processed')
     const chainHeightGauge = httpPrometheus.gauge('chain_height', 'current chain height')
 
+    const nextBlockSelection = {
+      startedAt: 0,
+      borshBlock: null,
+      height: 0,
+      set: function ({ borshBlock, lightClientBlock }) {
+        if (this.isEmpty()) {
+          this.startedAt = lightClientBlock.inner_lite.timestamp
+        }
+        this.borshBlock = borshBlock
+        this.height = lightClientBlock.inner_lite.height
+        console.log(`The new optimal block is found at height ${this.height}, ${this.borshBlock.length} bytes`)
+      },
+      clean: function () {
+        this.startedAt = 0
+        this.borshBlock = null
+        this.height = 0
+      },
+      isEmpty: function () {
+        return !this.borshBlock
+      },
+      isSuitable: function ({ borshBlock, lightClientBlock }) {
+        return this.isEmpty() ||
+               (!this.isEmpty() &&
+                 this.height !== lightClientBlock.inner_lite.height &&
+                 this.borshBlock.length >= borshBlock.length)
+      }
+    }
+    const getGasOptions = async (useEip1559, gasMultiplier) => {
+      const gasOptions = {}
+      if (useEip1559) {
+        const feeData = await robustWeb3.getFeeData(gasMultiplier)
+        gasOptions.maxPriorityFeePerGas = feeData.maxPriorityFeePerGas
+        gasOptions.maxFeePerGas = feeData.maxFeePerGas
+      } else {
+        const gasPrice = new BN(await web3.eth.getGasPrice())
+        gasOptions.gasPrice = gasPrice.mul(new BN(gasMultiplier))
+      }
+      return gasOptions
+    }
     while (true) {
       try {
         // Determine the next action: sleep or attempt an update.
@@ -209,7 +259,7 @@ class Near2EthRelay {
           await clientContract.methods.replaceDuration().call()
         )
         const nextValidAt = web3.utils.toBN(bridgeState.nextValidAt)
-        let replaceDelay
+        let replaceDelay = web3.utils.toBN(0)
         if (!nextValidAt.isZero()) {
           replaceDelay = web3.utils
             .toBN(bridgeState.nextTimestamp)
@@ -219,8 +269,30 @@ class Near2EthRelay {
         // console.log({bridgeState, currentBlockHash, lastBlock, replaceDuration}) // DEBUG
         if (bridgeState.currentHeight < lastBlock.inner_lite.height) {
           if (nextValidAt.isZero() || replaceDelay.cmpn(0) <= 0) {
+            // Serialize once here to avoid multiple 'borshify(...)' function calls
+            const blockCouple = {
+              lightClientBlock: lastBlock,
+              borshBlock: borshify(lastBlock)
+            }
+
+            if (nextBlockSelection.isSuitable(blockCouple)) {
+              nextBlockSelection.set(blockCouple)
+            }
+
+            const selectDelayNs = selectDurationNs
+              .add(web3.utils.toBN(nextBlockSelection.startedAt))
+              .sub(web3.utils.toBN(lastBlock.inner_lite.timestamp))
+            if (selectDelayNs.cmpn(0) > 0) {
+              if (logVerbose) {
+                const selectDelaySeconds = selectDelayNs.div(web3.utils.toBN(1000_000_000))
+                console.log(`Last block height ${lastBlock.inner_lite.height}, ${blockCouple.borshBlock.length} bytes, ${selectDelaySeconds.toString()}s left`)
+              }
+              await sleep(nextBlockSelectDelayMs)
+              continue
+            }
+
             console.log(
-              `Trying to submit new block at height ${lastBlock.inner_lite.height}.`
+              `Trying to submit new block at height ${nextBlockSelection.height}, ${nextBlockSelection.borshBlock.length} bytes`
             )
 
             // Check whether master account has enough balance at stake.
@@ -234,31 +306,29 @@ class Near2EthRelay {
               console.log(
                 `The sender account does not have enough stake. Transferring ${lockEthAmount} wei.`
               )
+              const gasOptions = await getGasOptions(ethUseEip1559, ethGasMultiplier)
               await clientContract.methods.deposit().send({
                 from: ethMasterAccount,
                 gas: 1000000,
                 handleRevert: true,
                 value: new BN(lockEthAmount),
-                gasPrice: new BN(await web3.eth.getGasPrice()).mul(new BN(ethGasMultiplier))
+                ...gasOptions
               })
               console.log('Transferred.')
             }
 
-            const borshBlock = borshify(lastBlock)
             if (submitInvalidBlock) {
               console.log('Mutate block by one byte')
-              console.log(borshBlock)
-              borshBlock[Math.floor(borshBlock.length * Math.random())] += 1
+              console.log(nextBlockSelection.borshBlock)
+              nextBlockSelection.borshBlock[Math.floor(nextBlockSelection.borshBlock.length * Math.random())] += 1
             }
 
-            const gasPrice = new BN(await web3.eth.getGasPrice())
-            console.log('Gas price:', gasPrice.toNumber())
-
-            await clientContract.methods.addLightClientBlock(borshBlock).send({
+            const gasOptions = await getGasOptions(ethUseEip1559, ethGasMultiplier)
+            await clientContract.methods.addLightClientBlock(nextBlockSelection.borshBlock).send({
               from: ethMasterAccount,
               gas: 10000000,
               handleRevert: true,
-              gasPrice: gasPrice.mul(new BN(ethGasMultiplier))
+              ...gasOptions
             })
 
             if (submitInvalidBlock) {
@@ -267,6 +337,7 @@ class Near2EthRelay {
             }
 
             console.log('Submitted.')
+            nextBlockSelection.clean()
             await sleep(afterSubmitDelayMs) // To prevent submitting the same block again
             continue
           }

testing/ci/e2e.sh

@@ -44,7 +44,7 @@ node index.js init-near-token-factory
 yarn run pm2 ping
 sleep 5
 yarn run pm2 list
-node index.js start near2eth-relay --near2eth-relay-min-delay 1 --near2eth-relay-max-delay 30 --near2eth-relay-after-submit-delay-ms 40000
+node index.js start near2eth-relay --near2eth-relay-min-delay 1 --near2eth-relay-max-delay 30 --near2eth-relay-after-submit-delay-ms 40000 --near2eth-relay-block-select-duration 5
 sleep 5
 yarn run pm2 list
 node index.js start eth2near-relay

testing/ci/e2e_deploy_contract.sh

@@ -51,7 +51,7 @@ cat /tmp/eth2neartransfer.out | xargs node index.js deploy-token myerc20
 yarn run pm2 ping
 sleep 5
 yarn run pm2 list
-node index.js start near2eth-relay --near2eth-relay-min-delay 1 --near2eth-relay-max-delay 30 --near2eth-relay-after-submit-delay-ms 40000
+node index.js start near2eth-relay --near2eth-relay-min-delay 1 --near2eth-relay-max-delay 30 --near2eth-relay-after-submit-delay-ms 40000 --near2eth-relay-block-select-duration 0
 sleep 5
 yarn run pm2 list
 node index.js start eth2near-relay