WIP - Integrate RPC failover into NetworkController

eslint-warning-thresholds.json

@@ -417,36 +417,22 @@
   "packages/network-controller/src/NetworkController.ts": {
     "@typescript-eslint/prefer-promise-reject-errors": 1,
     "@typescript-eslint/prefer-readonly": 2,
-    "jsdoc/tag-lines": 1,
-    "prettier/prettier": 1
-  },
-  "packages/network-controller/src/create-auto-managed-network-client.test.ts": {
-    "import-x/order": 1
-  },
-  "packages/network-controller/src/create-network-client.ts": {
-    "@typescript-eslint/no-unsafe-enum-comparison": 1
+    "jsdoc/tag-lines": 1
   },
   "packages/network-controller/tests/NetworkController.test.ts": {
     "@typescript-eslint/no-unused-vars": 1,
     "@typescript-eslint/prefer-promise-reject-errors": 1,
-    "import-x/order": 1,
-    "jest/no-conditional-in-test": 4
+    "import-x/order": 1
   },
   "packages/network-controller/tests/create-network-client.test.ts": {
     "import-x/order": 1
   },
-  "packages/network-controller/tests/provider-api-tests/block-param.ts": {
-    "jest/no-conditional-in-test": 1
-  },
   "packages/network-controller/tests/provider-api-tests/helpers.ts": {
     "@typescript-eslint/prefer-promise-reject-errors": 1,
     "import-x/namespace": 1,
     "import-x/no-named-as-default-member": 1,
     "promise/catch-or-return": 1
   },
-  "packages/network-controller/tests/provider-api-tests/no-block-param.ts": {
-    "jest/no-conditional-in-test": 2
-  },
   "packages/permission-controller/src/Permission.ts": {
     "prettier/prettier": 11
   },

package.json

@@ -46,7 +46,8 @@
   "resolutions": {
     "[email protected]": "^6.5.7",
     "fast-xml-parser@^4.3.4": "^4.4.1",
-    "[email protected]": "^7.5.10"
+    "[email protected]": "^7.5.10",
+    "@metamask/utils": "^11.1.0"
   },
   "devDependencies": {
     "@babel/core": "^7.23.5",

packages/assets-controllers/src/AssetsContractController.test.ts

@@ -91,6 +91,8 @@ async function setupAssetContractControllers({
       allowedActions: [],
       allowedEvents: [],
     }),
+    fetch,
+    btoa,
   });
   if (useNetworkControllerProvider) {
     await networkController.initializeProvider();

packages/gas-fee-controller/src/GasFeeController.test.ts

@@ -80,6 +80,8 @@ const setupNetworkController = async ({
     messenger: restrictedMessenger,
     state,
     infuraProjectId: '123',
+    fetch,
+    btoa,
   });
 
   if (initializeProvider) {

packages/network-controller/CHANGELOG.md

@@ -7,6 +7,29 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- Implement circuit breaker pattern when retrying requests to Infura and custom RPC endpoints
+  - If the network is perceived to be down after 5 attempts, further retries will be paused for 30 seconds
+- Use exponential backoff / jitter when retrying requests to Infura and custom RPC endpoints
+  - As requests are retried, the delay between retries will increase exponentially (using random variance to prevent bursts)
+
+### Changed
+
+- **BREAKING:** `NetworkController` constructor now takes two required options, `fetch` and `btoa`
+  - These are used when creating the JSON-RPC middleware.
+- Synchronize retry logic and error handling behavior between Infura and custom RPC endpoints
+  - A request to a custom endpoint that returns a 418 response will no longer return a JSON-RPC response with the error "Request is being rate limited"
+  - A request to a custom endpoint that returns a 429 response now returns a JSON-RPC response with the error "Request is being rate limited"
+  - A request to a custom endpoint that throws an "ECONNRESET" error will now be retried up to 5 times
+  - A request to a Infura endpoint that fails more than 5 times in a row will now respond with a JSON-RPC error that encompasses the failure instead of hiding it as "InfuraProvider - cannot complete request. All retries exhausted"
+  - A request to a Infura endpoint that returns a non-retriable, non-2xx response will now respond with a JSON-RPC error that has the underling message "Non-200 status code: '\<code\>'" rather than including the raw response from the endpoint
+  - A request to a custom endpoint that fails with a retriable error more than 5 times in a row will now respond with a JSON-RPC error that encompasses the failure instead of returning an empty response
+  - A "retriable error" is now regarded as:
+    - A failure to start the request due to a timeout, connection reset, or unknown connection error (depending on platform / HTTP client)
+    - When the request responds with non-JSON
+    - When the request returns a 503 or 504 response
+
 ## [22.2.0]
 
 ### Added

packages/network-controller/package.json

@@ -50,8 +50,8 @@
     "@metamask/base-controller": "^7.1.1",
     "@metamask/controller-utils": "^11.5.0",
     "@metamask/eth-block-tracker": "^11.0.3",
-    "@metamask/eth-json-rpc-infura": "^10.0.0",
-    "@metamask/eth-json-rpc-middleware": "^15.0.1",
+    "@metamask/eth-json-rpc-infura": "patch:@metamask/eth-json-rpc-infura@npm%3A10.0.0#~/.yarn/patches/@metamask-eth-json-rpc-infura-npm-10.0.0-c3afc1dd82.patch",
+    "@metamask/eth-json-rpc-middleware": "patch:@metamask/eth-json-rpc-middleware@npm%3A15.0.1#~/.yarn/patches/@metamask-eth-json-rpc-middleware-npm-15.0.1-2ae215e586.patch",
     "@metamask/eth-json-rpc-provider": "^4.1.8",
     "@metamask/eth-query": "^4.0.0",
     "@metamask/json-rpc-engine": "^10.0.3",
@@ -72,11 +72,13 @@
     "@types/jest": "^27.4.1",
     "@types/jest-when": "^2.7.3",
     "@types/lodash": "^4.14.191",
+    "@types/node-fetch": "^2.6.12",
     "deepmerge": "^4.2.2",
     "jest": "^27.5.1",
     "jest-when": "^3.4.2",
     "lodash": "^4.17.21",
     "nock": "^13.3.1",
+    "node-fetch": "^2.7.0",
     "sinon": "^9.2.4",
     "ts-jest": "^27.1.4",
     "typedoc": "^0.24.8",

packages/network-controller/src/NetworkController.ts

@@ -531,6 +531,15 @@ export type NetworkControllerOptions = {
   infuraProjectId: string;
   state?: Partial<NetworkState>;
   log?: Logger;
+  /**
+   * A function that can be used to make an HTTP request, compatible with the
+   * Fetch API.
+   */
+  fetch: typeof fetch;
+  /**
+   * A function that can be used to convert a binary string into base-64.
+   */
+  btoa: typeof btoa;
 };
 
 /**
@@ -909,6 +918,10 @@ export class NetworkController extends BaseController<
 
   #log: Logger | undefined;
 
+  readonly #fetch: typeof fetch;
+
+  readonly #btoa: typeof btoa;
+
   #networkConfigurationsByNetworkClientId: Map<
     NetworkClientId,
     NetworkConfiguration
@@ -919,6 +932,8 @@ export class NetworkController extends BaseController<
     state,
     infuraProjectId,
     log,
+    fetch: givenFetch,
+    btoa: givenBtoa,
   }: NetworkControllerOptions) {
     const initialState = { ...getDefaultNetworkControllerState(), ...state };
     validateNetworkControllerState(initialState);
@@ -948,6 +963,8 @@ export class NetworkController extends BaseController<
 
     this.#infuraProjectId = infuraProjectId;
     this.#log = log;
+    this.#fetch = givenFetch;
+    this.#btoa = givenBtoa;
 
     this.#previouslySelectedNetworkClientId =
       this.state.selectedNetworkClientId;
@@ -1222,9 +1239,8 @@ export class NetworkController extends BaseController<
     let updatedIsEIP1559Compatible: boolean | undefined;
 
     try {
-      updatedIsEIP1559Compatible = await this.#determineEIP1559Compatibility(
-        networkClientId,
-      );
+      updatedIsEIP1559Compatible =
+        await this.#determineEIP1559Compatibility(networkClientId);
       updatedNetworkStatus = NetworkStatus.Available;
     } catch (error) {
       debugLog('NetworkController: lookupNetworkByClientId: ', error);
@@ -2425,20 +2441,28 @@ export class NetworkController extends BaseController<
         autoManagedNetworkClientRegistry[NetworkClientType.Infura][
           addedRpcEndpoint.networkClientId
         ] = createAutoManagedNetworkClient({
-          type: NetworkClientType.Infura,
-          chainId: networkFields.chainId,
-          network: addedRpcEndpoint.networkClientId,
-          infuraProjectId: this.#infuraProjectId,
-          ticker: networkFields.nativeCurrency,
+          networkClientConfiguration: {
+            type: NetworkClientType.Infura,
+            chainId: networkFields.chainId,
+            network: addedRpcEndpoint.networkClientId,
+            infuraProjectId: this.#infuraProjectId,
+            ticker: networkFields.nativeCurrency,
+          },
+          fetch: this.#fetch,
+          btoa: this.#btoa,
         });
       } else {
         autoManagedNetworkClientRegistry[NetworkClientType.Custom][
           addedRpcEndpoint.networkClientId
         ] = createAutoManagedNetworkClient({
-          type: NetworkClientType.Custom,
-          chainId: networkFields.chainId,
-          rpcUrl: addedRpcEndpoint.url,
-          ticker: networkFields.nativeCurrency,
+          networkClientConfiguration: {
+            type: NetworkClientType.Custom,
+            chainId: networkFields.chainId,
+            rpcUrl: addedRpcEndpoint.url,
+            ticker: networkFields.nativeCurrency,
+          },
+          fetch: this.#fetch,
+          btoa: this.#btoa,
         });
       }
     }
@@ -2589,21 +2613,29 @@ export class NetworkController extends BaseController<
           return [
             rpcEndpoint.networkClientId,
             createAutoManagedNetworkClient({
-              type: NetworkClientType.Infura,
-              network: infuraNetworkName,
-              infuraProjectId: this.#infuraProjectId,
-              chainId: networkConfiguration.chainId,
-              ticker: networkConfiguration.nativeCurrency,
+              networkClientConfiguration: {
+                type: NetworkClientType.Infura,
+                network: infuraNetworkName,
+                infuraProjectId: this.#infuraProjectId,
+                chainId: networkConfiguration.chainId,
+                ticker: networkConfiguration.nativeCurrency,
+              },
+              fetch: this.#fetch,
+              btoa: this.#btoa,
             }),
           ] as const;
         }
         return [
           rpcEndpoint.networkClientId,
           createAutoManagedNetworkClient({
-            type: NetworkClientType.Custom,
-            chainId: networkConfiguration.chainId,
-            rpcUrl: rpcEndpoint.url,
-            ticker: networkConfiguration.nativeCurrency,
+            networkClientConfiguration: {
+              type: NetworkClientType.Custom,
+              chainId: networkConfiguration.chainId,
+              rpcUrl: rpcEndpoint.url,
+              ticker: networkConfiguration.nativeCurrency,
+            },
+            fetch: this.#fetch,
+            btoa: this.#btoa,
           }),
         ] as const;
       });