🗞️ Retry functionality for SDKs (LayerZero-Labs#558)

.changeset/many-toys-explain.md

@@ -0,0 +1,11 @@
+---
+"@layerzerolabs/protocol-devtools-evm": patch
+"@layerzerolabs/ua-devtools-evm": patch
+"@layerzerolabs/devtools": patch
+"@layerzerolabs/devtools-evm": patch
+"@layerzerolabs/devtools-evm-hardhat": patch
+"@layerzerolabs/toolbox-hardhat": patch
+"@layerzerolabs/ua-devtools-evm-hardhat": patch
+---
+
+Adding `AsyncRetriable` to SDKs

packages/devtools/.swcrc

@@ -0,0 +1,11 @@
+{
+    "jsc": {
+        "parser": {
+            "syntax": "typescript",
+            "decorators": true
+        },
+        "transform": {
+            "legacyDecorator": true
+        }
+    }
+}

packages/devtools/src/common/index.ts

@@ -1,4 +1,5 @@
 export * from './assertion'
 export * from './bytes'
 export * from './promise'
+export * from './retry'
 export * from './strings'

packages/devtools/src/common/retry.ts

@@ -0,0 +1,107 @@
+import { createModuleLogger, printJson } from '@layerzerolabs/io-devtools'
+import assert from 'assert'
+import { backOff } from 'exponential-backoff'
+
+export type OnRetry<TInstance, TArgs extends unknown[] = unknown[]> = (
+    attempt: number,
+    numAttempts: number,
+    error: unknown,
+    target: TInstance,
+    method: string,
+    args: TArgs
+) => boolean | void | undefined
+
+export interface RetriableConfig<TInstance = unknown> {
+    /**
+     * Enable / disable the retry behavior
+     */
+    enabled?: boolean
+    /**
+     * The maximum delay, in milliseconds, between two consecutive attempts.
+     *
+     * @default Infinity
+     */
+    maxDelay?: number
+    /**
+     * Number of times the method call will be retried. The default is 3
+     *
+     * @default 3
+     */
+    numAttempts?: number
+    /**
+     * Callback called on every failed attempt.
+     *
+     * @param {number} attempt 1-indexed number of attempt of executing the method
+     * @param {number} numAttempts Maximum/total number of attempts that will be executed
+     * @param {unknown} error The error that caused the function to be retried
+     * @param {unknown} target The object whose method is being retried
+     * @param {string} method The method name
+     * @param {unknown[]} args The method parameters
+     * @returns {boolean | undefined} This function can stop the retry train by returning false
+     */
+    onRetry?: OnRetry<TInstance>
+}
+
+/**
+ * Helper function that creates a default debug logger for the `onRetry`
+ * callback of `AsyncRetriable`
+ */
+export const createDefaultRetryHandler = (loggerName: string = 'AsyncRetriable'): OnRetry<unknown> => {
+    const logger = createModuleLogger(loggerName)
+
+    return (attempt, numAttempts, error, target, method, args) => {
+        logger.debug(`Attempt ${attempt}/${numAttempts}: ${method}() with arguments: ${printJson(args)}: ${error}`)
+    }
+}
+
+export const AsyncRetriable = ({
+    // We'll feature flag this functionality for the time being
+    enabled = !!process.env.LZ_EXPERIMENTAL_ENABLE_RETRY,
+    maxDelay,
+    numAttempts = 3,
+    onRetry = createDefaultRetryHandler(),
+}: RetriableConfig = {}) => {
+    return function AsyncRetriableDecorator<TArgs extends unknown[], TResult>(
+        target: unknown,
+        propertyKey: string,
+        descriptor: TypedPropertyDescriptor<(...args: TArgs) => Promise<TResult>>
+    ) {
+        // If we are disabled, we are disabled
+        if (!enabled) {
+            return descriptor
+        }
+
+        // Grab the original method and ensure that we are decorating a method
+        const originalMethod = descriptor.value
+        assert(
+            typeof originalMethod === 'function',
+            `AsyncRetriable must be applied to an instance method, ${propertyKey} property looks more like ${typeof originalMethod}`
+        )
+
+        // We'll wrap the retry handler from exponential backoff
+        // to make it a bit nicer to use
+        //
+        // - We'll put the attempt as the first argument
+        // - We'll add the decorator target as the last argument
+        //
+        // We'll curry this function so that it can pass the arguments to onRetry
+        const handleRetry =
+            (args: TArgs) =>
+            (error: unknown, attempt: number): boolean =>
+                onRetry?.(attempt, numAttempts, error, target, propertyKey, args) ?? true
+
+        // Create the retried method
+        const retriedMethod = (...args: TArgs): Promise<TResult> =>
+            backOff(() => originalMethod.apply(target, args), {
+                // A typical problem in our case is 429 Too many requests
+                // which would still happen if we didn't introduce a bit of randomness into the delay
+                jitter: 'full',
+                maxDelay,
+                numOfAttempts: numAttempts,
+                retry: handleRetry(args),
+            })
+
+        // return our new descriptor
+        return (descriptor.value = retriedMethod), descriptor
+    }
+}

packages/devtools/test/common/retry.test.ts

@@ -0,0 +1,168 @@
+import { AsyncRetriable } from '@/common/retry'
+
+describe('common/retry', () => {
+    describe('AsyncRetriable', () => {
+        describe('when LZ_EXPERIMENTAL_ENABLE_RETRY is off', () => {
+            beforeAll(() => {
+                // We'll enable the AsyncRetriable for these tests
+                process.env.LZ_EXPERIMENTAL_ENABLE_RETRY = ''
+            })
+
+            it('shoult not retry', async () => {
+                const error = new Error('Told ya')
+                const handleRetry = jest.fn()
+                const mock = jest.fn().mockRejectedValue(error)
+
+                class WithAsyncRetriable {
+                    @AsyncRetriable({ onRetry: handleRetry })
+                    async iAlwaysFail(value: string) {
+                        return mock(value)
+                    }
+                }
+
+                await expect(new WithAsyncRetriable().iAlwaysFail('y')).rejects.toBe(error)
+
+                expect(mock).toHaveBeenCalledTimes(1)
+                expect(handleRetry).not.toHaveBeenCalled()
+            })
+
+            it('should retry if enabled is set to true', async () => {
+                const error = new Error('Told ya')
+                const mock = jest.fn().mockRejectedValue(error)
+
+                class WithAsyncRetriable {
+                    @AsyncRetriable({ enabled: true })
+                    async iAlwaysFail(value: string) {
+                        return mock(value)
+                    }
+                }
+
+                await expect(new WithAsyncRetriable().iAlwaysFail('y')).rejects.toBe(error)
+
+                expect(mock).toHaveBeenCalledTimes(3)
+                expect(mock).toHaveBeenNthCalledWith(1, 'y')
+                expect(mock).toHaveBeenNthCalledWith(2, 'y')
+                expect(mock).toHaveBeenNthCalledWith(3, 'y')
+            })
+        })
+
+        describe('when LZ_EXPERIMENTAL_ENABLE_RETRY is on', () => {
+            beforeAll(() => {
+                // We'll enable the AsyncRetriable for these tests
+                process.env.LZ_EXPERIMENTAL_ENABLE_RETRY = '1'
+            })
+
+            afterAll(() => {
+                process.env.LZ_EXPERIMENTAL_ENABLE_RETRY = ''
+            })
+
+            it('should retry a method call 3 times by default', async () => {
+                const error = new Error('Told ya')
+                const mock = jest.fn().mockRejectedValue(error)
+
+                class WithAsyncRetriable {
+                    @AsyncRetriable()
+                    async iAlwaysFail(value: string) {
+                        return mock(value)
+                    }
+                }
+
+                await expect(new WithAsyncRetriable().iAlwaysFail('y')).rejects.toBe(error)
+
+                expect(mock).toHaveBeenCalledTimes(3)
+                expect(mock).toHaveBeenNthCalledWith(1, 'y')
+                expect(mock).toHaveBeenNthCalledWith(2, 'y')
+                expect(mock).toHaveBeenNthCalledWith(3, 'y')
+            })
+
+            it('should retry a method call N times if numAttempts is specified', async () => {
+                const error = new Error('Told ya')
+                const mock = jest.fn().mockRejectedValue(error)
+
+                class WithAsyncRetriable {
+                    @AsyncRetriable({ numAttempts: 2 })
+                    async iAlwaysFail(value: string) {
+                        return mock(value)
+                    }
+                }
+
+                await expect(new WithAsyncRetriable().iAlwaysFail('y')).rejects.toBe(error)
+
+                expect(mock).toHaveBeenCalledTimes(2)
+            })
+
+            it('should stop retrying if the onRetry handler returns false', async () => {
+                const error = new Error('Told ya')
+                const mock = jest.fn().mockRejectedValue(error)
+                const handleRetry = jest
+                    .fn()
+                    // We check that if we return undefined/void we'll keep trying
+                    .mockReturnValueOnce(undefined)
+                    // We check that if we return true we keep trying
+                    .mockReturnValueOnce(true)
+                    // After the third attempt we return false
+                    .mockReturnValueOnce(false)
+
+                class WithAsyncRetriable {
+                    @AsyncRetriable({ numAttempts: 10_000, onRetry: handleRetry })
+                    async iAlwaysFail(value: string) {
+                        return mock(value)
+                    }
+                }
+
+                await expect(new WithAsyncRetriable().iAlwaysFail('y')).rejects.toBe(error)
+
+                expect(mock).toHaveBeenCalledTimes(3)
+                expect(handleRetry).toHaveBeenCalledTimes(3)
+            })
+
+            it('should call the onRetry callback if provided', async () => {
+                const error = new Error('Told ya')
+                const handleRetry = jest.fn()
+                const mock = jest.fn().mockRejectedValue(error)
+
+                class WithAsyncRetriable {
+                    @AsyncRetriable({ onRetry: handleRetry })
+                    async iAlwaysFail(value: string) {
+                        return mock(value)
+                    }
+                }
+
+                const withAsyncRetriable = new WithAsyncRetriable()
+
+                await expect(withAsyncRetriable.iAlwaysFail('y')).rejects.toBe(error)
+
+                expect(handleRetry).toHaveBeenCalledTimes(3)
+                expect(handleRetry).toHaveBeenNthCalledWith(1, 1, 3, error, withAsyncRetriable, 'iAlwaysFail', ['y'])
+                expect(handleRetry).toHaveBeenNthCalledWith(2, 2, 3, error, withAsyncRetriable, 'iAlwaysFail', ['y'])
+                expect(handleRetry).toHaveBeenNthCalledWith(3, 3, 3, error, withAsyncRetriable, 'iAlwaysFail', ['y'])
+            })
+
+            it('should resolve if the method resolves within the specified number of attempts', async () => {
+                const error = new Error('Told ya')
+                const value = {}
+                const handleRetry = jest.fn()
+                const mock = jest
+                    .fn()
+                    .mockRejectedValueOnce(error)
+                    .mockRejectedValueOnce(error)
+                    .mockResolvedValue(value)
+
+                class WithAsyncRetriable {
+                    @AsyncRetriable({ onRetry: handleRetry })
+                    async iAlwaysFail(value: string) {
+                        return mock(value)
+                    }
+                }
+
+                const withAsyncRetriable = new WithAsyncRetriable()
+
+                await expect(withAsyncRetriable.iAlwaysFail('y')).resolves.toBe(value)
+
+                expect(handleRetry).toHaveBeenCalledTimes(2)
+                expect(handleRetry).toHaveBeenNthCalledWith(1, 1, 3, error, withAsyncRetriable, 'iAlwaysFail', ['y'])
+                expect(handleRetry).toHaveBeenNthCalledWith(2, 2, 3, error, withAsyncRetriable, 'iAlwaysFail', ['y'])
+            })
+        })
+    })
+})

packages/devtools/tsconfig.json

@@ -3,6 +3,7 @@
   "exclude": ["dist", "node_modules"],
   "include": ["src", "test"],
   "compilerOptions": {
+    "experimentalDecorators": true,
     "types": ["node", "jest"],
     "paths": {
       "@/*": ["./src/*"]

packages/protocol-devtools-evm/.swcrc

@@ -0,0 +1,11 @@
+{
+    "jsc": {
+        "parser": {
+            "syntax": "typescript",
+            "decorators": true
+        },
+        "transform": {
+            "legacyDecorator": true
+        }
+    }
+}