Merge pull request #4228 from owid/retry-user-fetches

✨ add automatic retry to client fetch requests
owid · Dec 1, 2024 · af57fc2 · af57fc2
2 parents 9548c2f + a570ac6
commit af57fc2
Show file tree

Hide file tree

Showing 8 changed files with 47 additions and 22 deletions.
diff --git a/packages/@ourworldindata/explorer/src/ExplorerProgram.ts b/packages/@ourworldindata/explorer/src/ExplorerProgram.ts
@@ -21,6 +21,7 @@ import {
     SerializedGridProgram,
     trimObject,
     merge,
+    fetchWithRetry,
 } from "@ourworldindata/utils"
 import {
     CellDef,
@@ -421,7 +422,7 @@ export class ExplorerProgram extends GridProgram {
      */
     private static tableDataLoader = new PromiseCache(
         async (url: string): Promise<CoreTableInputOption> => {
-            const response = await fetch(url)
+            const response = await fetchWithRetry(url)
             if (!response.ok) throw new Error(response.statusText)
             const tableInput: CoreTableInputOption = url.endsWith(".json")
                 ? await response.json()

diff --git a/packages/@ourworldindata/grapher/src/core/loadVariable.ts b/packages/@ourworldindata/grapher/src/core/loadVariable.ts
@@ -1,4 +1,5 @@
 import { OwidVariableDataMetadataDimensions } from "@ourworldindata/types"
+import { fetchWithRetry } from "@ourworldindata/utils"
 
 export const getVariableDataRoute = (
     dataApiUrl: string,
@@ -28,8 +29,10 @@ export async function loadVariableDataAndMetadata(
     variableId: number,
     dataApiUrl: string
 ): Promise<OwidVariableDataMetadataDimensions> {
-    const dataPromise = fetch(getVariableDataRoute(dataApiUrl, variableId))
-    const metadataPromise = fetch(
+    const dataPromise = fetchWithRetry(
+        getVariableDataRoute(dataApiUrl, variableId)
+    )
+    const metadataPromise = fetchWithRetry(
         getVariableMetadataRoute(dataApiUrl, variableId)
     )
     const [dataResponse, metadataResponse] = await Promise.all([

diff --git a/packages/@ourworldindata/utils/src/Util.ts b/packages/@ourworldindata/utils/src/Util.ts
@@ -583,7 +583,7 @@ export const trimObject = <Obj>(
 }
 
 export const fetchText = async (url: string): Promise<string> => {
-    return await fetch(url).then((res) => {
+    return await fetchWithRetry(url).then((res) => {
         if (!res.ok)
             throw new Error(`Fetch failed: ${res.status} ${res.statusText}`)
         return res.text()
@@ -593,7 +593,7 @@ export const fetchText = async (url: string): Promise<string> => {
 const _getUserCountryInformation = async (): Promise<
     UserCountryInformation | undefined
 > =>
-    await fetch("/detect-country")
+    await fetchWithRetry("/detect-country")
         .then((res) => res.json())
         .then((res) => res.country)
         .catch(() => undefined)
@@ -776,17 +776,34 @@ export const getYearFromISOStringAndDayOffset = (
 export const sleep = (ms: number): Promise<void> =>
     new Promise((resolve) => setTimeout(resolve, ms))
 
+interface RetryOptions {
+    maxRetries?: number
+    exponentialBackoff?: boolean
+    initialDelay?: number
+}
+
+export async function fetchWithRetry(
+    url: string,
+    fetchOptions?: RequestInit,
+    retryOptions?: RetryOptions
+): Promise<Response> {
+    const defaultRetryOptions: RetryOptions = {
+        maxRetries: 5,
+        exponentialBackoff: true,
+        initialDelay: 250,
+    }
+    return retryPromise(
+        () => fetch(url, fetchOptions),
+        retryOptions ?? defaultRetryOptions
+    )
+}
 export async function retryPromise<T>(
     promiseGetter: () => Promise<T>,
     {
         maxRetries = 3,
         exponentialBackoff = false,
         initialDelay = 200,
-    }: {
-        maxRetries?: number
-        exponentialBackoff?: boolean
-        initialDelay?: number
-    } = {}
+    }: RetryOptions = {}
 ): Promise<T> {
     let retried = 0
     let lastError

diff --git a/packages/@ourworldindata/utils/src/index.ts b/packages/@ourworldindata/utils/src/index.ts
@@ -91,6 +91,7 @@ export {
     moveArrayItemToIndex,
     getIndexableKeys,
     retryPromise,
+    fetchWithRetry,
     getOwidGdocFromJSON,
     formatDate,
     canWriteToClipboard,

diff --git a/site/detailsOnDemand.tsx b/site/detailsOnDemand.tsx
@@ -4,7 +4,7 @@ import { Instance } from "tippy.js"
 import { BAKED_BASE_URL } from "../settings/clientSettings.js"
 import { renderToStaticMarkup } from "react-dom/server.js"
 import { ArticleBlocks } from "./gdocs/components/ArticleBlocks.js"
-import { DetailDictionary } from "@ourworldindata/utils"
+import { DetailDictionary, fetchWithRetry } from "@ourworldindata/utils"
 import { SiteAnalytics } from "./SiteAnalytics.js"
 
 type Tippyfied<E> = E & {
@@ -20,7 +20,7 @@ declare global {
 const siteAnalytics = new SiteAnalytics()
 
 export async function runDetailsOnDemand() {
-    window.details = await fetch(`${BAKED_BASE_URL}/dods.json`, {
+    window.details = await fetchWithRetry(`${BAKED_BASE_URL}/dods.json`, {
         method: "GET",
         credentials: "same-origin",
         headers: {

diff --git a/site/multiDim/MultiDimDataPageContent.tsx b/site/multiDim/MultiDimDataPageContent.tsx
@@ -25,6 +25,7 @@ import {
     multiDimStateToQueryStr,
     merge,
     omit,
+    fetchWithRetry,
 } from "@ourworldindata/utils"
 import cx from "classnames"
 import { DebugProvider } from "../gdocs/DebugContext.js"
@@ -101,16 +102,16 @@ const getDatapageDataV2 = async (
 
 const cachedGetVariableMetadata = memoize(
     (variableId: number): Promise<OwidVariableWithSourceAndDimension> =>
-        fetch(getVariableMetadataRoute(DATA_API_URL, variableId)).then((resp) =>
-            resp.json()
+        fetchWithRetry(getVariableMetadataRoute(DATA_API_URL, variableId)).then(
+            (resp) => resp.json()
         )
 )
 
 const cachedGetGrapherConfigByUuid = memoize(
     (grapherConfigUuid: string): Promise<GrapherInterface> =>
-        fetch(`/grapher/by-uuid/${grapherConfigUuid}.config.json`).then(
-            (resp) => resp.json()
-        )
+        fetchWithRetry(
+            `/grapher/by-uuid/${grapherConfigUuid}.config.json`
+        ).then((resp) => resp.json())
 )
 
 const useTitleFragments = (config: MultiDimDataPageConfig) => {

diff --git a/site/multiembedder/MultiEmbedder.tsx b/site/multiembedder/MultiEmbedder.tsx
@@ -19,6 +19,7 @@ import {
     merge,
     MultiDimDataPageConfig,
     extractMultiDimChoicesFromQueryStr,
+    fetchWithRetry,
 } from "@ourworldindata/utils"
 import { action } from "mobx"
 import React from "react"
@@ -178,8 +179,8 @@ class MultiEmbedder {
             let configUrl
             if (isMultiDim) {
                 const mdimConfigUrl = `${MULTI_DIM_DYNAMIC_CONFIG_URL}/${slug}.json`
-                const mdimJsonConfig = await fetch(mdimConfigUrl).then((res) =>
-                    res.json()
+                const mdimJsonConfig = await fetchWithRetry(mdimConfigUrl).then(
+                    (res) => res.json()
                 )
                 const mdimConfig =
                     MultiDimDataPageConfig.fromObject(mdimJsonConfig)
@@ -199,8 +200,8 @@ class MultiEmbedder {
             } else {
                 configUrl = `${GRAPHER_DYNAMIC_CONFIG_URL}/${slug}.config.json`
             }
-            const grapherPageConfig = await fetch(configUrl).then((res) =>
-                res.json()
+            const grapherPageConfig = await fetchWithRetry(configUrl).then(
+                (res) => res.json()
             )
 
             const figureConfigAttr = figure.getAttribute(

diff --git a/site/search/evaluateSearch.ts b/site/search/evaluateSearch.ts
@@ -2,6 +2,7 @@
  * Simulate searches against our Algolia index and evaluate the results.
  */
 
+import { fetchWithRetry } from "@ourworldindata/utils"
 import {
     ALGOLIA_ID,
     ALGOLIA_SEARCH_KEY,
@@ -97,7 +98,7 @@ const getClient = (): any => {
 
 const fetchQueryDataset = async (name: string): Promise<QueryDataset> => {
     const url: string = `${SEARCH_EVAL_URL}/${name}`
-    const resp = await fetch(url)
+    const resp = await fetchWithRetry(url)
     const jsonData = await resp.json()
     return { name, queries: jsonData }
 }