create helper function to fetch data with concurrent requests

packages/utils/package.json

@@ -30,11 +30,11 @@
         "@patternfly/react-core": "^5.0.0",
         "@patternfly/react-table": "^5.0.0",
         "@redhat-cloud-services/rbac-client": "^1.0.100",
+        "cypress": ">=12.0.0 < 14.0.0",
         "react": "^18.2.0",
         "react-dom": "^18.2.0",
         "react-redux": ">=7.0.0",
-        "react-router-dom": "^5.0.0 || ^6.0.0",
-        "cypress": ">=12.0.0 < 14.0.0"
+        "react-router-dom": "^5.0.0 || ^6.0.0"
     },
     "dependencies": {
         "@redhat-cloud-services/types": "^0.0.24",
@@ -43,6 +43,7 @@
         "axios": "^0.27.2",
         "commander": "^2.20.3",
         "mkdirp": "^1.0.4",
+        "p-all": "^5.0.0",
         "react-content-loader": "^6.2.0"
     },
     "devDependencies": {

packages/utils/src/index.ts

@@ -18,3 +18,4 @@ export { useInventory } from './useInventory';
 export * from './CypressUtils';
 export * from './useInsightsNavigate';
 export * from './useExportPDF';
+export * from './paginatedRequest';

packages/utils/src/paginatedRequest/index.js

@@ -0,0 +1 @@
+export * from './paginatedRequest';

packages/utils/src/paginatedRequest/paginatedRequest.js

@@ -0,0 +1,40 @@
+import pAll from 'p-all';
+
+/***
+ * Helper function to allow fetching huge number of data from APIs using paginated concurrent requests.
+ * @param {function} [fetchFunction] function to query your endpoint
+ * @param {object} [fetchParameters] contains the query parameters associated to your API request
+ * @param {object} [fetchOptions] options that change the function behavior.
+ * requestSize -  how many data should be fetched per request
+ * responseLength - how many data should be fetched in total using this function
+ * responseTransformer - you can pass function to do some process after all items are fetched
+ * concurrencyCount - how many concurrent api calls should be made
+ * @returns {historyPusher} function to trigger the push
+ */
+const paginatedRequest = async (fetchFunction, fetchParams, fetchOptions) => {
+  const { requestSize = 100, responseLength, responseTransformer, concurrencyCount = 5 } = fetchOptions;
+
+  const requests = [];
+  let totalItems = responseLength,
+    offset = 0;
+
+  const requestQueueSize = totalItems / requestSize;
+
+  while (requestQueueSize > offset) {
+    requests.push(
+      fetchFunction({
+        ...fetchParams,
+        limit: requestSize,
+        offset: offset++,
+      })
+    );
+  }
+
+  const results = await pAll(requests, {
+    concurrency: concurrencyCount,
+  }).then((response) => (responseTransformer ? responseTransformer(response) : response));
+
+  return results;
+};
+
+export default paginatedRequest;