Merge pull request #249 from DestinyItemManager/throttle-pause

Add some delays to bulk actions to try and avoid throttling

api/routes/import.ts

@@ -32,7 +32,7 @@ import { importSearches } from '../stately/searches-queries.js';
 import { convertToStatelyItem } from '../stately/settings-queries.js';
 import { batches } from '../stately/stately-utils.js';
 import { importTriumphs } from '../stately/triumphs-queries.js';
-import { badRequest, subtractObject } from '../utils.js';
+import { badRequest, delay, subtractObject } from '../utils.js';
 import { deleteAllData } from './delete-all-data.js';
 
 export const importHandler = asyncHandler(async (req, res) => {
@@ -265,6 +265,7 @@ export async function statelyImport(
   // OK now put them in as fast as we can
   for (const batch of batches(items)) {
     await client.putBatch(...batch);
+    await delay(100); // give it some time to flush
   }
 
   return numTriumphs;

api/stately/bulk-queries.ts

@@ -4,7 +4,7 @@ import { ExportResponse } from '../shapes/export.js';
 import { DestinyVersion } from '../shapes/general.js';
 import { ProfileResponse } from '../shapes/profile.js';
 import { defaultSettings } from '../shapes/settings.js';
-import { subtractObject } from '../utils.js';
+import { delay, subtractObject } from '../utils.js';
 import { client } from './client.js';
 import { AnyItem } from './generated/index.js';
 import { convertItemAnnotation, keyFor as tagKeyFor } from './item-annotations-queries.js';
@@ -76,6 +76,7 @@ async function deleteAllDataForProfile(
   // Then delete them all. We're not in a transaction!
   for (const batch of batches(keys)) {
     await client.del(...batch);
+    await delay(100); // give it some time to flush
   }
   return response;
 }

api/utils.ts

@@ -148,3 +148,7 @@ export function subtractObject<T extends object>(obj: Partial<T>, defaults: T):
   }
   return result;
 }
+
+export function delay(ms: number) {
+  return new Promise((resolve) => setTimeout(resolve, ms));
+}