From 4dea8bc647ad02d856f4a4bb3f2fe9884dfb88a7 Mon Sep 17 00:00:00 2001 From: Michael Taylor Date: Fri, 4 Oct 2024 16:38:31 -0400 Subject: [PATCH 1/4] feat: sync store optimizations --- package-lock.json | 8 ++++---- package.json | 2 +- src/controllers/storeController.ts | 17 ++++++++++++++++- src/tasks/sync_stores.ts | 15 +++++++++++++-- 4 files changed, 34 insertions(+), 8 deletions(-) diff --git a/package-lock.json b/package-lock.json index de0bd6a..fdc9602 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,7 @@ "license": "ISC", "dependencies": { "@dignetwork/datalayer-driver": "^0.1.28", - "@dignetwork/dig-sdk": "^0.0.1-alpha.129", + "@dignetwork/dig-sdk": "^0.0.1-alpha.130", "async-mutex": "^0.5.0", "busboy": "^1.6.0", "express": "^4.19.2", @@ -249,9 +249,9 @@ } }, "node_modules/@dignetwork/dig-sdk": { - "version": "0.0.1-alpha.129", - "resolved": "https://registry.npmjs.org/@dignetwork/dig-sdk/-/dig-sdk-0.0.1-alpha.129.tgz", - "integrity": "sha512-h7rFPX3l8PGrY6Oot7Gyvzm681TWPbaftnBB3u4lpwzRd8XbE0OPQv+HYkicA/xcmyN52rKAFBtd9eJK68Gsdw==", + "version": "0.0.1-alpha.130", + "resolved": "https://registry.npmjs.org/@dignetwork/dig-sdk/-/dig-sdk-0.0.1-alpha.130.tgz", + "integrity": "sha512-E53Oav/Bz0y5iUhxKM4TkgjeHDvqdHxVCPtbkZDMHCbpUWkI/stujFBWKLmxwmuPpo/9vtY0OPuJof9dTFdEZA==", "dependencies": { "@dignetwork/datalayer-driver": "^0.1.29", "@dignetwork/dig-sdk": "^0.0.1-alpha.124", diff --git a/package.json b/package.json index 6fcd0da..9246c85 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,7 @@ ], "dependencies": { "@dignetwork/datalayer-driver": "^0.1.28", - "@dignetwork/dig-sdk": "^0.0.1-alpha.129", + "@dignetwork/dig-sdk": "^0.0.1-alpha.130", "async-mutex": "^0.5.0", "busboy": "^1.6.0", "express": "^4.19.2", diff --git a/src/controllers/storeController.ts b/src/controllers/storeController.ts index 4a1ffd0..02c1e11 100644 --- a/src/controllers/storeController.ts +++ b/src/controllers/storeController.ts @@ -3,7 +3,13 @@ import path from "path"; import fs from "fs"; // @ts-ignore -import { DigNetwork, DataStore, DigPeer } from "@dignetwork/dig-sdk"; +import { + DigNetwork, + DataStore, + DigPeer, + ServerCoin, + NconfManager, +} from "@dignetwork/dig-sdk"; import { getStorageLocation } from "../utils/storage"; export const subscribeToStore = async ( @@ -31,6 +37,15 @@ export const subscribeToStore = async ( const digNetwork = new DigNetwork(storeId); await digNetwork.syncStoreFromPeers(); + const nconfManager = new NconfManager("config.json"); + const publicIp: string | null | undefined = + await nconfManager.getConfigValue("publicIp"); + + if (publicIp) { + const serverCoin = new ServerCoin(storeId); + await serverCoin.ensureServerCoinExists(publicIp); + } + res.status(200).json({ message: `Subscribing to store ${storeId}` }); } catch (error) { console.error( diff --git a/src/tasks/sync_stores.ts b/src/tasks/sync_stores.ts index b964dec..f269cd8 100644 --- a/src/tasks/sync_stores.ts +++ b/src/tasks/sync_stores.ts @@ -112,11 +112,23 @@ const initializeStoreMonitor = async (): Promise => { const storeList = getStoresList(); + const publicIp: string | null | undefined = + await nconfManager.getConfigValue(PUBLIC_IP_KEY); + // Register each store with the store monitor storeList.forEach((storeId) => { + const serverCoin = new ServerCoin(storeId); storeMonitor.registerStore(storeId, async () => { + if (publicIp) { + await serverCoin.ensureServerCoinExists(publicIp); + } + console.log(`Store update detected for ${storeId}. Syncing...`); await syncStore(storeId); + + if (publicIp) { + await serverCoin.ensureServerCoinExists(publicIp); + } }); }); @@ -141,7 +153,6 @@ const initializeStoreMonitor = async (): Promise => { const syncStoresTask = new Task("sync-stores", async () => { if (!mutex.isLocked()) { const releaseMutex = await mutex.acquire(); - let mnemonic: string | undefined; try { console.log("Starting sync-stores task..."); @@ -197,7 +208,7 @@ const syncStoresTask = new Task("sync-stores", async () => { const job = new SimpleIntervalJob( { - seconds: 60, + minutes: 5, runImmediately: true, }, syncStoresTask, From c80f96098b2816b6567a3080ec60f67b6786033e Mon Sep 17 00:00:00 2001 From: Michael Taylor Date: Fri, 4 Oct 2024 16:39:23 -0400 Subject: [PATCH 2/4] chore(release): 0.0.1-alpha.94 --- CHANGELOG.md | 7 +++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0265da0..9183b20 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [0.0.1-alpha.94](https://github.com/DIG-Network/dig-propagation-server/compare/v0.0.1-alpha.93...v0.0.1-alpha.94) (2024-10-04) + + +### Features + +* sync store optimizations ([4dea8bc](https://github.com/DIG-Network/dig-propagation-server/commit/4dea8bc647ad02d856f4a4bb3f2fe9884dfb88a7)) + ### [0.0.1-alpha.93](https://github.com/DIG-Network/dig-propagation-server/compare/v0.0.1-alpha.92...v0.0.1-alpha.93) (2024-10-04) diff --git a/package-lock.json b/package-lock.json index fdc9602..3ca2c0f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "dig-propagation-server", - "version": "0.0.1-alpha.93", + "version": "0.0.1-alpha.94", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "dig-propagation-server", - "version": "0.0.1-alpha.93", + "version": "0.0.1-alpha.94", "license": "ISC", "dependencies": { "@dignetwork/datalayer-driver": "^0.1.28", diff --git a/package.json b/package.json index 9246c85..314ac22 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "dig-propagation-server", - "version": "0.0.1-alpha.93", + "version": "0.0.1-alpha.94", "description": "", "type": "commonjs", "main": "./dist/index.js", From 2e9d32e9d3a4289d24e5f39acbee1af86defaa3c Mon Sep 17 00:00:00 2001 From: Michael Taylor Date: Fri, 4 Oct 2024 16:40:22 -0400 Subject: [PATCH 3/4] feat: sync store optimizations --- src/tasks/sync_stores.ts | 48 ++++++---------------------------------- 1 file changed, 7 insertions(+), 41 deletions(-) diff --git a/src/tasks/sync_stores.ts b/src/tasks/sync_stores.ts index f269cd8..a948cc8 100644 --- a/src/tasks/sync_stores.ts +++ b/src/tasks/sync_stores.ts @@ -1,9 +1,6 @@ -import fs from "fs"; -import path from "path"; import { SimpleIntervalJob, Task } from "toad-scheduler"; import { getStoresList, - Wallet, DataStore, DigNetwork, NconfManager, @@ -11,10 +8,6 @@ import { StoreMonitorRegistry, } from "@dignetwork/dig-sdk"; import { Mutex } from "async-mutex"; -import { getStorageLocation } from "../utils/storage"; - -const STORE_PATH = path.join(getStorageLocation(), "stores"); - const mutex = new Mutex(); const PUBLIC_IP_KEY = "publicIp"; @@ -24,23 +17,6 @@ const nconfManager = new NconfManager("config.json"); // Helper Functions // ------------------------- -/** - * Synchronizes a specific store. - * @param storeId - The ID of the store to synchronize. - */ -const syncStore = async (storeId: string): Promise => { - console.log(`Starting sync process for store ${storeId}...`); - - try { - console.log(`Store ${storeId} is out of date. Syncing...`); - await syncStoreFromNetwork(storeId); - } catch (error: any) { - console.trace(`Error processing store ${storeId}: ${error.message}`); - } finally { - await finalizeStoreSync(storeId); - } -}; - /** * Attempts to synchronize a store from the network. * @param storeId - The ID of the store to synchronize. @@ -48,8 +24,13 @@ const syncStore = async (storeId: string): Promise => { const syncStoreFromNetwork = async (storeId: string): Promise => { try { console.log(`Attempting to sync store ${storeId} from the network...`); + const digNetwork = new DigNetwork(storeId); await digNetwork.syncStoreFromPeers(); + + const dataStore = await DataStore.from(storeId); + await dataStore.fetchCoinInfo(); + console.log(`Store ${storeId} synchronized successfully.`); } catch (error: any) { console.warn( @@ -62,20 +43,6 @@ const syncStoreFromNetwork = async (storeId: string): Promise => { } }; -/** - * Finalizes the synchronization process for a store. - * @param storeId - The ID of the store to finalize. - */ -const finalizeStoreSync = async (storeId: string): Promise => { - try { - console.log(`Finalizing sync for store ${storeId}...`); - const dataStore = await DataStore.from(storeId); - await dataStore.fetchCoinInfo(); - console.log(`Finalization complete for store ${storeId}.`); - } catch (error: any) { - console.error(`Error in finalizing store ${storeId}: ${error.message}`); - } -}; /** * Ensures that the server coin exists and is valid for a specific store. @@ -90,7 +57,6 @@ const ensureServerCoin = async ( const serverCoin = new ServerCoin(storeId); await serverCoin.ensureServerCoinExists(publicIp); await serverCoin.meltOutdatedEpochs(publicIp); - console.log(`Server coin ensured for store ${storeId}.`); } catch (error: any) { console.error( `Failed to ensure server coin for store ${storeId}: ${error.message}` @@ -124,7 +90,7 @@ const initializeStoreMonitor = async (): Promise => { } console.log(`Store update detected for ${storeId}. Syncing...`); - await syncStore(storeId); + await syncStoreFromNetwork(storeId); if (publicIp) { await serverCoin.ensureServerCoinExists(publicIp); @@ -134,7 +100,7 @@ const initializeStoreMonitor = async (): Promise => { // Attempt to sync each store initially for (const storeId of storeList) { - await syncStore(storeId); + await syncStoreFromNetwork(storeId); } console.log("All stores have been initialized and synchronized."); From 8763784ceffd1e47bbe49cc3d81c87369f6bceaf Mon Sep 17 00:00:00 2001 From: Michael Taylor Date: Fri, 4 Oct 2024 16:40:58 -0400 Subject: [PATCH 4/4] chore(release): 0.0.1-alpha.95 --- CHANGELOG.md | 7 +++++++ package-lock.json | 4 ++-- package.json | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9183b20..875a371 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,13 @@ All notable changes to this project will be documented in this file. See [standard-version](https://github.com/conventional-changelog/standard-version) for commit guidelines. +### [0.0.1-alpha.95](https://github.com/DIG-Network/dig-propagation-server/compare/v0.0.1-alpha.94...v0.0.1-alpha.95) (2024-10-04) + + +### Features + +* sync store optimizations ([2e9d32e](https://github.com/DIG-Network/dig-propagation-server/commit/2e9d32e9d3a4289d24e5f39acbee1af86defaa3c)) + ### [0.0.1-alpha.94](https://github.com/DIG-Network/dig-propagation-server/compare/v0.0.1-alpha.93...v0.0.1-alpha.94) (2024-10-04) diff --git a/package-lock.json b/package-lock.json index 3ca2c0f..1154046 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "dig-propagation-server", - "version": "0.0.1-alpha.94", + "version": "0.0.1-alpha.95", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "dig-propagation-server", - "version": "0.0.1-alpha.94", + "version": "0.0.1-alpha.95", "license": "ISC", "dependencies": { "@dignetwork/datalayer-driver": "^0.1.28", diff --git a/package.json b/package.json index 314ac22..838d547 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "dig-propagation-server", - "version": "0.0.1-alpha.94", + "version": "0.0.1-alpha.95", "description": "", "type": "commonjs", "main": "./dist/index.js",