subquery · jiqiang90 · Oct 20, 2023 · Oct 20, 2023
diff --git a/packages/common-ethereum/CHANGELOG.md b/packages/common-ethereum/CHANGELOG.md
@@ -6,6 +6,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [3.0.5] - 2023-10-20
+### Changed
+- Version bump with `type-ethereum` 3.1.0
+
 ## [3.0.4] - 2023-10-18
 ### Changed
 - Version bump with `common` 3.1.3
@@ -97,7 +101,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Sync with main sdk (#14)
 
 ## [0.1.0] - 2022-10-31
-[Unreleased]: https://github.com/subquery/subql-ethereum/compare/common-ethereum/3.0.4...HEAD
+[Unreleased]: https://github.com/subquery/subql-ethereum/compare/common-ethereum/3.0.5...HEAD
+[3.0.5]: https://github.com/subquery/subql-ethereum/compare/common-ethereum/3.0.4...common-ethereum/3.0.5
 [3.0.4]: https://github.com/subquery/subql-ethereum/compare/common-ethereum/3.0.3...common-ethereum/3.0.4
 [3.0.3]: https://github.com/subquery/subql-ethereum/compare/common-ethereum/3.0.2...common-ethereum/3.0.3
 [3.0.2]: https://github.com/subquery/subql-ethereum/compare/common-ethereum/3.0.1...common-ethereum/3.0.2

diff --git a/packages/common-ethereum/package.json b/packages/common-ethereum/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@subql/common-ethereum",
-  "version": "3.0.4",
+  "version": "3.0.5",
   "description": "",
   "scripts": {
     "build": "rm -rf dist && tsc -b",

diff --git a/packages/node/CHANGELOG.md b/packages/node/CHANGELOG.md
@@ -6,20 +6,28 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [3.1.0] - 2023-10-20
+### Added
+- Inject in-memory cache to sandbox
+
+### Fixed
+- Bump with `@subq/node-core` 3.1.0 , fixed poi migration init check, and improve logging
+
 ## [3.0.6] - 2023-10-18
 ### Fixed
 - Update node-core, additional fix for store bulk methods failing with workers
 
 ## [3.0.5] - 2023-10-17
 ### Fixed
-- Update `@subql/node-core` to fix:
+- Update 
   - Dictionary validation error causing application exit (#2101)
   - Auto queue flush getting the queue into a bad state (#2103)
   - Fix getCache could not been cleared after reindex, and could have been re-used and lead to error, such as syncPoi
 
 ## [3.0.4] - 2023-10-12
 ### Changed
 - debug has changed from a boolean to a string to allow scoping debug log level (#2077)
+
 ### Fixed
 - Sync with node-core.
   - Fix issues with using object destructing for store interface and workers
@@ -285,11 +293,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 - Init release
 
-[Unreleased]: https://github.com/subquery/subql-ethereum/compare/node-ethereum/3.0.6...HEAD
+[Unreleased]: https://github.com/subquery/subql-ethereum/compare/node-ethereum/3.1.0...HEAD
+[3.1.0]: https://github.com/subquery/subql-ethereum/compare/node-ethereum/3.0.6...node-ethereum/3.1.0
 [3.0.6]: https://github.com/subquery/subql-ethereum/compare/node-ethereum/3.0.5...node-ethereum/3.0.6
 [3.0.5]: https://github.com/subquery/subql-ethereum/compare/node-ethereum/3.0.4...node-ethereum/3.0.5
 [3.0.4]: https://github.com/subquery/subql-ethereum/compare/node-ethereum/3.0.3...node-ethereum/3.0.4
-[3.0.4]: https://github.com/subquery/subql-ethereum/compare/node-ethereum/3.0.3...node-ethereum/3.0.4
 [3.0.3]: https://github.com/subquery/subql-ethereum/compare/node-ethereum/3.0.2...node-ethereum/3.0.3
 [3.0.2]: https://github.com/subquery/subql-ethereum/compare/node-ethereum/3.0.1...node-ethereum/3.0.2
 [3.0.1]: https://github.com/subquery/subql-ethereum/compare/node-ethereum/3.0.0...node-ethereum/3.0.1

diff --git a/packages/node/package.json b/packages/node/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@subql/node-ethereum",
-  "version": "3.0.6",
+  "version": "3.1.0",
   "description": "",
   "author": "Ian He",
   "license": "GPL-3.0",
@@ -27,7 +27,7 @@
     "@nestjs/schedule": "^3.0.1",
     "@subql/common": "3.1.3",
     "@subql/common-ethereum": "workspace:*",
-    "@subql/node-core": "^6.0.4",
+    "@subql/node-core": "^6.1.0",
     "@subql/testing": "^2.0.2",
     "@subql/types-ethereum": "workspace:*",
     "cacheable-lookup": "6",

diff --git a/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts b/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts
@@ -21,12 +21,14 @@ import {
   connectionPoolStateHostFunctions,
   baseWorkerFunctions,
   storeHostFunctions,
+  cacheHostFunctions,
   dynamicDsHostFunctions,
   IProjectUpgradeService,
   HostUnfinalizedBlocks,
   PoiSyncService,
+  InMemoryCacheService,
 } from '@subql/node-core';
-import { Store } from '@subql/types-core';
+import { Cache, Store } from '@subql/types-core';
 import {
   EthereumProjectDs,
   SubqueryProject,
@@ -43,6 +45,7 @@ type IndexerWorker = IIndexerWorker & {
 
 async function createIndexerWorker(
   store: Store,
+  cache: Cache,
   dynamicDsService: IDynamicDsService<EthereumProjectDs>,
   unfinalizedBlocksService: IUnfinalizedBlocksService<BlockContent>,
   connectionPoolState: ConnectionPoolStateManager<EthereumApiConnection>,
@@ -56,6 +59,7 @@ async function createIndexerWorker(
     path.resolve(__dirname, '../../../dist/indexer/worker/worker.js'),
     [...baseWorkerFunctions, 'initWorker'],
     {
+      ...cacheHostFunctions(cache),
       ...storeHostFunctions(store),
       ...dynamicDsHostFunctions(dynamicDsService),
       unfinalizedBlocksProcess:
@@ -85,6 +89,7 @@ export class WorkerBlockDispatcherService
     @Inject('IProjectUpgradeService')
     projectUpgadeService: IProjectUpgradeService,
     smartBatchService: SmartBatchService,
+    cacheService: InMemoryCacheService,
     storeService: StoreService,
     storeCacheService: StoreCacheService,
     poiService: PoiService,
@@ -109,6 +114,7 @@ export class WorkerBlockDispatcherService
       () =>
         createIndexerWorker(
           storeService.getStore(),
+          cacheService.getCache(),
           dynamicDsService,
           unfinalizedBlocksSevice,
           connectionPoolState,

diff --git a/packages/node/src/indexer/fetch.module.ts b/packages/node/src/indexer/fetch.module.ts
@@ -16,6 +16,7 @@ import {
   StoreCacheService,
   IProjectUpgradeService,
   PoiSyncService,
+  InMemoryCacheService,
 } from '@subql/node-core';
 import { SubqueryProject } from '../configure/SubqueryProject';
 import { EthereumApiConnection } from '../ethereum/api.connection';
@@ -35,6 +36,7 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service';
 
 @Module({
   providers: [
+    InMemoryCacheService,
     StoreService,
     StoreCacheService,
     {
@@ -88,6 +90,7 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service';
         apiService: EthereumApiService,
         indexerManager: IndexerManager,
         smartBatchService: SmartBatchService,
+        cacheService: InMemoryCacheService,
         storeService: StoreService,
         storeCacheService: StoreCacheService,
         poiService: PoiService,
@@ -104,6 +107,7 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service';
               projectService,
               projectUpgradeService,
               smartBatchService,
+              cacheService,
               storeService,
               storeCacheService,
               poiService,
@@ -136,6 +140,7 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service';
         ApiService,
         IndexerManager,
         SmartBatchService,
+        InMemoryCacheService,
         StoreService,
         StoreCacheService,
         PoiService,

diff --git a/packages/node/src/indexer/sandbox.service.ts b/packages/node/src/indexer/sandbox.service.ts
@@ -10,6 +10,7 @@ import {
   hostStoreToStore,
   ISubqueryProject,
   ApiService,
+  InMemoryCacheService,
 } from '@subql/node-core';
 import { BaseDataSource, Store } from '@subql/types-core';
 import SafeEthProvider from '../ethereum/safe-api';
@@ -23,6 +24,7 @@ export class SandboxService {
     private readonly apiService: ApiService,
     @Inject(isMainThread ? StoreService : 'Null')
     private readonly storeService: StoreService,
+    private readonly cacheService: InMemoryCacheService,
     private readonly nodeConfig: NodeConfig,
     @Inject('ISubqueryProject') private readonly project: ISubqueryProject,
   ) {}
@@ -32,11 +34,13 @@ export class SandboxService {
       ? this.storeService.getStore()
       : hostStoreToStore((global as any).host); // Provided in worker.ts
 
+    const cache = this.cacheService.getCache();
     const entry = this.getDataSourceEntry(ds);
     let processor = this.processorCache[entry];
     if (!processor) {
       processor = new IndexerSandbox(
         {
+          cache,
           store,
           root: this.project.root,
           entry,

diff --git a/packages/node/src/indexer/worker-fetch.module.ts b/packages/node/src/indexer/worker-fetch.module.ts
@@ -10,6 +10,7 @@ import {
   WorkerConnectionPoolStateManager,
   ConnectionPoolStateManager,
   NodeConfig,
+  InMemoryCacheService,
 } from '@subql/node-core';
 import { SubqueryProject } from '../configure/SubqueryProject';
 import { EthereumApiService } from '../ethereum';
@@ -20,6 +21,7 @@ import { IndexerManager } from './indexer.manager';
 import { ProjectService } from './project.service';
 import { SandboxService } from './sandbox.service';
 import { UnfinalizedBlocksService } from './unfinalizedBlocks.service';
+import { WorkerInMemoryCacheService } from './worker/worker.inMemoryCache.service';
 import { WorkerService } from './worker/worker.service';
 import { WorkerUnfinalizedBlocksService } from './worker/worker.unfinalizedBlocks.service';
 
@@ -72,6 +74,10 @@ import { WorkerUnfinalizedBlocksService } from './worker/worker.unfinalizedBlock
         new WorkerUnfinalizedBlocksService((global as any).host),
     },
     WorkerService,
+    {
+      provide: InMemoryCacheService,
+      useFactory: () => new WorkerInMemoryCacheService((global as any).host),
+    },
   ],
 })
 export class WorkerFetchModule {}
diff --git a/packages/node/src/indexer/worker/worker.inMemoryCache.service.ts b/packages/node/src/indexer/worker/worker.inMemoryCache.service.ts
@@ -0,0 +1,20 @@
+// Copyright 2020-2023 SubQuery Pte Ltd authors & contributors
+// SPDX-License-Identifier: GPL-3.0
+
+import { isMainThread } from 'worker_threads';
+import { Injectable } from '@nestjs/common';
+import { HostCache, hostCacheToCache } from '@subql/node-core';
+import { Cache } from '@subql/types-core';
+
+@Injectable()
+export class WorkerInMemoryCacheService {
+  constructor(private host: HostCache) {
+    if (isMainThread) {
+      throw new Error('Expected to be worker thread');
+    }
+  }
+
+  getCache(): Cache {
+    return hostCacheToCache(this.host);
+  }
+}
diff --git a/packages/types/CHANGELOG.md b/packages/types/CHANGELOG.md
@@ -6,6 +6,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+## [3.1.0] - 2023-10-20
+### Changed
+- Version bump with `types-core` 0.2.0
+
 ## [3.0.2] - 2023-10-05
 ### Fixed
 - Fixed RuntimeDatasourceTemplate's generic typing (#177)
@@ -75,7 +79,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Sync with main sdk (#14)
 
 ## [0.1.0] - 2022-10-31
-[Unreleased]: https://github.com/subquery/subql-ethereum/compare/types/3.0.2...HEAD
+[Unreleased]: https://github.com/subquery/subql-ethereum/compare/types-ethereum/3.1.0...HEAD
+[3.1.0]: https://github.com/subquery/subql-ethereum/compare/types-ethereum/3.0.2...types-ethereum/3.1.0
 [3.0.2]: https://github.com/subquery/subql-ethereum/compare/types/3.0.1...types/3.0.2
 [3.0.1]: https://github.com/subquery/subql-ethereum/compare/types/3.0.0...types/3.0.1
 [3.0.0]: https://github.com/subquery/subql-ethereum/compare/types/2.2.5...types/3.0.0

diff --git a/packages/types/package.json b/packages/types/package.json
@@ -1,6 +1,6 @@
 {
   "name": "@subql/types-ethereum",
-  "version": "3.0.2",
+  "version": "3.1.0",
   "description": "",
   "homepage": "https://github.com/subquery/subql",
   "repository": "github:subquery/subql",
@@ -18,6 +18,6 @@
   ],
   "dependencies": {
     "@ethersproject/abstract-provider": "^5.6.1",
-    "@subql/types-core": "0.1.1"
+    "@subql/types-core": "0.2.0"
   }
 }
diff --git a/yarn.lock b/yarn.lock
@@ -3210,9 +3210,9 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@subql/node-core@npm:^6.0.4":
-  version: 6.0.4
-  resolution: "@subql/node-core@npm:6.0.4"
+"@subql/node-core@npm:^6.1.0":
+  version: 6.1.0
+  resolution: "@subql/node-core@npm:6.1.0"
   dependencies:
     "@apollo/client": ^3.7.16
     "@nestjs/common": ^9.4.0
@@ -3221,7 +3221,7 @@ __metadata:
     "@subql/apollo-links": ^1.0.2
     "@subql/common": 3.1.3
     "@subql/testing": 2.1.0
-    "@subql/types": 3.1.3-1
+    "@subql/types": 3.2.0
     "@subql/utils": 2.4.5-0
     "@subql/x-sequelize": 6.32.0-0.0.2
     "@willsoto/nestjs-prometheus": ^5.4.0
@@ -3237,7 +3237,7 @@ __metadata:
     tar: ^6.1.11
     vm2: ^3.9.19
     yargs: ^16.2.0
-  checksum: 164a25af46d3ad47573bb8fb5e037707fe0835be995f24dff0210e0edd38589d7f15774f20b824bf8671d5c9f52bdca4dbecf854a27b78cb02459bd3d07f7ea3
+  checksum: 935abc9b53644eb7f2b87ae93764e07e03c3bfee00f90b922fc33224ce76c720d163b30d36b3f71f25ee62d085287b4df361d43744c4e682dc7a3ef29032343e
   languageName: node
   linkType: hard
 
@@ -3255,7 +3255,7 @@ __metadata:
     "@nestjs/testing": ^9.4.0
     "@subql/common": 3.1.3
     "@subql/common-ethereum": "workspace:*"
-    "@subql/node-core": ^6.0.4
+    "@subql/node-core": ^6.1.0
     "@subql/testing": ^2.0.2
     "@subql/types-ethereum": "workspace:*"
     "@types/express": ^4.17.13
@@ -3304,21 +3304,21 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@subql/types-core@npm:0.1.1":
-  version: 0.1.1
-  resolution: "@subql/types-core@npm:0.1.1"
+"@subql/types-core@npm:0.1.2-0, @subql/types-core@npm:^0.1.2-0":
+  version: 0.1.2-0
+  resolution: "@subql/types-core@npm:0.1.2-0"
   dependencies:
     package-json-type: ^1.0.3
-  checksum: 6a88547e5091795d2f9f24b5373ce531066bc04602b18f05cad77c5d1953523be479801447cd85e4f03468997e988b8c853fdb409631f8a396525cc723978afa
+  checksum: a70258c1cbe8aa407946ed4cd8fa13c6fea72caa0efa1837a4c400c535b0e4d483388d8c54eedd1afa483e65ac85706d637e0112230129e9313a90a552d7244f
   languageName: node
   linkType: hard
 
-"@subql/types-core@npm:0.1.2-0, @subql/types-core@npm:^0.1.2-0":
-  version: 0.1.2-0
-  resolution: "@subql/types-core@npm:0.1.2-0"
+"@subql/types-core@npm:0.2.0":
+  version: 0.2.0
+  resolution: "@subql/types-core@npm:0.2.0"
   dependencies:
     package-json-type: ^1.0.3
-  checksum: a70258c1cbe8aa407946ed4cd8fa13c6fea72caa0efa1837a4c400c535b0e4d483388d8c54eedd1afa483e65ac85706d637e0112230129e9313a90a552d7244f
+  checksum: ef538b84cb0c32f41269fdb6708e9e0657da9e7f0469ef24882e0b392a797eaf35364a7ef9368c0c89b799d6522a66cbf17781deae1edfdc8a17f9cd43aed95f
   languageName: node
   linkType: hard
 
@@ -3327,18 +3327,18 @@ __metadata:
   resolution: "@subql/types-ethereum@workspace:packages/types"
   dependencies:
     "@ethersproject/abstract-provider": ^5.6.1
-    "@subql/types-core": 0.1.1
+    "@subql/types-core": 0.2.0
   languageName: unknown
   linkType: soft
 
-"@subql/types@npm:3.1.3-1":
-  version: 3.1.3-1
-  resolution: "@subql/types@npm:3.1.3-1"
+"@subql/types@npm:3.2.0":
+  version: 3.2.0
+  resolution: "@subql/types@npm:3.2.0"
   dependencies:
-    "@subql/types-core": 0.1.2-0
+    "@subql/types-core": 0.2.0
   peerDependencies:
     "@polkadot/api": ^10
-  checksum: ce803e2dac06e242ba4d5444eb4d665e0f9a63e94d964fcb283f2ffe0749567c7be8805c0ec26247f2523f901528da0b63da16efbddbe7443d204bf2869ff03a
+  checksum: dbd592a2b7322bcea301bbcd320e1706f0af67330f8a0bf7893f5de5d7df58e139a635a69b4d0a1fc14b7b54972d898ebe54c8303ae3a139db892d673f74cc02
   languageName: node
   linkType: hard