From 52c2493c3a137abd5455bae355402ef03ac52fa5 Mon Sep 17 00:00:00 2001 From: JQQQ Date: Fri, 20 Oct 2023 15:18:10 +1300 Subject: [PATCH] [release] 20231020 eth --- packages/common-ethereum/CHANGELOG.md | 7 +++- packages/common-ethereum/package.json | 2 +- packages/node/CHANGELOG.md | 14 +++++-- packages/node/package.json | 4 +- .../worker-block-dispatcher.service.ts | 8 +++- packages/node/src/indexer/fetch.module.ts | 5 +++ packages/node/src/indexer/sandbox.service.ts | 4 ++ .../node/src/indexer/worker-fetch.module.ts | 6 +++ .../worker/worker.inMemoryCache.service.ts | 20 ++++++++++ packages/types/CHANGELOG.md | 7 +++- packages/types/package.json | 4 +- yarn.lock | 40 +++++++++---------- 12 files changed, 90 insertions(+), 31 deletions(-) create mode 100644 packages/node/src/indexer/worker/worker.inMemoryCache.service.ts diff --git a/packages/common-ethereum/CHANGELOG.md b/packages/common-ethereum/CHANGELOG.md index 77e3e9732c..abbf875181 100644 --- a/packages/common-ethereum/CHANGELOG.md +++ b/packages/common-ethereum/CHANGELOG.md @@ -6,6 +6,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [3.0.5] - 2023-10-20 +### Changed +- Version bump with `type-ethereum` 3.1.0 + ## [3.0.4] - 2023-10-18 ### Changed - Version bump with `common` 3.1.3 @@ -97,7 +101,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Sync with main sdk (#14) ## [0.1.0] - 2022-10-31 -[Unreleased]: https://github.com/subquery/subql-ethereum/compare/common-ethereum/3.0.4...HEAD +[Unreleased]: https://github.com/subquery/subql-ethereum/compare/common-ethereum/3.0.5...HEAD +[3.0.5]: https://github.com/subquery/subql-ethereum/compare/common-ethereum/3.0.4...common-ethereum/3.0.5 [3.0.4]: https://github.com/subquery/subql-ethereum/compare/common-ethereum/3.0.3...common-ethereum/3.0.4 [3.0.3]: https://github.com/subquery/subql-ethereum/compare/common-ethereum/3.0.2...common-ethereum/3.0.3 [3.0.2]: https://github.com/subquery/subql-ethereum/compare/common-ethereum/3.0.1...common-ethereum/3.0.2 diff --git a/packages/common-ethereum/package.json b/packages/common-ethereum/package.json index 25aee7bd38..620a1fa454 100644 --- a/packages/common-ethereum/package.json +++ b/packages/common-ethereum/package.json @@ -1,6 +1,6 @@ { "name": "@subql/common-ethereum", - "version": "3.0.4", + "version": "3.0.5", "description": "", "scripts": { "build": "rm -rf dist && tsc -b", diff --git a/packages/node/CHANGELOG.md b/packages/node/CHANGELOG.md index 9ee28bebb8..a7b730636e 100644 --- a/packages/node/CHANGELOG.md +++ b/packages/node/CHANGELOG.md @@ -6,13 +6,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [3.1.0] - 2023-10-20 +### Added +- Inject in-memory cache to sandbox + +### Fixed +- Bump with `@subq/node-core` 3.1.0 , fixed poi migration init check, and improve logging + ## [3.0.6] - 2023-10-18 ### Fixed - Update node-core, additional fix for store bulk methods failing with workers ## [3.0.5] - 2023-10-17 ### Fixed -- Update `@subql/node-core` to fix: +- Update - Dictionary validation error causing application exit (#2101) - Auto queue flush getting the queue into a bad state (#2103) - Fix getCache could not been cleared after reindex, and could have been re-used and lead to error, such as syncPoi @@ -20,6 +27,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [3.0.4] - 2023-10-12 ### Changed - debug has changed from a boolean to a string to allow scoping debug log level (#2077) + ### Fixed - Sync with node-core. - Fix issues with using object destructing for store interface and workers @@ -285,11 +293,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added - Init release -[Unreleased]: https://github.com/subquery/subql-ethereum/compare/node-ethereum/3.0.6...HEAD +[Unreleased]: https://github.com/subquery/subql-ethereum/compare/node-ethereum/3.1.0...HEAD +[3.1.0]: https://github.com/subquery/subql-ethereum/compare/node-ethereum/3.0.6...node-ethereum/3.1.0 [3.0.6]: https://github.com/subquery/subql-ethereum/compare/node-ethereum/3.0.5...node-ethereum/3.0.6 [3.0.5]: https://github.com/subquery/subql-ethereum/compare/node-ethereum/3.0.4...node-ethereum/3.0.5 [3.0.4]: https://github.com/subquery/subql-ethereum/compare/node-ethereum/3.0.3...node-ethereum/3.0.4 -[3.0.4]: https://github.com/subquery/subql-ethereum/compare/node-ethereum/3.0.3...node-ethereum/3.0.4 [3.0.3]: https://github.com/subquery/subql-ethereum/compare/node-ethereum/3.0.2...node-ethereum/3.0.3 [3.0.2]: https://github.com/subquery/subql-ethereum/compare/node-ethereum/3.0.1...node-ethereum/3.0.2 [3.0.1]: https://github.com/subquery/subql-ethereum/compare/node-ethereum/3.0.0...node-ethereum/3.0.1 diff --git a/packages/node/package.json b/packages/node/package.json index fc57019d0a..7a1ef750f0 100644 --- a/packages/node/package.json +++ b/packages/node/package.json @@ -1,6 +1,6 @@ { "name": "@subql/node-ethereum", - "version": "3.0.6", + "version": "3.1.0", "description": "", "author": "Ian He", "license": "GPL-3.0", @@ -27,7 +27,7 @@ "@nestjs/schedule": "^3.0.1", "@subql/common": "3.1.3", "@subql/common-ethereum": "workspace:*", - "@subql/node-core": "^6.0.4", + "@subql/node-core": "^6.1.0", "@subql/testing": "^2.0.2", "@subql/types-ethereum": "workspace:*", "cacheable-lookup": "6", diff --git a/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts b/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts index edfb59645c..9e20a7404b 100644 --- a/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts +++ b/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts @@ -21,12 +21,14 @@ import { connectionPoolStateHostFunctions, baseWorkerFunctions, storeHostFunctions, + cacheHostFunctions, dynamicDsHostFunctions, IProjectUpgradeService, HostUnfinalizedBlocks, PoiSyncService, + InMemoryCacheService, } from '@subql/node-core'; -import { Store } from '@subql/types-core'; +import { Cache, Store } from '@subql/types-core'; import { EthereumProjectDs, SubqueryProject, @@ -43,6 +45,7 @@ type IndexerWorker = IIndexerWorker & { async function createIndexerWorker( store: Store, + cache: Cache, dynamicDsService: IDynamicDsService, unfinalizedBlocksService: IUnfinalizedBlocksService, connectionPoolState: ConnectionPoolStateManager, @@ -56,6 +59,7 @@ async function createIndexerWorker( path.resolve(__dirname, '../../../dist/indexer/worker/worker.js'), [...baseWorkerFunctions, 'initWorker'], { + ...cacheHostFunctions(cache), ...storeHostFunctions(store), ...dynamicDsHostFunctions(dynamicDsService), unfinalizedBlocksProcess: @@ -85,6 +89,7 @@ export class WorkerBlockDispatcherService @Inject('IProjectUpgradeService') projectUpgadeService: IProjectUpgradeService, smartBatchService: SmartBatchService, + cacheService: InMemoryCacheService, storeService: StoreService, storeCacheService: StoreCacheService, poiService: PoiService, @@ -109,6 +114,7 @@ export class WorkerBlockDispatcherService () => createIndexerWorker( storeService.getStore(), + cacheService.getCache(), dynamicDsService, unfinalizedBlocksSevice, connectionPoolState, diff --git a/packages/node/src/indexer/fetch.module.ts b/packages/node/src/indexer/fetch.module.ts index 21dda083a4..d36547d754 100644 --- a/packages/node/src/indexer/fetch.module.ts +++ b/packages/node/src/indexer/fetch.module.ts @@ -16,6 +16,7 @@ import { StoreCacheService, IProjectUpgradeService, PoiSyncService, + InMemoryCacheService, } from '@subql/node-core'; import { SubqueryProject } from '../configure/SubqueryProject'; import { EthereumApiConnection } from '../ethereum/api.connection'; @@ -35,6 +36,7 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service'; @Module({ providers: [ + InMemoryCacheService, StoreService, StoreCacheService, { @@ -88,6 +90,7 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service'; apiService: EthereumApiService, indexerManager: IndexerManager, smartBatchService: SmartBatchService, + cacheService: InMemoryCacheService, storeService: StoreService, storeCacheService: StoreCacheService, poiService: PoiService, @@ -104,6 +107,7 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service'; projectService, projectUpgradeService, smartBatchService, + cacheService, storeService, storeCacheService, poiService, @@ -136,6 +140,7 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service'; ApiService, IndexerManager, SmartBatchService, + InMemoryCacheService, StoreService, StoreCacheService, PoiService, diff --git a/packages/node/src/indexer/sandbox.service.ts b/packages/node/src/indexer/sandbox.service.ts index 4dba1de32a..83b98faa8f 100644 --- a/packages/node/src/indexer/sandbox.service.ts +++ b/packages/node/src/indexer/sandbox.service.ts @@ -10,6 +10,7 @@ import { hostStoreToStore, ISubqueryProject, ApiService, + InMemoryCacheService, } from '@subql/node-core'; import { BaseDataSource, Store } from '@subql/types-core'; import SafeEthProvider from '../ethereum/safe-api'; @@ -23,6 +24,7 @@ export class SandboxService { private readonly apiService: ApiService, @Inject(isMainThread ? StoreService : 'Null') private readonly storeService: StoreService, + private readonly cacheService: InMemoryCacheService, private readonly nodeConfig: NodeConfig, @Inject('ISubqueryProject') private readonly project: ISubqueryProject, ) {} @@ -32,11 +34,13 @@ export class SandboxService { ? this.storeService.getStore() : hostStoreToStore((global as any).host); // Provided in worker.ts + const cache = this.cacheService.getCache(); const entry = this.getDataSourceEntry(ds); let processor = this.processorCache[entry]; if (!processor) { processor = new IndexerSandbox( { + cache, store, root: this.project.root, entry, diff --git a/packages/node/src/indexer/worker-fetch.module.ts b/packages/node/src/indexer/worker-fetch.module.ts index 231a414b27..5f2e9c4b29 100644 --- a/packages/node/src/indexer/worker-fetch.module.ts +++ b/packages/node/src/indexer/worker-fetch.module.ts @@ -10,6 +10,7 @@ import { WorkerConnectionPoolStateManager, ConnectionPoolStateManager, NodeConfig, + InMemoryCacheService, } from '@subql/node-core'; import { SubqueryProject } from '../configure/SubqueryProject'; import { EthereumApiService } from '../ethereum'; @@ -20,6 +21,7 @@ import { IndexerManager } from './indexer.manager'; import { ProjectService } from './project.service'; import { SandboxService } from './sandbox.service'; import { UnfinalizedBlocksService } from './unfinalizedBlocks.service'; +import { WorkerInMemoryCacheService } from './worker/worker.inMemoryCache.service'; import { WorkerService } from './worker/worker.service'; import { WorkerUnfinalizedBlocksService } from './worker/worker.unfinalizedBlocks.service'; @@ -72,6 +74,10 @@ import { WorkerUnfinalizedBlocksService } from './worker/worker.unfinalizedBlock new WorkerUnfinalizedBlocksService((global as any).host), }, WorkerService, + { + provide: InMemoryCacheService, + useFactory: () => new WorkerInMemoryCacheService((global as any).host), + }, ], }) export class WorkerFetchModule {} diff --git a/packages/node/src/indexer/worker/worker.inMemoryCache.service.ts b/packages/node/src/indexer/worker/worker.inMemoryCache.service.ts new file mode 100644 index 0000000000..1011f95455 --- /dev/null +++ b/packages/node/src/indexer/worker/worker.inMemoryCache.service.ts @@ -0,0 +1,20 @@ +// Copyright 2020-2023 SubQuery Pte Ltd authors & contributors +// SPDX-License-Identifier: GPL-3.0 + +import { isMainThread } from 'worker_threads'; +import { Injectable } from '@nestjs/common'; +import { HostCache, hostCacheToCache } from '@subql/node-core'; +import { Cache } from '@subql/types-core'; + +@Injectable() +export class WorkerInMemoryCacheService { + constructor(private host: HostCache) { + if (isMainThread) { + throw new Error('Expected to be worker thread'); + } + } + + getCache(): Cache { + return hostCacheToCache(this.host); + } +} diff --git a/packages/types/CHANGELOG.md b/packages/types/CHANGELOG.md index 6909ae9535..0428e251ca 100644 --- a/packages/types/CHANGELOG.md +++ b/packages/types/CHANGELOG.md @@ -6,6 +6,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [3.1.0] - 2023-10-20 +### Changed +- Version bump with `types-core` 0.2.0 + ## [3.0.2] - 2023-10-05 ### Fixed - Fixed RuntimeDatasourceTemplate's generic typing (#177) @@ -75,7 +79,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Sync with main sdk (#14) ## [0.1.0] - 2022-10-31 -[Unreleased]: https://github.com/subquery/subql-ethereum/compare/types/3.0.2...HEAD +[Unreleased]: https://github.com/subquery/subql-ethereum/compare/types-ethereum/3.1.0...HEAD +[3.1.0]: https://github.com/subquery/subql-ethereum/compare/types-ethereum/3.0.2...types-ethereum/3.1.0 [3.0.2]: https://github.com/subquery/subql-ethereum/compare/types/3.0.1...types/3.0.2 [3.0.1]: https://github.com/subquery/subql-ethereum/compare/types/3.0.0...types/3.0.1 [3.0.0]: https://github.com/subquery/subql-ethereum/compare/types/2.2.5...types/3.0.0 diff --git a/packages/types/package.json b/packages/types/package.json index edf1ad6614..172125d8e7 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -1,6 +1,6 @@ { "name": "@subql/types-ethereum", - "version": "3.0.2", + "version": "3.1.0", "description": "", "homepage": "https://github.com/subquery/subql", "repository": "github:subquery/subql", @@ -18,6 +18,6 @@ ], "dependencies": { "@ethersproject/abstract-provider": "^5.6.1", - "@subql/types-core": "0.1.1" + "@subql/types-core": "0.2.0" } } diff --git a/yarn.lock b/yarn.lock index ca7b509ce8..c09a407454 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3210,9 +3210,9 @@ __metadata: languageName: node linkType: hard -"@subql/node-core@npm:^6.0.4": - version: 6.0.4 - resolution: "@subql/node-core@npm:6.0.4" +"@subql/node-core@npm:^6.1.0": + version: 6.1.0 + resolution: "@subql/node-core@npm:6.1.0" dependencies: "@apollo/client": ^3.7.16 "@nestjs/common": ^9.4.0 @@ -3221,7 +3221,7 @@ __metadata: "@subql/apollo-links": ^1.0.2 "@subql/common": 3.1.3 "@subql/testing": 2.1.0 - "@subql/types": 3.1.3-1 + "@subql/types": 3.2.0 "@subql/utils": 2.4.5-0 "@subql/x-sequelize": 6.32.0-0.0.2 "@willsoto/nestjs-prometheus": ^5.4.0 @@ -3237,7 +3237,7 @@ __metadata: tar: ^6.1.11 vm2: ^3.9.19 yargs: ^16.2.0 - checksum: 164a25af46d3ad47573bb8fb5e037707fe0835be995f24dff0210e0edd38589d7f15774f20b824bf8671d5c9f52bdca4dbecf854a27b78cb02459bd3d07f7ea3 + checksum: 935abc9b53644eb7f2b87ae93764e07e03c3bfee00f90b922fc33224ce76c720d163b30d36b3f71f25ee62d085287b4df361d43744c4e682dc7a3ef29032343e languageName: node linkType: hard @@ -3255,7 +3255,7 @@ __metadata: "@nestjs/testing": ^9.4.0 "@subql/common": 3.1.3 "@subql/common-ethereum": "workspace:*" - "@subql/node-core": ^6.0.4 + "@subql/node-core": ^6.1.0 "@subql/testing": ^2.0.2 "@subql/types-ethereum": "workspace:*" "@types/express": ^4.17.13 @@ -3304,21 +3304,21 @@ __metadata: languageName: node linkType: hard -"@subql/types-core@npm:0.1.1": - version: 0.1.1 - resolution: "@subql/types-core@npm:0.1.1" +"@subql/types-core@npm:0.1.2-0, @subql/types-core@npm:^0.1.2-0": + version: 0.1.2-0 + resolution: "@subql/types-core@npm:0.1.2-0" dependencies: package-json-type: ^1.0.3 - checksum: 6a88547e5091795d2f9f24b5373ce531066bc04602b18f05cad77c5d1953523be479801447cd85e4f03468997e988b8c853fdb409631f8a396525cc723978afa + checksum: a70258c1cbe8aa407946ed4cd8fa13c6fea72caa0efa1837a4c400c535b0e4d483388d8c54eedd1afa483e65ac85706d637e0112230129e9313a90a552d7244f languageName: node linkType: hard -"@subql/types-core@npm:0.1.2-0, @subql/types-core@npm:^0.1.2-0": - version: 0.1.2-0 - resolution: "@subql/types-core@npm:0.1.2-0" +"@subql/types-core@npm:0.2.0": + version: 0.2.0 + resolution: "@subql/types-core@npm:0.2.0" dependencies: package-json-type: ^1.0.3 - checksum: a70258c1cbe8aa407946ed4cd8fa13c6fea72caa0efa1837a4c400c535b0e4d483388d8c54eedd1afa483e65ac85706d637e0112230129e9313a90a552d7244f + checksum: ef538b84cb0c32f41269fdb6708e9e0657da9e7f0469ef24882e0b392a797eaf35364a7ef9368c0c89b799d6522a66cbf17781deae1edfdc8a17f9cd43aed95f languageName: node linkType: hard @@ -3327,18 +3327,18 @@ __metadata: resolution: "@subql/types-ethereum@workspace:packages/types" dependencies: "@ethersproject/abstract-provider": ^5.6.1 - "@subql/types-core": 0.1.1 + "@subql/types-core": 0.2.0 languageName: unknown linkType: soft -"@subql/types@npm:3.1.3-1": - version: 3.1.3-1 - resolution: "@subql/types@npm:3.1.3-1" +"@subql/types@npm:3.2.0": + version: 3.2.0 + resolution: "@subql/types@npm:3.2.0" dependencies: - "@subql/types-core": 0.1.2-0 + "@subql/types-core": 0.2.0 peerDependencies: "@polkadot/api": ^10 - checksum: ce803e2dac06e242ba4d5444eb4d665e0f9a63e94d964fcb283f2ffe0749567c7be8805c0ec26247f2523f901528da0b63da16efbddbe7443d204bf2869ff03a + checksum: dbd592a2b7322bcea301bbcd320e1706f0af67330f8a0bf7893f5de5d7df58e139a635a69b4d0a1fc14b7b54972d898ebe54c8303ae3a139db892d673f74cc02 languageName: node linkType: hard