From b67cad1c98faddcfbfad686511fdaa19fcde14ac Mon Sep 17 00:00:00 2001 From: Scott Twiname Date: Tue, 30 Apr 2024 09:38:49 +1200 Subject: [PATCH] Sync @subql dependencies (#40) * Update @subql deps * Apply changes relating to updated deps * Update changelogs --------- Co-authored-by: stwiname --- packages/common-concordium/CHANGELOG.md | 2 + packages/common-concordium/package.json | 2 +- .../common-concordium/src/project/utils.ts | 18 +-- packages/node/CHANGELOG.md | 2 + packages/node/package.json | 4 +- .../block-dispatcher.service.ts | 6 - .../worker-block-dispatcher.service.ts | 4 - .../src/indexer/dictionary/v1/dictionaryV1.ts | 7 +- .../node/src/indexer/ds-processor.service.ts | 65 +---------- .../node/src/indexer/dynamic-ds.service.ts | 28 ++--- packages/node/src/indexer/fetch.module.ts | 15 +-- packages/node/src/indexer/fetch.service.ts | 4 - packages/node/src/indexer/indexer.manager.ts | 12 +- packages/node/src/indexer/project.service.ts | 2 +- packages/node/src/indexer/sandbox.service.ts | 64 ----------- .../src/indexer/worker/worker-fetch.module.ts | 4 +- .../worker.unfinalizedBlocks.service.ts | 50 --------- .../node/src/subcommands/forceClean.init.ts | 18 +-- packages/node/src/subcommands/reindex.init.ts | 28 +---- .../node/src/subcommands/testing.module.ts | 2 +- packages/types/CHANGELOG.md | 2 + packages/types/package.json | 2 +- packages/types/src/project.ts | 105 +++--------------- yarn.lock | 97 ++++++++-------- 24 files changed, 113 insertions(+), 430 deletions(-) delete mode 100644 packages/node/src/indexer/sandbox.service.ts delete mode 100644 packages/node/src/indexer/worker/worker.unfinalizedBlocks.service.ts diff --git a/packages/common-concordium/CHANGELOG.md b/packages/common-concordium/CHANGELOG.md index 8dd55aff..8513f8b8 100644 --- a/packages/common-concordium/CHANGELOG.md +++ b/packages/common-concordium/CHANGELOG.md @@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Changed +- Update dependencies and apply changes to match (#40) ## [3.5.1] - 2024-04-11 ### Fixed diff --git a/packages/common-concordium/package.json b/packages/common-concordium/package.json index d2dbd6bf..0c9654b9 100644 --- a/packages/common-concordium/package.json +++ b/packages/common-concordium/package.json @@ -15,7 +15,7 @@ "license": "GPL-3.0", "dependencies": { "@concordium/node-sdk": "9.4.0", - "@subql/common": "^3.5.0", + "@subql/common": "^3.5.1", "@subql/types-concordium": "workspace:*", "js-yaml": "^4.1.0", "pino": "^6.13.3", diff --git a/packages/common-concordium/src/project/utils.ts b/packages/common-concordium/src/project/utils.ts index 9726f98e..2d5de6a8 100644 --- a/packages/common-concordium/src/project/utils.ts +++ b/packages/common-concordium/src/project/utils.ts @@ -10,27 +10,29 @@ import { ConcordiumRuntimeDatasource, } from '@subql/types-concordium'; +type DefaultFilter = Record; + export function isBlockHandlerProcessor( - hp: SecondLayerHandlerProcessor -): hp is SecondLayerHandlerProcessor { + hp: SecondLayerHandlerProcessor +): hp is SecondLayerHandlerProcessor { return hp.baseHandlerKind === ConcordiumHandlerKind.Block; } export function isTransactionHandlerProcessor( - hp: SecondLayerHandlerProcessor -): hp is SecondLayerHandlerProcessor { + hp: SecondLayerHandlerProcessor +): hp is SecondLayerHandlerProcessor { return hp.baseHandlerKind === ConcordiumHandlerKind.Transaction; } export function isTransactionEventHandlerProcessor( - hp: SecondLayerHandlerProcessor -): hp is SecondLayerHandlerProcessor { + hp: SecondLayerHandlerProcessor +): hp is SecondLayerHandlerProcessor { return hp.baseHandlerKind === ConcordiumHandlerKind.TransactionEvent; } export function isSpecialEventHandlerProcessor( - hp: SecondLayerHandlerProcessor -): hp is SecondLayerHandlerProcessor { + hp: SecondLayerHandlerProcessor +): hp is SecondLayerHandlerProcessor { return hp.baseHandlerKind === ConcordiumHandlerKind.SpecialEvent; } diff --git a/packages/node/CHANGELOG.md b/packages/node/CHANGELOG.md index f4cd7de1..2f704368 100644 --- a/packages/node/CHANGELOG.md +++ b/packages/node/CHANGELOG.md @@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Changed +- Update dependencies and apply changes to match (#40) ## [3.10.1] - 2024-04-11 ### Fixed diff --git a/packages/node/package.json b/packages/node/package.json index 8fe05859..d79c5396 100644 --- a/packages/node/package.json +++ b/packages/node/package.json @@ -26,9 +26,9 @@ "@nestjs/event-emitter": "^2.0.0", "@nestjs/platform-express": "^9.4.0", "@nestjs/schedule": "^3.0.1", - "@subql/common": "^3.5.0", + "@subql/common": "^3.5.1", "@subql/common-concordium": "workspace:*", - "@subql/node-core": "^8.0.1", + "@subql/node-core": "^10.0.0", "@subql/types-concordium": "workspace:*", "cacheable-lookup": "6", "cron-converter": "^1.0.2", diff --git a/packages/node/src/indexer/blockDispatcher/block-dispatcher.service.ts b/packages/node/src/indexer/blockDispatcher/block-dispatcher.service.ts index 4a17dbcf..0aff16e7 100644 --- a/packages/node/src/indexer/blockDispatcher/block-dispatcher.service.ts +++ b/packages/node/src/indexer/blockDispatcher/block-dispatcher.service.ts @@ -5,7 +5,6 @@ import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; import { EventEmitter2 } from '@nestjs/event-emitter'; import { NodeConfig, - SmartBatchService, StoreCacheService, StoreService, IProjectService, @@ -18,7 +17,6 @@ import { } from '@subql/node-core'; import { ConcordiumBlock, ConcordiumDatasource } from '@subql/types-concordium'; import { SubqueryProject } from '../../configure/SubqueryProject'; -import { DynamicDsService } from '../dynamic-ds.service'; import { IndexerManager } from '../indexer.manager'; /** @@ -38,24 +36,20 @@ export class BlockDispatcherService projectService: IProjectService, @Inject('IProjectUpgradeService') projectUpgradeService: IProjectUpgradeService, - smartBatchService: SmartBatchService, storeService: StoreService, storeCacheService: StoreCacheService, poiSyncService: PoiSyncService, @Inject('ISubqueryProject') project: SubqueryProject, - dynamicDsService: DynamicDsService, ) { super( nodeConfig, eventEmitter, projectService, projectUpgradeService, - smartBatchService, storeService, storeCacheService, poiSyncService, project, - dynamicDsService, apiService.fetchBlocks.bind(apiService), ); } diff --git a/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts b/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts index 6307dd3d..589c872f 100644 --- a/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts +++ b/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts @@ -6,7 +6,6 @@ import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common'; import { EventEmitter2 } from '@nestjs/event-emitter'; import { NodeConfig, - SmartBatchService, StoreService, StoreCacheService, IProjectService, @@ -45,7 +44,6 @@ export class WorkerBlockDispatcherService projectService: IProjectService, @Inject('IProjectUpgradeService') projectUpgadeService: IProjectUpgradeService, - smartBatchService: SmartBatchService, cacheService: InMemoryCacheService, storeService: StoreService, storeCacheService: StoreCacheService, @@ -60,12 +58,10 @@ export class WorkerBlockDispatcherService eventEmitter, projectService, projectUpgadeService, - smartBatchService, storeService, storeCacheService, poiSyncService, project, - dynamicDsService, () => createIndexerWorker< IIndexerWorker, diff --git a/packages/node/src/indexer/dictionary/v1/dictionaryV1.ts b/packages/node/src/indexer/dictionary/v1/dictionaryV1.ts index 2ce46049..232c32fa 100644 --- a/packages/node/src/indexer/dictionary/v1/dictionaryV1.ts +++ b/packages/node/src/indexer/dictionary/v1/dictionaryV1.ts @@ -7,11 +7,14 @@ import { ConcordiumSpecialEventFilter, ConcordiumTransactionEventFilter, ConcordiumTransactionFilter, - DictionaryQueryCondition, ConcordiumDatasource, ConcordiumHandlerKind, } from '@subql/types-concordium'; -import { DictionaryQueryEntry, DsProcessor } from '@subql/types-core'; +import { + DictionaryQueryCondition, + DictionaryQueryEntry, + DsProcessor, +} from '@subql/types-core'; import { setWith, sortBy, uniqBy } from 'lodash'; import { SubqueryProject } from '../../../configure/SubqueryProject'; diff --git a/packages/node/src/indexer/ds-processor.service.ts b/packages/node/src/indexer/ds-processor.service.ts index ab366afd..3ac0f71d 100644 --- a/packages/node/src/indexer/ds-processor.service.ts +++ b/packages/node/src/indexer/ds-processor.service.ts @@ -2,76 +2,13 @@ // SPDX-License-Identifier: GPL-3.0 import { Injectable } from '@nestjs/common'; -import { - ConcordiumHandlerKind, - isCustomDs, - SubqlDatasourceProcessor, -} from '@subql/common-concordium'; +import { isCustomDs, SubqlDatasourceProcessor } from '@subql/common-concordium'; import { BaseDsProcessorService } from '@subql/node-core'; import { ConcordiumCustomDatasource, ConcordiumDatasource, - SecondLayerHandlerProcessor_0_0_0, - SecondLayerHandlerProcessor_1_0_0, } from '@subql/types-concordium'; -export function isSecondLayerHandlerProcessor_0_0_0< - K extends ConcordiumHandlerKind, - F extends Record, - E, - DS extends ConcordiumCustomDatasource = ConcordiumCustomDatasource, ->( - processor: - | SecondLayerHandlerProcessor_0_0_0 - | SecondLayerHandlerProcessor_1_0_0, -): processor is SecondLayerHandlerProcessor_0_0_0 { - // Exisiting datasource processors had no concept of specVersion, therefore undefined is equivalent to 0.0.0 - return processor.specVersion === undefined; -} - -export function isSecondLayerHandlerProcessor_1_0_0< - K extends ConcordiumHandlerKind, - F extends Record, - E, - DS extends ConcordiumCustomDatasource = ConcordiumCustomDatasource, ->( - processor: - | SecondLayerHandlerProcessor_0_0_0 - | SecondLayerHandlerProcessor_1_0_0, -): processor is SecondLayerHandlerProcessor_1_0_0 { - return processor.specVersion === '1.0.0'; -} - -export function asSecondLayerHandlerProcessor_1_0_0< - K extends ConcordiumHandlerKind, - F extends Record, - E, - DS extends ConcordiumCustomDatasource = ConcordiumCustomDatasource, ->( - processor: - | SecondLayerHandlerProcessor_0_0_0 - | SecondLayerHandlerProcessor_1_0_0, -): SecondLayerHandlerProcessor_1_0_0 { - if (isSecondLayerHandlerProcessor_1_0_0(processor)) { - return processor; - } - - if (!isSecondLayerHandlerProcessor_0_0_0(processor)) { - throw new Error('Unsupported ds processor version'); - } - - return { - ...processor, - specVersion: '1.0.0', - filterProcessor: (params) => - processor.filterProcessor(params.filter, params.input, params.ds), - transformer: (params) => - processor - .transformer(params.input, params.ds, params.api, params.assets) - .then((res) => [res]), - }; -} - @Injectable() export class DsProcessorService extends BaseDsProcessorService< ConcordiumDatasource, diff --git a/packages/node/src/indexer/dynamic-ds.service.ts b/packages/node/src/indexer/dynamic-ds.service.ts index c4fe1fe9..8fc6bbcd 100644 --- a/packages/node/src/indexer/dynamic-ds.service.ts +++ b/packages/node/src/indexer/dynamic-ds.service.ts @@ -14,38 +14,28 @@ import { import { ConcordiumDatasource } from '@subql/types-concordium'; import { plainToClass } from 'class-transformer'; import { validateSync } from 'class-validator'; -import { cloneDeep } from 'lodash'; import { SubqueryProject } from '../configure/SubqueryProject'; import { DsProcessorService } from './ds-processor.service'; @Injectable() -export class DynamicDsService extends BaseDynamicDsService { +export class DynamicDsService extends BaseDynamicDsService< + ConcordiumDatasource, + SubqueryProject +> { constructor( private readonly dsProcessorService: DsProcessorService, - @Inject('ISubqueryProject') private readonly project: SubqueryProject, + @Inject('ISubqueryProject') project: SubqueryProject, ) { - super(); + super(project); } protected async getDatasource( params: DatasourceParams, ): Promise { - const t = this.project.templates?.find( - (t) => t.name === params.templateName, + const dsObj = this.getTemplate( + params.templateName, + params.startBlock, ); - - if (!t) { - throw new Error( - `Unable to find matching template in project for name: "${params.templateName}"`, - ); - } - - const { name, ...template } = cloneDeep(t); - - const dsObj = { - ...template, - startBlock: params.startBlock, - } as ConcordiumDatasource; try { if (isCustomDs(dsObj)) { dsObj.processor.options = { diff --git a/packages/node/src/indexer/fetch.module.ts b/packages/node/src/indexer/fetch.module.ts index 325a3af2..c38527a3 100644 --- a/packages/node/src/indexer/fetch.module.ts +++ b/packages/node/src/indexer/fetch.module.ts @@ -12,11 +12,11 @@ import { NodeConfig, ConnectionPoolService, ConnectionPoolStateManager, - SmartBatchService, StoreCacheService, IProjectUpgradeService, PoiSyncService, InMemoryCacheService, + SandboxService, } from '@subql/node-core'; import { ConcordiumApiConnection } from '../concordium/api.connection'; import { ConcordiumApiService } from '../concordium/api.service.concordium'; @@ -31,7 +31,6 @@ import { DynamicDsService } from './dynamic-ds.service'; import { FetchService } from './fetch.service'; import { IndexerManager } from './indexer.manager'; import { ProjectService } from './project.service'; -import { SandboxService } from './sandbox.service'; import { UnfinalizedBlocksService } from './unfinalizedBlocks.service'; @Module({ @@ -67,13 +66,6 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service'; ConnectionPoolStateManager, ConnectionPoolService, PoiSyncService, - { - provide: SmartBatchService, - useFactory: (nodeConfig: NodeConfig) => { - return new SmartBatchService(nodeConfig.batchSize); - }, - inject: [NodeConfig], - }, { provide: 'IBlockDispatcher', useFactory: ( @@ -83,7 +75,6 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service'; projectUpgradeService: IProjectUpgradeService, apiService: ConcordiumApiService, indexerManager: IndexerManager, - smartBatchService: SmartBatchService, cacheService: InMemoryCacheService, storeService: StoreService, storeCacheService: StoreCacheService, @@ -99,7 +90,6 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service'; eventEmitter, projectService, projectUpgradeService, - smartBatchService, cacheService, storeService, storeCacheService, @@ -116,12 +106,10 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service'; eventEmitter, projectService, projectUpgradeService, - smartBatchService, storeService, storeCacheService, poiSyncService, project, - dynamicDsService, ), inject: [ NodeConfig, @@ -130,7 +118,6 @@ import { UnfinalizedBlocksService } from './unfinalizedBlocks.service'; 'IProjectUpgradeService', ApiService, IndexerManager, - SmartBatchService, InMemoryCacheService, StoreService, StoreCacheService, diff --git a/packages/node/src/indexer/fetch.service.ts b/packages/node/src/indexer/fetch.service.ts index bfa7f73f..df71c654 100644 --- a/packages/node/src/indexer/fetch.service.ts +++ b/packages/node/src/indexer/fetch.service.ts @@ -65,10 +65,6 @@ export class FetchService extends BaseFetchService< return this.apiService.unsafeApi; } - protected getGenesisHash(): string { - return this.apiService.networkMeta.genesisHash; - } - protected async getFinalizedHeight(): Promise { const block = await this.api.getFinalizedBlock(); diff --git a/packages/node/src/indexer/indexer.manager.ts b/packages/node/src/indexer/indexer.manager.ts index ba21fe2b..302dcfa9 100644 --- a/packages/node/src/indexer/indexer.manager.ts +++ b/packages/node/src/indexer/indexer.manager.ts @@ -20,6 +20,7 @@ import { BaseIndexerManager, ApiService, IBlock, + SandboxService, } from '@subql/node-core'; import { ConcordiumTransaction, @@ -41,21 +42,17 @@ import { filterTxEventProcessor, } from '../concordium/block.concordium'; import SafeConcordiumGRPCClient from '../concordium/safe-api'; -import { - asSecondLayerHandlerProcessor_1_0_0, - DsProcessorService, -} from './ds-processor.service'; +import { DsProcessorService } from './ds-processor.service'; import { DynamicDsService } from './dynamic-ds.service'; import { ProjectService } from './project.service'; -import { SandboxService } from './sandbox.service'; import { UnfinalizedBlocksService } from './unfinalizedBlocks.service'; const logger = getLogger('indexer'); @Injectable() export class IndexerManager extends BaseIndexerManager< - SafeConcordiumGRPCClient, ConcordiumApi, + SafeConcordiumGRPCClient, ConcordiumBlock, ApiService, ConcordiumDatasource, @@ -66,12 +63,11 @@ export class IndexerManager extends BaseIndexerManager< > { protected isRuntimeDs = isRuntimeDs; protected isCustomDs = isCustomDs; - protected updateCustomProcessor = asSecondLayerHandlerProcessor_1_0_0; constructor( apiService: ApiService, nodeConfig: NodeConfig, - sandboxService: SandboxService, + sandboxService: SandboxService, dsProcessorService: DsProcessorService, dynamicDsService: DynamicDsService, unfinalizedBlocksService: UnfinalizedBlocksService, diff --git a/packages/node/src/indexer/project.service.ts b/packages/node/src/indexer/project.service.ts index 9ccb4916..387bca70 100644 --- a/packages/node/src/indexer/project.service.ts +++ b/packages/node/src/indexer/project.service.ts @@ -48,7 +48,7 @@ export class ProjectService extends BaseProjectService< @Inject(isMainThread ? Sequelize : 'Null') sequelize: Sequelize, @Inject('ISubqueryProject') project: SubqueryProject, @Inject('IProjectUpgradeService') - protected readonly projectUpgradeService: IProjectUpgradeService, + projectUpgradeService: IProjectUpgradeService, @Inject(isMainThread ? StoreService : 'Null') storeService: StoreService, nodeConfig: NodeConfig, dynamicDsService: DynamicDsService, diff --git a/packages/node/src/indexer/sandbox.service.ts b/packages/node/src/indexer/sandbox.service.ts deleted file mode 100644 index 9c03c195..00000000 --- a/packages/node/src/indexer/sandbox.service.ts +++ /dev/null @@ -1,64 +0,0 @@ -// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors -// SPDX-License-Identifier: GPL-3.0 - -import { isMainThread } from 'worker_threads'; -import { Inject, Injectable } from '@nestjs/common'; -import { - NodeConfig, - StoreService, - IndexerSandbox, - hostStoreToStore, - ISubqueryProject, - ApiService, - InMemoryCacheService, -} from '@subql/node-core'; -import { Store, BaseDataSource } from '@subql/types-core'; -import SafeEthProvider from '../concordium/safe-api'; - -/* It would be nice to move this to node core but need to find a way to inject other things into the sandbox */ -@Injectable() -export class SandboxService { - private processorCache: Record = {}; - - constructor( - private readonly apiService: ApiService, - @Inject(isMainThread ? StoreService : 'Null') - private readonly storeService: StoreService, - private readonly cacheService: InMemoryCacheService, - private readonly nodeConfig: NodeConfig, - @Inject('ISubqueryProject') private readonly project: ISubqueryProject, - ) {} - - getDsProcessor(ds: BaseDataSource, api: SafeEthProvider): IndexerSandbox { - const store: Store = isMainThread - ? this.storeService.getStore() - : hostStoreToStore((global as any).host); // Provided in worker.ts - - const cache = this.cacheService.getCache(); - const entry = this.getDataSourceEntry(ds); - let processor = this.processorCache[entry]; - if (!processor) { - processor = new IndexerSandbox( - { - cache, - store, - root: this.project.root, - entry, - chainId: this.project.network.chainId, - }, - this.nodeConfig, - ); - this.processorCache[entry] = processor; - } - processor.freeze(api, 'api'); - if (this.nodeConfig.unsafe) { - processor.freeze(this.apiService.api.api, 'unsafeApi'); - } - processor.freeze(this.project.network.chainId, 'chainId'); - return processor; - } - - private getDataSourceEntry(ds: BaseDataSource): string { - return ds.mapping.file; - } -} diff --git a/packages/node/src/indexer/worker/worker-fetch.module.ts b/packages/node/src/indexer/worker/worker-fetch.module.ts index 68dc8898..e63afbc9 100644 --- a/packages/node/src/indexer/worker/worker-fetch.module.ts +++ b/packages/node/src/indexer/worker/worker-fetch.module.ts @@ -12,6 +12,8 @@ import { NodeConfig, InMemoryCacheService, WorkerInMemoryCacheService, + WorkerUnfinalizedBlocksService, + SandboxService, } from '@subql/node-core'; import { ConcordiumApiService } from '../../concordium'; import { ConcordiumApiConnection } from '../../concordium/api.connection'; @@ -20,10 +22,8 @@ import { DsProcessorService } from '../ds-processor.service'; import { DynamicDsService } from '../dynamic-ds.service'; import { IndexerManager } from '../indexer.manager'; import { ProjectService } from '../project.service'; -import { SandboxService } from '../sandbox.service'; import { UnfinalizedBlocksService } from '../unfinalizedBlocks.service'; import { WorkerService } from './worker.service'; -import { WorkerUnfinalizedBlocksService } from './worker.unfinalizedBlocks.service'; @Module({ providers: [ diff --git a/packages/node/src/indexer/worker/worker.unfinalizedBlocks.service.ts b/packages/node/src/indexer/worker/worker.unfinalizedBlocks.service.ts deleted file mode 100644 index a8d2da16..00000000 --- a/packages/node/src/indexer/worker/worker.unfinalizedBlocks.service.ts +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2020-2024 SubQuery Pte Ltd authors & contributors -// SPDX-License-Identifier: GPL-3.0 - -import { isMainThread } from 'worker_threads'; -import { Injectable } from '@nestjs/common'; -import { - Header, - HostUnfinalizedBlocks, - IBlock, - IUnfinalizedBlocksService, -} from '@subql/node-core'; -import { ConcordiumBlock } from '@subql/types-concordium'; - -/** - * @deprecated use node-core version once released https://github.com/subquery/subql/pull/2346 */ -@Injectable() -export class WorkerUnfinalizedBlocksService - implements IUnfinalizedBlocksService -{ - constructor(private host: HostUnfinalizedBlocks) { - if (isMainThread) { - throw new Error('Expected to be worker thread'); - } - } - - async processUnfinalizedBlockHeader(header: Header): Promise { - return this.host.unfinalizedBlocksProcess(header); - } - - async processUnfinalizedBlocks( - block: IBlock, - ): Promise { - return this.host.unfinalizedBlocksProcess(block.getHeader()); - } - - // eslint-disable-next-line @typescript-eslint/promise-function-async - init(reindex: (targetHeight: number) => Promise): Promise { - throw new Error('This method should not be called from a worker'); - } - resetUnfinalizedBlocks(): void { - throw new Error('This method should not be called from a worker'); - } - resetLastFinalizedVerifiedHeight(): void { - throw new Error('This method should not be called from a worker'); - } - // eslint-disable-next-line @typescript-eslint/promise-function-async - getMetadataUnfinalizedBlocks(): Promise { - throw new Error('This method should not be called from a worker'); - } -} diff --git a/packages/node/src/subcommands/forceClean.init.ts b/packages/node/src/subcommands/forceClean.init.ts index d4b2d7d3..2fadff28 100644 --- a/packages/node/src/subcommands/forceClean.init.ts +++ b/packages/node/src/subcommands/forceClean.init.ts @@ -1,21 +1,7 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -import { NestFactory } from '@nestjs/core'; -import { ForceCleanService, getLogger } from '@subql/node-core'; +import { forceClean } from '@subql/node-core'; import { ForceCleanModule } from './forceClean.module'; -const logger = getLogger('CLI'); -export async function forceCleanInit(): Promise { - try { - const app = await NestFactory.create(ForceCleanModule); - await app.init(); - const forceCleanService = app.get(ForceCleanService); - await forceCleanService.forceClean(); - } catch (e) { - logger.error(e, 'Force-clean failed to execute'); - process.exit(1); - } - - process.exit(0); -} +export const forceCleanInit = (): Promise => forceClean(ForceCleanModule); diff --git a/packages/node/src/subcommands/reindex.init.ts b/packages/node/src/subcommands/reindex.init.ts index 99fbb1ad..2f9fc100 100644 --- a/packages/node/src/subcommands/reindex.init.ts +++ b/packages/node/src/subcommands/reindex.init.ts @@ -1,30 +1,8 @@ // Copyright 2020-2024 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 -import { NestFactory } from '@nestjs/core'; -import { getLogger, ReindexService } from '@subql/node-core'; +import { reindexInit as baseReindexInit } from '@subql/node-core'; import { ReindexModule } from './reindex.module'; -const logger = getLogger('CLI-Reindex'); -export async function reindexInit(targetHeight: number): Promise { - try { - const app = await NestFactory.create(ReindexModule); - - await app.init(); - const reindexService = app.get(ReindexService); - - await reindexService.init(); - const actualReindexHeight = - await reindexService.getTargetHeightWithUnfinalizedBlocks(targetHeight); - if (actualReindexHeight !== targetHeight) { - logger.info( - `Found index target height ${targetHeight} beyond indexed unfinalized block ${actualReindexHeight}, will index to ${actualReindexHeight}`, - ); - } - await reindexService.reindex(actualReindexHeight); - } catch (e) { - logger.error(e, 'Reindex failed to execute'); - process.exit(1); - } - process.exit(0); -} +export const reindexInit = (targetHeight: number) => + baseReindexInit(ReindexModule, targetHeight); diff --git a/packages/node/src/subcommands/testing.module.ts b/packages/node/src/subcommands/testing.module.ts index 5b5807a2..a9961ae6 100644 --- a/packages/node/src/subcommands/testing.module.ts +++ b/packages/node/src/subcommands/testing.module.ts @@ -16,6 +16,7 @@ import { StoreCacheService, StoreService, TestRunner, + SandboxService, } from '@subql/node-core'; import { ConcordiumApiService } from '../concordium'; import { ConcordiumApiConnection } from '../concordium/api.connection'; @@ -25,7 +26,6 @@ import { DsProcessorService } from '../indexer/ds-processor.service'; import { DynamicDsService } from '../indexer/dynamic-ds.service'; import { IndexerManager } from '../indexer/indexer.manager'; import { ProjectService } from '../indexer/project.service'; -import { SandboxService } from '../indexer/sandbox.service'; import { UnfinalizedBlocksService } from '../indexer/unfinalizedBlocks.service'; @Module({ diff --git a/packages/types/CHANGELOG.md b/packages/types/CHANGELOG.md index c64087c9..36409e57 100644 --- a/packages/types/CHANGELOG.md +++ b/packages/types/CHANGELOG.md @@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Changed +- Update dependencies and apply changes to match (#40) ## [3.5.1] - 2024-04-11 ### Fixed diff --git a/packages/types/package.json b/packages/types/package.json index ec0edf38..c8f42339 100644 --- a/packages/types/package.json +++ b/packages/types/package.json @@ -14,7 +14,7 @@ "license": "GPL-3.0", "dependencies": { "@concordium/node-sdk": "9.4.0", - "@subql/types-core": "^0.6.0" + "@subql/types-core": "^0.7.0" }, "files": [ "src/global.d.ts", diff --git a/packages/types/src/project.ts b/packages/types/src/project.ts index c7b23c98..8806c7b6 100644 --- a/packages/types/src/project.ts +++ b/packages/types/src/project.ts @@ -9,6 +9,9 @@ import { IProjectNetworkConfig, Processor, ProjectManifestV1_0_0, + SecondLayerHandlerProcessor_0_0_0, + SecondLayerHandlerProcessor_1_0_0, + DsProcessor, } from '@subql/types-core'; import { ConcordiumBlock, @@ -65,13 +68,6 @@ export type ConcordiumRuntimeHandlerInputMap = { [ConcordiumHandlerKind.SpecialEvent]: ConcordiumSpecialEvent; }; -type ConcordiumRuntimeFilterMap = { - [ConcordiumHandlerKind.Block]: ConcordiumBlockFilter; - [ConcordiumHandlerKind.Transaction]: ConcordiumTransactionFilter; - [ConcordiumHandlerKind.TransactionEvent]: ConcordiumTransactionEventFilter; - [ConcordiumHandlerKind.SpecialEvent]: ConcordiumSpecialEvent; -}; - /** * Represents a handler for Concordium blocks. * @type {SubqlCustomHandler} @@ -202,48 +198,18 @@ export interface ConcordiumCustomDatasource< processor: Processor; } -export interface HandlerInputTransformer_0_0_0< - T extends ConcordiumHandlerKind, - E, - DS extends ConcordiumCustomDatasource = ConcordiumCustomDatasource -> { - ( - input: ConcordiumRuntimeHandlerInputMap[T], - ds: DS, - api: ConcordiumGRPCClient, - assets?: Record - ): Promise; // | SubstrateBuiltinDataSource -} - -export interface HandlerInputTransformer_1_0_0< - T extends ConcordiumHandlerKind, - F, +export type SecondLayerHandlerProcessor< + K extends ConcordiumHandlerKind, + F extends Record, E, DS extends ConcordiumCustomDatasource = ConcordiumCustomDatasource -> { - (params: { - input: ConcordiumRuntimeHandlerInputMap[T]; - ds: DS; - filter?: F; - api: ConcordiumGRPCClient; - assets?: Record; - }): Promise; // | SubstrateBuiltinDataSource -} - -export interface DictionaryQueryCondition { - field: string; - value: string | string[]; - matcher?: string; -} - -export interface DictionaryQueryEntry { - entity: string; - conditions: DictionaryQueryCondition[]; -} +> = + | SecondLayerHandlerProcessor_0_0_0 + | SecondLayerHandlerProcessor_1_0_0; export type SecondLayerHandlerProcessorArray< K extends string, - F, + F extends Record, T, DS extends ConcordiumCustomDatasource = ConcordiumCustomDatasource > = @@ -252,60 +218,15 @@ export type SecondLayerHandlerProcessorArray< | SecondLayerHandlerProcessor | SecondLayerHandlerProcessor; -export interface SubqlDatasourceProcessor< +export type SubqlDatasourceProcessor< K extends string, - F, + F extends Record, DS extends ConcordiumCustomDatasource = ConcordiumCustomDatasource, P extends Record> = Record< string, SecondLayerHandlerProcessorArray > -> { - kind: K; - validate(ds: DS, assets: Record): void; - dsFilterProcessor(ds: DS, api: ConcordiumGRPCClient): boolean; - handlerProcessors: P; -} - -interface SecondLayerHandlerProcessorBase< - K extends ConcordiumHandlerKind, - F, - DS extends ConcordiumCustomDatasource = ConcordiumCustomDatasource -> { - baseHandlerKind: K; - baseFilter: ConcordiumRuntimeFilterMap[K] | ConcordiumRuntimeFilterMap[K][]; - filterValidator: (filter?: F) => void; - dictionaryQuery?: (filter: F, ds: DS) => DictionaryQueryEntry | undefined; -} - -export interface SecondLayerHandlerProcessor_0_0_0< - K extends ConcordiumHandlerKind, - F, - E, - DS extends ConcordiumCustomDatasource = ConcordiumCustomDatasource -> extends SecondLayerHandlerProcessorBase { - specVersion: undefined; - transformer: HandlerInputTransformer_0_0_0; - filterProcessor: (filter: F | undefined, input: ConcordiumRuntimeHandlerInputMap[K], ds: DS) => boolean; -} - -export interface SecondLayerHandlerProcessor_1_0_0< - K extends ConcordiumHandlerKind, - F, - E, - DS extends ConcordiumCustomDatasource = ConcordiumCustomDatasource -> extends SecondLayerHandlerProcessorBase { - specVersion: '1.0.0'; - transformer: HandlerInputTransformer_1_0_0; - filterProcessor: (params: {filter: F | undefined; input: ConcordiumRuntimeHandlerInputMap[K]; ds: DS}) => boolean; -} - -export type SecondLayerHandlerProcessor< - K extends ConcordiumHandlerKind, - F, - E, - DS extends ConcordiumCustomDatasource = ConcordiumCustomDatasource -> = SecondLayerHandlerProcessor_0_0_0 | SecondLayerHandlerProcessor_1_0_0; +> = DsProcessor; /** * Represents a Ethereum subquery network configuration, which is based on the CommonSubqueryNetworkConfig template. diff --git a/yarn.lock b/yarn.lock index 8a6410e0..2dab6913 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2771,7 +2771,7 @@ __metadata: resolution: "@subql/common-concordium@workspace:packages/common-concordium" dependencies: "@concordium/node-sdk": 9.4.0 - "@subql/common": ^3.5.0 + "@subql/common": ^3.5.1 "@subql/types-concordium": "workspace:*" "@types/js-yaml": ^4.0.4 "@types/pino": ^6.3.12 @@ -2784,11 +2784,11 @@ __metadata: languageName: unknown linkType: soft -"@subql/common@npm:3.5.0, @subql/common@npm:^3.5.0": - version: 3.5.0 - resolution: "@subql/common@npm:3.5.0" +"@subql/common@npm:3.5.1, @subql/common@npm:^3.5.1": + version: 3.5.1 + resolution: "@subql/common@npm:3.5.1" dependencies: - "@subql/types-core": 0.6.0 + "@subql/types-core": 0.7.0 axios: ^0.28.0 class-transformer: ^0.5.1 class-validator: ^0.14.0 @@ -2798,7 +2798,7 @@ __metadata: reflect-metadata: ^0.1.13 semver: ^7.5.2 update-notifier: 5.1.0 - checksum: 63c374b82de90f2db5c76d8f3808e37100d3616707736806d3f43249ea0ca63f0c115104815cb842becb0d350e16e893a8d3942c0e865d62890e43362a455799 + checksum: df16b2025635d5096b96c299805d3f9c85f7603b245037b2aae48fc0406e2eedab14af7f9e9b07247f166042854427c68a54d90e83588c8e1850471a14e43a0e languageName: node linkType: hard @@ -2815,9 +2815,9 @@ __metadata: "@nestjs/schedule": ^3.0.1 "@nestjs/schematics": ^9.2.0 "@nestjs/testing": ^9.4.0 - "@subql/common": ^3.5.0 + "@subql/common": ^3.5.1 "@subql/common-concordium": "workspace:*" - "@subql/node-core": ^8.0.1 + "@subql/node-core": ^10.0.0 "@subql/types-concordium": "workspace:*" "@types/express": ^4.17.13 "@types/jest": ^27.4.0 @@ -2844,18 +2844,18 @@ __metadata: languageName: unknown linkType: soft -"@subql/node-core@npm:^8.0.1": - version: 8.0.1 - resolution: "@subql/node-core@npm:8.0.1" +"@subql/node-core@npm:^10.0.0": + version: 10.0.0 + resolution: "@subql/node-core@npm:10.0.0" dependencies: "@apollo/client": ^3.8.8 "@nestjs/common": ^9.4.0 "@nestjs/event-emitter": ^2.0.0 "@nestjs/schedule": ^3.0.1 - "@subql/common": 3.5.0 - "@subql/testing": 2.1.1-0 - "@subql/types": 3.4.0 - "@subql/utils": 2.9.0 + "@subql/common": 3.5.1 + "@subql/testing": 2.1.1 + "@subql/types": 3.5.0 + "@subql/utils": 2.9.1 "@willsoto/nestjs-prometheus": ^5.4.0 async-lock: ^1.4.0 async-mutex: ^0.4.0 @@ -2867,20 +2867,20 @@ __metadata: merkle-tools: ^1.4.1 prom-client: ^14.0.1 source-map: ^0.7.4 - tar: ^6.1.11 + tar: ^6.2.1 toposort-class: ^1.0.1 vm2: ^3.9.19 yargs: ^16.2.0 - checksum: ead9feeef201b2ed9db1726e70d375d6a3ca5d44f7576dd14aa2949308d11a6a34abcc1013cd4d84eb8e57fd236609cdb93e359c0b85ff5b225257c99261635d + checksum: c52c304ea23c10a171ef411b877032c88733eb6a2262b4a167de9484f7e1fed567bdc36ad1bcb7f9a610d1774a348bd203dc82dd1f7c52f693ff7756c4097d4f languageName: node linkType: hard -"@subql/testing@npm:2.1.1-0": - version: 2.1.1-0 - resolution: "@subql/testing@npm:2.1.1-0" +"@subql/testing@npm:2.1.1": + version: 2.1.1 + resolution: "@subql/testing@npm:2.1.1" dependencies: - "@subql/types-core": ^0.4.1-0 - checksum: d27d1181f2afb1a394a80fc7d7e36bdd1e64981728d41214412cdeab930c45a7d3bd7cdc58d25d4653c3dcdaf2a456a4e7ae67e4f2ccb39f5494905d06cdecad + "@subql/types-core": ^0.7.0 + checksum: 6b81d0130d364984e1a565228523b414c2fd66d128b2795161872657546ea83a7e23b42d59f968241f18c114afd7c511886418db8e89679f8a35861e8c24e5a9 languageName: node linkType: hard @@ -2889,42 +2889,33 @@ __metadata: resolution: "@subql/types-concordium@workspace:packages/types" dependencies: "@concordium/node-sdk": 9.4.0 - "@subql/types-core": ^0.6.0 + "@subql/types-core": ^0.7.0 languageName: unknown linkType: soft -"@subql/types-core@npm:0.6.0, @subql/types-core@npm:^0.6.0": - version: 0.6.0 - resolution: "@subql/types-core@npm:0.6.0" - dependencies: - package-json-type: ^1.0.3 - checksum: 0688f1973f830d1d2ab34864b00fc7efb105228b0328ac485c69950957da3a67a92d9477bc94945e57e3cccbf1b732bdea097c1a452a907e23704e9cbeff31fc - languageName: node - linkType: hard - -"@subql/types-core@npm:^0.4.1-0": - version: 0.4.1-0 - resolution: "@subql/types-core@npm:0.4.1-0" +"@subql/types-core@npm:0.7.0, @subql/types-core@npm:^0.7.0": + version: 0.7.0 + resolution: "@subql/types-core@npm:0.7.0" dependencies: package-json-type: ^1.0.3 - checksum: f7a51202b6e350e65f65a55136a2b48c8d6ae403850e96215995648656b3ef0779ef33ae526c7ffaa51b221b29255137e84ffd0abb5c1155095b1abfb15ef340 + checksum: ba8d140333d315e80c28b44f29f56511302d0179001048e55ef2bbdded51deb7ddd37d77eee3d6fff1c18f2b944f2cd9f4e39da39e9f4bd9216d4e1b4bbf6378 languageName: node linkType: hard -"@subql/types@npm:3.4.0": - version: 3.4.0 - resolution: "@subql/types@npm:3.4.0" +"@subql/types@npm:3.5.0": + version: 3.5.0 + resolution: "@subql/types@npm:3.5.0" dependencies: - "@subql/types-core": 0.6.0 + "@subql/types-core": 0.7.0 peerDependencies: "@polkadot/api": ^10 - checksum: a13e02ebdf35abc05104615f3884faf9f2d9037eece58ac05ba6cc4e07812ef40479645578327147ac82872e9466b5e2995a22076a7fdbc27fd6dd28f1cb669d + checksum: acecfafa56d3488f9ea25a8dd8d3038b1012f090d9f8c8046c74cbf17b983cf19a3d5711a88cb228bc48ce5d5348c9f0dd881a3eb504f319dcbb9b0c6303c526 languageName: node linkType: hard -"@subql/utils@npm:2.9.0": - version: 2.9.0 - resolution: "@subql/utils@npm:2.9.0" +"@subql/utils@npm:2.9.1": + version: 2.9.1 + resolution: "@subql/utils@npm:2.9.1" dependencies: "@polkadot/util": ^12.5.1 "@polkadot/util-crypto": ^12.5.1 @@ -2939,8 +2930,8 @@ __metadata: pino: ^6.13.3 rotating-file-stream: ^3.0.2 semver: ^7.5.2 - tar: ^6.1.11 - checksum: 2536add6833f9a1487ea17eccf0288e48f7a89bb67dfedbc46034902e350d4474be7f38cef06e938ee49ba25274c2271e2b5b6b46e04c965cdba5105f9af4599 + tar: ^6.2.1 + checksum: 555c215f44ac6f6828ade30c3643e260bf99ef3c14381d14a9e9ba3c9fa5cfd0a4e938b3a2fbd907379ad2b832bb79a5071f96966b32d751824affb9cea266c8 languageName: node linkType: hard @@ -11298,6 +11289,20 @@ __metadata: languageName: node linkType: hard +"tar@npm:^6.2.1": + version: 6.2.1 + resolution: "tar@npm:6.2.1" + dependencies: + chownr: ^2.0.0 + fs-minipass: ^2.0.0 + minipass: ^5.0.0 + minizlib: ^2.1.1 + mkdirp: ^1.0.3 + yallist: ^4.0.0 + checksum: f1322768c9741a25356c11373bce918483f40fa9a25c69c59410c8a1247632487edef5fe76c5f12ac51a6356d2f1829e96d2bc34098668a2fc34d76050ac2b6c + languageName: node + linkType: hard + "tdigest@npm:^0.1.1": version: 0.1.2 resolution: "tdigest@npm:0.1.2"