subquery · jiqiang90 · Oct 31, 2023 · Oct 31, 2023 · Oct 31, 2023 · Oct 31, 2023
diff --git a/packages/common-ethereum/CHANGELOG.md b/packages/common-ethereum/CHANGELOG.md
@@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Added
+- Update `@subql/common` and relevant changes to support endBlock feature (#195)
 
 ## [3.0.6] - 2023-10-25
 ### Changed

diff --git a/packages/common-ethereum/package.json b/packages/common-ethereum/package.json
@@ -14,7 +14,7 @@
   "main": "dist/index.js",
   "license": "GPL-3.0",
   "dependencies": {
-    "@subql/common": "^3.2.0",
+    "@subql/common": "^3.3.0",
     "@subql/types-ethereum": "workspace:*",
     "@typechain/ethers-v5": "^11.1.1",
     "js-yaml": "^4.1.0",

diff --git a/packages/common-ethereum/src/project/models.ts b/packages/common-ethereum/src/project/models.ts
@@ -1,6 +1,7 @@
 // Copyright 2020-2023 SubQuery Pte Ltd authors & contributors
 // SPDX-License-Identifier: GPL-3.0
 
+import {BaseDataSource} from '@subql/common';
 import {FileReference} from '@subql/types-core';
 import {
   EthereumHandlerKind,
@@ -62,8 +63,8 @@
  function?: string;
 }

 export function forbidNonWhitelisted(keys: any, validationOptions?: ValidationOptions) {
  return function (object: object, propertyName: string) {
    registerDecorator({
      name: 'forbidNonWhitelisted',
      target: object.constructor,
@@ -71,7 +72,7 @@
      constraints: [],
      options: validationOptions,
      validator: {
        validate(value: any, args: ValidationArguments) {
          const isValid = !Object.keys(value).some((key) => !(key in keys));
          if (!isValid) {
            throw new Error(
@@ -150,7 +151,7 @@
        case SubqlEthereumHandlerKind.EthBlock:
          return plainToClass(BlockHandler, handler);
        default:
          throw new Error(`handler ${(handler as any).kind} not supported`);
      }
    });
  })
@@ -179,7 +180,10 @@
   address?: string;
 }
 
-export class RuntimeDataSourceBase<M extends SubqlMapping<SubqlRuntimeHandler>> implements SubqlRuntimeDatasource<M> {
+export class RuntimeDataSourceBase<M extends SubqlMapping<SubqlRuntimeHandler>>
+  extends BaseDataSource
+  implements SubqlRuntimeDatasource<M>
+{
   @IsEnum(SubqlEthereumDatasourceKind, {
     groups: [SubqlEthereumDatasourceKind.FlareRuntime, SubqlEthereumDatasourceKind.EthRuntime],
   })
@@ -188,9 +192,6 @@
   @ValidateNested()
   mapping: M;
   @IsOptional()
-  @IsInt()
-  startBlock?: number;
-  @IsOptional()
   assets?: Map<string, FileReference>;
   @IsOptional()
   @ValidateNested()
@@ -204,16 +205,14 @@
 }
 
 export class CustomDataSourceBase<K extends string, M extends SubqlMapping = SubqlMapping<SubqlCustomHandler>>
+  extends BaseDataSource
   implements SubqlCustomDatasource<K, M>
 {
   @IsString()
   kind: K;
   @Type(() => CustomMapping)
   @ValidateNested()
   mapping: M;
-  @IsOptional()
-  @IsInt()
-  startBlock?: number;
   @Type(() => FileReferenceImpl)
   @ValidateNested({each: true})
   assets: Map<string, CustomDataSourceAsset>;

diff --git a/packages/node/CHANGELOG.md b/packages/node/CHANGELOG.md
@@ -8,6 +8,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Fixed
 - Unfinalized blocks not working with Cronos (#193)
 
+### Changed
+- Update `@subql/node-core` with fixes and support for endBlock feature (#195)
+
 ## [3.1.1] - 2023-10-25
 ### Fixed
 - Fix crash when creating new dynamic datasources

diff --git a/packages/node/package.json b/packages/node/package.json
@@ -25,9 +25,9 @@
     "@nestjs/event-emitter": "^2.0.0",
     "@nestjs/platform-express": "^9.4.0",
     "@nestjs/schedule": "^3.0.1",
-    "@subql/common": "^3.2.0",
+    "@subql/common": "^3.3.0",
     "@subql/common-ethereum": "workspace:*",
-    "@subql/node-core": "^6.1.1",
+    "@subql/node-core": "^6.2.0",
     "@subql/testing": "^2.0.2",
     "@subql/types-ethereum": "workspace:*",
     "cacheable-lookup": "6",

diff --git a/packages/node/src/ethereum/api.ethereum.test.ts b/packages/node/src/ethereum/api.ethereum.test.ts
@@ -17,7 +17,8 @@ import {
 } from './block.ethereum';
 
 // Add api key to work
-const HTTP_ENDPOINT = 'https://eth.api.onfinality.io/public';
+// const HTTP_ENDPOINT = 'https://eth.api.onfinality.io/public';
+const HTTP_ENDPOINT = 'https://eth.llamarpc.com';
 const BLOCK_CONFIRMATIONS = 20;
 
 const ds: SubqlRuntimeDatasource = {
@@ -256,7 +257,7 @@ describe('Api.ethereum', () => {
 
     // Polygon
     ethApi = new EthereumApi(
-      'https://polygon.api.onfinality.io/public',
+      'https://polygon.llamarpc.com',
       BLOCK_CONFIRMATIONS,
       eventEmitter,
     );

diff --git a/packages/node/src/ethereum/api.service.ethereum.test.ts b/packages/node/src/ethereum/api.service.ethereum.test.ts
@@ -16,7 +16,8 @@ import { EthereumApiService } from './api.service.ethereum';
 
 // Add api key to work
 const WS_ENDPOINT = 'wss://eth.api.onfinality.io/ws?apikey=';
-const HTTP_ENDPOINT = 'https://eth.api.onfinality.io/public';
+// const HTTP_ENDPOINT = 'https://eth.api.onfinality.io/public';
+const HTTP_ENDPOINT = 'https://eth.llamarpc.com';
 
 function testSubqueryProject(endpoint: string): SubqueryProject {
   return {

diff --git a/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts b/packages/node/src/indexer/blockDispatcher/worker-block-dispatcher.service.ts
@@ -6,29 +6,18 @@ import { Inject, Injectable, OnApplicationShutdown } from '@nestjs/common';
 import { EventEmitter2 } from '@nestjs/event-emitter';
 import {
   NodeConfig,
-  Worker,
   SmartBatchService,
   StoreService,
   PoiService,
   StoreCacheService,
   IProjectService,
-  IDynamicDsService,
-  HostStore,
-  HostDynamicDS,
   WorkerBlockDispatcher,
-  IUnfinalizedBlocksService,
   ConnectionPoolStateManager,
-  connectionPoolStateHostFunctions,
-  baseWorkerFunctions,
-  storeHostFunctions,
-  cacheHostFunctions,
-  dynamicDsHostFunctions,
   IProjectUpgradeService,
-  HostUnfinalizedBlocks,
   PoiSyncService,
   InMemoryCacheService,
+  createIndexerWorker,
 } from '@subql/node-core';
-import { Cache, Store } from '@subql/types-core';
 import {
   EthereumProjectDs,
   SubqueryProject,
@@ -37,45 +26,12 @@ import { EthereumApiConnection } from '../../ethereum/api.connection';
 import { DynamicDsService } from '../dynamic-ds.service';
 import { BlockContent } from '../types';
 import { UnfinalizedBlocksService } from '../unfinalizedBlocks.service';
-import { IIndexerWorker, IInitIndexerWorker } from '../worker/worker';
+import { IIndexerWorker } from '../worker/worker';
 
 type IndexerWorker = IIndexerWorker & {
   terminate: () => Promise<number>;
 };
 
-async function createIndexerWorker(
-  store: Store,
-  cache: Cache,
-  dynamicDsService: IDynamicDsService<EthereumProjectDs>,
-  unfinalizedBlocksService: IUnfinalizedBlocksService<BlockContent>,
-  connectionPoolState: ConnectionPoolStateManager<EthereumApiConnection>,
-  root: string,
-  startHeight: number,
-): Promise<IndexerWorker> {
-  const indexerWorker = Worker.create<
-    IInitIndexerWorker,
-    HostDynamicDS<EthereumProjectDs> & HostStore & HostUnfinalizedBlocks
-  >(
-    path.resolve(__dirname, '../../../dist/indexer/worker/worker.js'),
-    [...baseWorkerFunctions, 'initWorker'],
-    {
-      ...cacheHostFunctions(cache),
-      ...storeHostFunctions(store),
-      ...dynamicDsHostFunctions(dynamicDsService),
-      unfinalizedBlocksProcess:
-        unfinalizedBlocksService.processUnfinalizedBlockHeader.bind(
-          unfinalizedBlocksService,
-        ),
-      ...connectionPoolStateHostFunctions(connectionPoolState),
-    },
-    root,
-  );
-
-  await indexerWorker.initWorker(startHeight);
-
-  return indexerWorker;
-}
-
 @Injectable()
 export class WorkerBlockDispatcherService
   extends WorkerBlockDispatcher<EthereumProjectDs, IndexerWorker>
@@ -112,7 +68,14 @@ export class WorkerBlockDispatcherService
       project,
       dynamicDsService,
       () =>
-        createIndexerWorker(
+        createIndexerWorker<
+          IIndexerWorker,
+          EthereumApiConnection,
+          BlockContent,
+          EthereumProjectDs
+        >(
+          path.resolve(__dirname, '../../../dist/indexer/worker/worker.js'),
+          [],
           storeService.getStore(),
           cacheService.getCache(),
           dynamicDsService,

diff --git a/packages/node/src/indexer/fetch.service.spec.ts b/packages/node/src/indexer/fetch.service.spec.ts
@@ -13,7 +13,8 @@ import {
 } from '../configure/SubqueryProject';
 import { buildDictionaryQueryEntries, FetchService } from './fetch.service';
 
-const HTTP_ENDPOINT = 'https://eth.api.onfinality.io/public';
+// const HTTP_ENDPOINT = 'https://eth.api.onfinality.io/public';
+const HTTP_ENDPOINT = 'https://eth.llamarpc.com';
 const mockTempDs: EthereumProjectDsTemplate[] = [
   {
     name: 'ERC721',

diff --git a/packages/node/src/indexer/project.service.test.ts b/packages/node/src/indexer/project.service.test.ts
@@ -5,12 +5,10 @@ import { EventEmitter2 } from '@nestjs/event-emitter';
 import { EthereumApi, EthereumApiService } from '../ethereum';
 import { ProjectService } from './project.service';
 
+const HTTP_ENDPOINT = 'https://eth.llamarpc.com';
+
 const mockApiService = (): EthereumApiService => {
-  const ethApi = new EthereumApi(
-    'https://eth.api.onfinality.io/public',
-    20,
-    new EventEmitter2(),
-  );
+  const ethApi = new EthereumApi(HTTP_ENDPOINT, 20, new EventEmitter2());
 
   // await ethApi.init();
 

diff --git a/packages/node/src/indexer/worker-fetch.module.ts b/packages/node/src/indexer/worker-fetch.module.ts
@@ -11,6 +11,7 @@ import {
   ConnectionPoolStateManager,
   NodeConfig,
   InMemoryCacheService,
+  WorkerInMemoryCacheService,
 } from '@subql/node-core';
 import { SubqueryProject } from '../configure/SubqueryProject';
 import { EthereumApiService } from '../ethereum';
@@ -21,7 +22,6 @@ import { IndexerManager } from './indexer.manager';
 import { ProjectService } from './project.service';
 import { SandboxService } from './sandbox.service';
 import { UnfinalizedBlocksService } from './unfinalizedBlocks.service';
-import { WorkerInMemoryCacheService } from './worker/worker.inMemoryCache.service';
 import { WorkerService } from './worker/worker.service';
 import { WorkerUnfinalizedBlocksService } from './worker/worker.unfinalizedBlocks.service';
 

diff --git a/packages/node/src/indexer/worker/worker.inMemoryCache.service.ts b/packages/node/src/indexer/worker/worker.inMemoryCache.service.ts
diff --git a/packages/node/src/indexer/worker/worker.ts b/packages/node/src/indexer/worker/worker.ts
@@ -55,10 +55,6 @@ async function initWorker(startHeight: number): Promise<void> {
 
 export type IIndexerWorker = IBaseIndexerWorker;
 
-export type IInitIndexerWorker = IIndexerWorker & {
-  initWorker: typeof initWorker;
-};
-
 (global as any).host = createWorkerHost([], {
   initWorker,
 });
diff --git a/packages/node/src/subcommands/testing.module.ts b/packages/node/src/subcommands/testing.module.ts
@@ -9,8 +9,10 @@ import {
   ConnectionPoolService,
   ConnectionPoolStateManager,
   DbModule,
+  InMemoryCacheService,
   NodeConfig,
   PoiService,
+  PoiSyncService,
   StoreCacheService,
   StoreService,
   TestRunner,
@@ -28,10 +30,12 @@ import { UnfinalizedBlocksService } from '../indexer/unfinalizedBlocks.service';
 
 @Module({
   providers: [
+    InMemoryCacheService,
     StoreService,
     StoreCacheService,
     EventEmitter2,
     PoiService,
+    PoiSyncService,
     SandboxService,
     DsProcessorService,
     DynamicDsService,

diff --git a/packages/node/src/yargs.ts b/packages/node/src/yargs.ts
@@ -90,7 +90,7 @@ export const yargsOptions = yargs(hideBin(process.argv))
     },
     debug: {
       demandOption: false,
-      describe: `Enable debug logging for specific scopes, this will override log-level. "*" will enable debug everywhere, or comma separated strings for specific scopes. e.g. "SQL,dictionary"`,
+      describe: `Enable debug logging for specific scopes, this will override log-level. "*" will enable debug everywhere, or comma separated strings for specific scopes. e.g. "SQL,dictionary". To disable specific scopes you can prefix them with '-'. e.g. "*,-SQL"`,
       type: 'string',
     },
     'dictionary-resolver': {

diff --git a/packages/types/CHANGELOG.md b/packages/types/CHANGELOG.md
@@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Changed
+- Import `@subql/types-core` global into global so its no longer needed to update tsconfig in projects (#195)
 
 ## [3.1.0] - 2023-10-20
 ### Changed

diff --git a/packages/types/package.json b/packages/types/package.json
@@ -18,7 +18,7 @@
   ],
   "dependencies": {
     "@ethersproject/abstract-provider": "^5.6.1",
-    "@subql/types-core": "^0.2.0"
+    "@subql/types-core": "^0.3.0"
   },
   "stableVersion": "3.1.0"
 }
diff --git a/packages/types/src/global.ts b/packages/types/src/global.ts
@@ -2,6 +2,7 @@
 // SPDX-License-Identifier: GPL-3.0
 
 import {Provider} from '@ethersproject/abstract-provider';
+import '@subql/types-core/dist/global';
 
 declare global {
   const api: Provider;