diff --git a/packages/common-concordium/CHANGELOG.md b/packages/common-concordium/CHANGELOG.md index caf56b84..a0f94638 100644 --- a/packages/common-concordium/CHANGELOG.md +++ b/packages/common-concordium/CHANGELOG.md @@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Removed +- `options` field on datasources as they were not used anywhere (#16) ## [3.2.0] - 2023-11-01 ### Added diff --git a/packages/common-concordium/src/project/models.ts b/packages/common-concordium/src/project/models.ts index 0de4f400..70aae3b0 100644 --- a/packages/common-concordium/src/project/models.ts +++ b/packages/common-concordium/src/project/models.ts @@ -13,7 +13,6 @@ import { SubqlRuntimeDatasource, SubqlCustomDatasource, CustomDataSourceAsset, - ConcordiumBlockFilter, SubqlBlockHandler, ConcordiumTransactionFilter, ConcordiumTransactionEventFilter, @@ -25,7 +24,7 @@ import { import {FileReference, Processor} from '@subql/types-core'; import {plainToClass, Transform, Type} from 'class-transformer'; import {IsArray, IsEnum, IsInt, IsOptional, IsString, IsObject, ValidateNested} from 'class-validator'; -import {SubqlConcordiumDatasourceKind, SubqlConcordiumHandlerKind, SubqlConcordiumProcessorOptions} from './types'; +import {SubqlConcordiumDatasourceKind, SubqlConcordiumHandlerKind} from './types'; export class TransactionFilter implements ConcordiumTransactionFilter { @IsOptional() @@ -142,12 +141,6 @@ export class CustomMapping implements SubqlMapping { file: string; } -export class ConcordiumProcessorOptions implements SubqlConcordiumProcessorOptions { - @IsOptional() - @IsString() - address?: string; -} - export class RuntimeDataSourceBase> extends BaseDataSource implements SubqlRuntimeDatasource @@ -161,10 +154,6 @@ export class RuntimeDataSourceBase> mapping: M; @IsOptional() assets?: Map; - @IsOptional() - @ValidateNested() - @Type(() => ConcordiumProcessorOptions) - options?: ConcordiumProcessorOptions; } export class FileReferenceImpl implements FileReference { @@ -190,7 +179,4 @@ export class CustomDataSourceBase ProcessorImpl) @IsObject() processor: Processor; - @IsOptional() - @ValidateNested() - options?: ConcordiumProcessorOptions; } diff --git a/packages/common-concordium/src/project/types.ts b/packages/common-concordium/src/project/types.ts index f3f09004..d3e4ef42 100644 --- a/packages/common-concordium/src/project/types.ts +++ b/packages/common-concordium/src/project/types.ts @@ -6,7 +6,6 @@ import {IProjectManifest, ProjectNetworkConfig} from '@subql/types-core'; // All of these used to be redefined in this file, re-exporting for simplicity export { - SubqlConcordiumProcessorOptions, SubqlRuntimeHandler, SubqlCustomHandler, SubqlHandler, diff --git a/packages/node/CHANGELOG.md b/packages/node/CHANGELOG.md index 9bb1a5f0..86862a08 100644 --- a/packages/node/CHANGELOG.md +++ b/packages/node/CHANGELOG.md @@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added +- Improved filtering of account and contract addresses as well as bigints (#19) ## [3.4.0] - 2023-11-13 ### Changed diff --git a/packages/node/src/concordium/block.concordium.spec.ts b/packages/node/src/concordium/block.concordium.spec.ts index 6570e780..39454f8b 100644 --- a/packages/node/src/concordium/block.concordium.spec.ts +++ b/packages/node/src/concordium/block.concordium.spec.ts @@ -4,6 +4,7 @@ import { TransactionEventTag, TransactionSummaryType, + TransactionKindString, } from '@concordium/node-sdk'; import { ConcordiumBlock, @@ -13,28 +14,11 @@ import { } from '@subql/types-concordium'; import { filterBlocksProcessor, - filterSpecialEventProcessor, filterTransactionsProcessor, filterTxEventProcessor, } from './block.concordium'; describe('ConcordiumBlockWrapped', () => { - beforeEach(() => { - const block = { - blockHeight: '10', - blockHash: 'hash', - } as unknown as ConcordiumBlock; - const transactions = [ - { type: 'transfer', amount: '100' } as unknown as ConcordiumTransaction, - ]; - const txEvents = [ - { tag: 'event1', amount: '100' } as unknown as ConcordiumTransactionEvent, - ]; - const specialEvents = [ - { tag: 'event2', amount: '100' } as unknown as ConcordiumSpecialEvent, - ]; - }); - it('should filter blocks', () => { const block = { blockHeight: '10', @@ -152,7 +136,7 @@ describe('ConcordiumBlockWrapped', () => { } as unknown as ConcordiumSpecialEvent; const filter = { type: 'event2' }; - expect(filterSpecialEventProcessor(specialEvent, filter)).toBe(true); + expect(filterTxEventProcessor(specialEvent, filter)).toBe(true); }); it('should filter special events - without values - for false', () => { @@ -162,7 +146,7 @@ describe('ConcordiumBlockWrapped', () => { } as unknown as ConcordiumSpecialEvent; const filter = { type: 'event1' }; - expect(filterSpecialEventProcessor(specialEvent, filter)).toBe(false); + expect(filterTxEventProcessor(specialEvent, filter)).toBe(false); }); it('should filter special events - with values - for true', () => { @@ -177,7 +161,7 @@ describe('ConcordiumBlockWrapped', () => { }, }; - expect(filterSpecialEventProcessor(specialEvent, filter)).toBe(true); + expect(filterTxEventProcessor(specialEvent, filter)).toBe(true); }); it('should filter special events - with values - for false', () => { @@ -192,6 +176,51 @@ describe('ConcordiumBlockWrapped', () => { }, }; - expect(filterSpecialEventProcessor(specialEvent, filter)).toBe(false); + expect(filterTxEventProcessor(specialEvent, filter)).toBe(false); + }); + + it('should filter nested values', () => { + const tx = { + index: BigInt(0), + energyCost: BigInt(3739), + hash: '574289208dfd6dff2065e2549164bcfdd97f91e65cb42941a40e574ceccbf7b9', + type: TransactionSummaryType.AccountTransaction, + cost: BigInt(14685685), + sender: '4AuT5RRmBwcdkLMA6iVjxTDb1FQmxwAh3wHBS22mggWL8xH6s3', + transactionType: TransactionKindString.Update, + } as ConcordiumTransaction; + + const match = filterTransactionsProcessor(tx, { + type: TransactionSummaryType.AccountTransaction, + values: { + sender: '4AuT5RRmBwcdkLMA6iVjxTDb1FQmxwAh3wHBS22mggWL8xH6s3', + }, + }); + expect(match).toBeTruthy(); + + const event = { + tag: 'Updated', + contractVersion: 1, + address: { index: BigInt(6536), subindex: BigInt(0) }, + instigator: { + type: 'AddressAccount', + address: '4AuT5RRmBwcdkLMA6iVjxTDb1FQmxwAh3wHBS22mggWL8xH6s3', + }, + amount: BigInt(0), + message: + '35000000b0373129de61634f2c72ee0c008a9d37f6d10367875108b622bd9480cff50d9f0c000000544539445156524a5430343d005e94526500000000', + receiveName: 'Provenance-tag.update_tag_log', + events: [], + } as ConcordiumTransactionEvent; + + const matchEvent = filterTxEventProcessor(event, { + type: TransactionEventTag.Updated, + values: { + instigator: '4AuT5RRmBwcdkLMA6iVjxTDb1FQmxwAh3wHBS22mggWL8xH6s3', + address: '6536', + amount: '0', + }, + }); + expect(matchEvent).toBeTruthy(); }); }); diff --git a/packages/node/src/concordium/block.concordium.ts b/packages/node/src/concordium/block.concordium.ts index 06372fc2..33dfe538 100644 --- a/packages/node/src/concordium/block.concordium.ts +++ b/packages/node/src/concordium/block.concordium.ts @@ -1,6 +1,8 @@ // Copyright 2020-2023 SubQuery Pte Ltd authors & contributors // SPDX-License-Identifier: GPL-3.0 +import { ContractAddress, AddressAccount, Address } from '@concordium/node-sdk'; + import { ConcordiumBlock, ConcordiumTransactionFilter, @@ -15,7 +17,6 @@ import { export function filterBlocksProcessor( block: ConcordiumBlock, filter: ConcordiumBlockFilter, - address?: string, ): boolean { if (filter?.modulo && Number(block.blockHeight) % filter.modulo !== 0) { return false; @@ -26,7 +27,6 @@ export function filterBlocksProcessor( export function filterTransactionsProcessor( transaction: ConcordiumTransaction, filter: ConcordiumTransactionFilter, - address?: string, ): boolean { if (!filter) return true; @@ -42,9 +42,8 @@ export function filterTransactionsProcessor( } export function filterTxEventProcessor( - txEvent: ConcordiumTransactionEvent, - filter: ConcordiumTransactionEventFilter, - address?: string, + txEvent: ConcordiumTransactionEvent | ConcordiumSpecialEvent, + filter: ConcordiumTransactionEventFilter | ConcordiumSpecialEventFilter, ): boolean { if (!filter) return true; @@ -52,26 +51,56 @@ export function filterTxEventProcessor( if (filter.values) { for (const key in filter.values) { - if (filter.values[key] !== txEvent[key]) return false; + const filterValue = filter.values[key]; + const eventValue = txEvent[key]; + if ( + filterValue !== eventValue && + !equalsAddressAccount(eventValue, filterValue) && + !equalsContractAddress(eventValue, filterValue) && + !equalsAddress(eventValue, filterValue) && + !equalsBigInt(eventValue, filterValue) + ) { + return false; + } } } return true; } -export function filterSpecialEventProcessor( - specialEvent: ConcordiumSpecialEvent, - filter: ConcordiumSpecialEventFilter, -): boolean { - if (!filter) return true; +function isAddressAccount(t: any): t is AddressAccount { + return t.type === 'AddressAccount'; +} - if (filter.type && specialEvent.tag !== filter.type) return false; +function equalsAddressAccount(t: any, address: string): boolean { + return isAddressAccount(t) && t.address === address; +} - if (filter.values) { - for (const key in filter.values) { - if (filter.values[key] !== specialEvent[key]) return false; - } - } +function isContractAddress(t: any): t is ContractAddress { + return typeof t.index === 'bigint' && typeof t.subindex === 'bigint'; +} - return true; +function equalsContractAddress(t: any, index: string): boolean { + return isContractAddress(t) && t.index === BigInt(index); +} + +function isAddress(t: any): t is Address { + return t.type === 'AddressContract' || isAddressAccount(t); +} + +function equalsAddress(t: any, addressOrIndex: string): boolean { + if (!isAddress(t)) return false; + + return ( + equalsAddressAccount(t, addressOrIndex) || + equalsContractAddress(t.address, addressOrIndex) + ); +} + +function equalsBigInt(a: bigint, b: string): boolean { + try { + return a === BigInt(b); + } catch (e) { + return false; + } } diff --git a/packages/node/src/indexer/dynamic-ds.service.ts b/packages/node/src/indexer/dynamic-ds.service.ts index d8dbc484..e1e17219 100644 --- a/packages/node/src/indexer/dynamic-ds.service.ts +++ b/packages/node/src/indexer/dynamic-ds.service.ts @@ -54,11 +54,6 @@ export class DynamicDsService extends BaseDynamicDsService }; await this.dsProcessorService.validateCustomDs([dsObj]); } else if (isRuntimeDs(dsObj)) { - dsObj.options = { - ...dsObj.options, - ...params.args, - }; - const parsedDs = plainToClass(ConcordiumRuntimeDataSourceImpl, dsObj); const errors = validateSync(parsedDs, { diff --git a/packages/node/src/indexer/fetch.service.ts b/packages/node/src/indexer/fetch.service.ts index 119cf649..bf53dca8 100644 --- a/packages/node/src/indexer/fetch.service.ts +++ b/packages/node/src/indexer/fetch.service.ts @@ -8,7 +8,6 @@ import { SchedulerRegistry } from '@nestjs/schedule'; import { isCustomDs, SubqlConcordiumHandlerKind, - SubqlConcordiumProcessorOptions, ConcordiumTransactionFilter, } from '@subql/common-concordium'; import { @@ -27,7 +26,7 @@ import { DictionaryQueryCondition, DictionaryQueryEntry, } from '@subql/types-core'; -import { groupBy, partition, setWith, sortBy, uniqBy } from 'lodash'; +import { setWith, sortBy, uniqBy } from 'lodash'; import { ConcordiumApi } from '../concordium'; import { calcInterval } from '../concordium/utils.concordium'; import { SubqueryProject } from '../configure/SubqueryProject'; @@ -140,12 +139,8 @@ export function speicalEventFilterToQueryEntry( }; } -type GroupedConcordiumProjectDs = SubqlDatasource & { - groupedOptions?: SubqlConcordiumProcessorOptions[]; -}; - export function buildDictionaryQueryEntries( - dataSources: GroupedConcordiumProjectDs[], + dataSources: SubqlDatasource[], ): DictionaryQueryEntry[] { const queryEntries: DictionaryQueryEntry[] = []; @@ -246,31 +241,7 @@ export class FetchService extends BaseFetchService< protected buildDictionaryQueryEntries( dataSources: (SubqlDatasource & { name?: string })[], ): DictionaryQueryEntry[] { - const [normalDataSources, templateDataSources] = partition( - dataSources, - (ds) => !ds.name, - ); - - // Group templ - const groupedDataSources = Object.values( - groupBy(templateDataSources, (ds) => ds.name), - ).map((grouped) => { - if (grouped.length === 1) { - return grouped[0]; - } - - const options = grouped.map((ds) => ds.options); - const ref = grouped[0]; - - return { - ...ref, - groupedOptions: options, - }; - }); - - const filteredDs = [...normalDataSources, ...groupedDataSources]; - - return buildDictionaryQueryEntries(filteredDs); + return buildDictionaryQueryEntries(dataSources); } protected async getFinalizedHeight(): Promise { diff --git a/packages/node/src/indexer/indexer.manager.ts b/packages/node/src/indexer/indexer.manager.ts index aa515d2e..a0ecb62c 100644 --- a/packages/node/src/indexer/indexer.manager.ts +++ b/packages/node/src/indexer/indexer.manager.ts @@ -37,7 +37,6 @@ import { import { ConcordiumApi, ConcordiumApiService } from '../concordium'; import { filterBlocksProcessor, - filterSpecialEventProcessor, filterTransactionsProcessor, filterTxEventProcessor, } from '../concordium/block.concordium'; @@ -224,20 +223,20 @@ const FilterTypeMap = { data: ConcordiumBlock, filter: ConcordiumBlockFilter, ds: SubqlConcordiumDataSource, - ) => filterBlocksProcessor(data, filter, ds.options?.address), + ) => filterBlocksProcessor(data, filter), [ConcordiumHandlerKind.SpecialEvent]: ( data: ConcordiumSpecialEvent, filter: ConcordiumSpecialEventFilter, ds: SubqlConcordiumDataSource, - ) => filterSpecialEventProcessor(data, filter), + ) => filterTxEventProcessor(data, filter), [ConcordiumHandlerKind.TransactionEvent]: ( data: ConcordiumTransactionEvent, filter: ConcordiumTransactionEventFilter, ds: SubqlConcordiumDataSource, - ) => filterTxEventProcessor(data, filter, ds.options?.address), + ) => filterTxEventProcessor(data, filter), [ConcordiumHandlerKind.Transaction]: ( data: ConcordiumTransaction, filter: ConcordiumTransactionFilter, ds: SubqlConcordiumDataSource, - ) => filterTransactionsProcessor(data, filter, ds.options?.address), + ) => filterTransactionsProcessor(data, filter), }; diff --git a/packages/types/CHANGELOG.md b/packages/types/CHANGELOG.md index eef53f7b..765fa791 100644 --- a/packages/types/CHANGELOG.md +++ b/packages/types/CHANGELOG.md @@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Removed +- `options` field on datasources as they were not used anywhere (#16) ## [3.3.0] - 2023-11-01 ### Changed diff --git a/packages/types/src/project.ts b/packages/types/src/project.ts index 3633d6b4..15cd2671 100644 --- a/packages/types/src/project.ts +++ b/packages/types/src/project.ts @@ -170,16 +170,6 @@ interface ISubqlDatasource { mapping: M; } -export interface SubqlConcordiumProcessorOptions { - /** - * The specific contract that this datasource should filter. - * Alternatively this can be left blank and a transaction to filter can be used instead - * @example - * address: '', - * */ - address?: string; -} - /** * Represents a runtime datasource for Concordium. * @interface @@ -192,14 +182,6 @@ export interface SubqlRuntimeDatasource; } @@ -218,7 +200,6 @@ export interface SubqlCustomDatasource< > extends ISubqlDatasource { kind: K; assets: Map; - options?: SubqlConcordiumProcessorOptions; processor: Processor; }