Merge pull request #16 from subquery/improve-filters

Improve filters for nested values

packages/common-concordium/CHANGELOG.md

@@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Removed
+- `options` field on datasources as they were not used anywhere (#16)
 
 ## [3.2.0] - 2023-11-01
 ### Added

packages/common-concordium/src/project/models.ts

@@ -13,7 +13,6 @@ import {
   SubqlRuntimeDatasource,
   SubqlCustomDatasource,
   CustomDataSourceAsset,
-  ConcordiumBlockFilter,
   SubqlBlockHandler,
   ConcordiumTransactionFilter,
   ConcordiumTransactionEventFilter,
@@ -25,7 +24,7 @@ import {
 import {FileReference, Processor} from '@subql/types-core';
 import {plainToClass, Transform, Type} from 'class-transformer';
 import {IsArray, IsEnum, IsInt, IsOptional, IsString, IsObject, ValidateNested} from 'class-validator';
-import {SubqlConcordiumDatasourceKind, SubqlConcordiumHandlerKind, SubqlConcordiumProcessorOptions} from './types';
+import {SubqlConcordiumDatasourceKind, SubqlConcordiumHandlerKind} from './types';
 
 export class TransactionFilter implements ConcordiumTransactionFilter {
   @IsOptional()
@@ -142,12 +141,6 @@ export class CustomMapping implements SubqlMapping<SubqlCustomHandler> {
   file: string;
 }
 
-export class ConcordiumProcessorOptions implements SubqlConcordiumProcessorOptions {
-  @IsOptional()
-  @IsString()
-  address?: string;
-}
-
 export class RuntimeDataSourceBase<M extends SubqlMapping<SubqlRuntimeHandler>>
   extends BaseDataSource
   implements SubqlRuntimeDatasource<M>
@@ -161,10 +154,6 @@ export class RuntimeDataSourceBase<M extends SubqlMapping<SubqlRuntimeHandler>>
   mapping: M;
   @IsOptional()
   assets?: Map<string, FileReference>;
-  @IsOptional()
-  @ValidateNested()
-  @Type(() => ConcordiumProcessorOptions)
-  options?: ConcordiumProcessorOptions;
 }
 
 export class FileReferenceImpl implements FileReference {
@@ -190,7 +179,4 @@ export class CustomDataSourceBase<K extends string, M extends SubqlMapping = Sub
   @Type(() => ProcessorImpl)
   @IsObject()
   processor: Processor<O>;
-  @IsOptional()
-  @ValidateNested()
-  options?: ConcordiumProcessorOptions;
 }

packages/common-concordium/src/project/types.ts

@@ -6,7 +6,6 @@ import {IProjectManifest, ProjectNetworkConfig} from '@subql/types-core';
 
 // All of these used to be redefined in this file, re-exporting for simplicity
 export {
-  SubqlConcordiumProcessorOptions,
   SubqlRuntimeHandler,
   SubqlCustomHandler,
   SubqlHandler,

packages/node/CHANGELOG.md

@@ -5,6 +5,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Added
+- Improved filtering of account and contract addresses as well as bigints (#19)
 
 ## [3.4.0] - 2023-11-13
 ### Changed

packages/node/src/concordium/block.concordium.spec.ts

@@ -4,6 +4,7 @@
 import {
   TransactionEventTag,
   TransactionSummaryType,
+  TransactionKindString,
 } from '@concordium/node-sdk';
 import {
   ConcordiumBlock,
@@ -13,28 +14,11 @@ import {
 } from '@subql/types-concordium';
 import {
   filterBlocksProcessor,
-  filterSpecialEventProcessor,
   filterTransactionsProcessor,
   filterTxEventProcessor,
 } from './block.concordium';
 
 describe('ConcordiumBlockWrapped', () => {
-  beforeEach(() => {
-    const block = {
-      blockHeight: '10',
-      blockHash: 'hash',
-    } as unknown as ConcordiumBlock;
-    const transactions = [
-      { type: 'transfer', amount: '100' } as unknown as ConcordiumTransaction,
-    ];
-    const txEvents = [
-      { tag: 'event1', amount: '100' } as unknown as ConcordiumTransactionEvent,
-    ];
-    const specialEvents = [
-      { tag: 'event2', amount: '100' } as unknown as ConcordiumSpecialEvent,
-    ];
-  });
-
   it('should filter blocks', () => {
     const block = {
       blockHeight: '10',
@@ -152,7 +136,7 @@ describe('ConcordiumBlockWrapped', () => {
     } as unknown as ConcordiumSpecialEvent;
     const filter = { type: 'event2' };
 
-    expect(filterSpecialEventProcessor(specialEvent, filter)).toBe(true);
+    expect(filterTxEventProcessor(specialEvent, filter)).toBe(true);
   });
 
   it('should filter special events - without values - for false', () => {
@@ -162,7 +146,7 @@ describe('ConcordiumBlockWrapped', () => {
     } as unknown as ConcordiumSpecialEvent;
     const filter = { type: 'event1' };
 
-    expect(filterSpecialEventProcessor(specialEvent, filter)).toBe(false);
+    expect(filterTxEventProcessor(specialEvent, filter)).toBe(false);
   });
 
   it('should filter special events - with values - for true', () => {
@@ -177,7 +161,7 @@ describe('ConcordiumBlockWrapped', () => {
       },
     };
 
-    expect(filterSpecialEventProcessor(specialEvent, filter)).toBe(true);
+    expect(filterTxEventProcessor(specialEvent, filter)).toBe(true);
   });
 
   it('should filter special events - with values - for false', () => {
@@ -192,6 +176,51 @@ describe('ConcordiumBlockWrapped', () => {
       },
     };
 
-    expect(filterSpecialEventProcessor(specialEvent, filter)).toBe(false);
+    expect(filterTxEventProcessor(specialEvent, filter)).toBe(false);
+  });
+
+  it('should filter nested values', () => {
+    const tx = {
+      index: BigInt(0),
+      energyCost: BigInt(3739),
+      hash: '574289208dfd6dff2065e2549164bcfdd97f91e65cb42941a40e574ceccbf7b9',
+      type: TransactionSummaryType.AccountTransaction,
+      cost: BigInt(14685685),
+      sender: '4AuT5RRmBwcdkLMA6iVjxTDb1FQmxwAh3wHBS22mggWL8xH6s3',
+      transactionType: TransactionKindString.Update,
+    } as ConcordiumTransaction;
+
+    const match = filterTransactionsProcessor(tx, {
+      type: TransactionSummaryType.AccountTransaction,
+      values: {
+        sender: '4AuT5RRmBwcdkLMA6iVjxTDb1FQmxwAh3wHBS22mggWL8xH6s3',
+      },
+    });
+    expect(match).toBeTruthy();
+
+    const event = {
+      tag: 'Updated',
+      contractVersion: 1,
+      address: { index: BigInt(6536), subindex: BigInt(0) },
+      instigator: {
+        type: 'AddressAccount',
+        address: '4AuT5RRmBwcdkLMA6iVjxTDb1FQmxwAh3wHBS22mggWL8xH6s3',
+      },
+      amount: BigInt(0),
+      message:
+        '35000000b0373129de61634f2c72ee0c008a9d37f6d10367875108b622bd9480cff50d9f0c000000544539445156524a5430343d005e94526500000000',
+      receiveName: 'Provenance-tag.update_tag_log',
+      events: [],
+    } as ConcordiumTransactionEvent;
+
+    const matchEvent = filterTxEventProcessor(event, {
+      type: TransactionEventTag.Updated,
+      values: {
+        instigator: '4AuT5RRmBwcdkLMA6iVjxTDb1FQmxwAh3wHBS22mggWL8xH6s3',
+        address: '6536',
+        amount: '0',
+      },
+    });
+    expect(matchEvent).toBeTruthy();
   });
 });

packages/node/src/concordium/block.concordium.ts

@@ -1,6 +1,8 @@
 // Copyright 2020-2023 SubQuery Pte Ltd authors & contributors
 // SPDX-License-Identifier: GPL-3.0
 
+import { ContractAddress, AddressAccount, Address } from '@concordium/node-sdk';
+
 import {
   ConcordiumBlock,
   ConcordiumTransactionFilter,
@@ -15,7 +17,6 @@ import {
 export function filterBlocksProcessor(
   block: ConcordiumBlock,
   filter: ConcordiumBlockFilter,
-  address?: string,
 ): boolean {
   if (filter?.modulo && Number(block.blockHeight) % filter.modulo !== 0) {
     return false;
@@ -26,7 +27,6 @@ export function filterBlocksProcessor(
 export function filterTransactionsProcessor(
   transaction: ConcordiumTransaction,
   filter: ConcordiumTransactionFilter,
-  address?: string,
 ): boolean {
   if (!filter) return true;
 
@@ -42,36 +42,65 @@ export function filterTransactionsProcessor(
 }
 
 export function filterTxEventProcessor(
-  txEvent: ConcordiumTransactionEvent,
-  filter: ConcordiumTransactionEventFilter,
-  address?: string,
+  txEvent: ConcordiumTransactionEvent | ConcordiumSpecialEvent,
+  filter: ConcordiumTransactionEventFilter | ConcordiumSpecialEventFilter,
 ): boolean {
   if (!filter) return true;
 
   if (filter.type && txEvent.tag !== filter.type) return false;
 
   if (filter.values) {
     for (const key in filter.values) {
-      if (filter.values[key] !== txEvent[key]) return false;
+      const filterValue = filter.values[key];
+      const eventValue = txEvent[key];
+      if (
+        filterValue !== eventValue &&
+        !equalsAddressAccount(eventValue, filterValue) &&
+        !equalsContractAddress(eventValue, filterValue) &&
+        !equalsAddress(eventValue, filterValue) &&
+        !equalsBigInt(eventValue, filterValue)
+      ) {
+        return false;
+      }
     }
   }
 
   return true;
 }
 
-export function filterSpecialEventProcessor(
-  specialEvent: ConcordiumSpecialEvent,
-  filter: ConcordiumSpecialEventFilter,
-): boolean {
-  if (!filter) return true;
+function isAddressAccount(t: any): t is AddressAccount {
+  return t.type === 'AddressAccount';
+}
 
-  if (filter.type && specialEvent.tag !== filter.type) return false;
+function equalsAddressAccount(t: any, address: string): boolean {
+  return isAddressAccount(t) && t.address === address;
+}
 
-  if (filter.values) {
-    for (const key in filter.values) {
-      if (filter.values[key] !== specialEvent[key]) return false;
-    }
-  }
+function isContractAddress(t: any): t is ContractAddress {
+  return typeof t.index === 'bigint' && typeof t.subindex === 'bigint';
+}
 
-  return true;
+function equalsContractAddress(t: any, index: string): boolean {
+  return isContractAddress(t) && t.index === BigInt(index);
+}
+
+function isAddress(t: any): t is Address {
+  return t.type === 'AddressContract' || isAddressAccount(t);
+}
+
+function equalsAddress(t: any, addressOrIndex: string): boolean {
+  if (!isAddress(t)) return false;
+
+  return (
+    equalsAddressAccount(t, addressOrIndex) ||
+    equalsContractAddress(t.address, addressOrIndex)
+  );
+}
+
+function equalsBigInt(a: bigint, b: string): boolean {
+  try {
+    return a === BigInt(b);
+  } catch (e) {
+    return false;
+  }
 }

packages/node/src/indexer/dynamic-ds.service.ts

@@ -54,11 +54,6 @@ export class DynamicDsService extends BaseDynamicDsService<ConcordiumProjectDs>
         };
         await this.dsProcessorService.validateCustomDs([dsObj]);
       } else if (isRuntimeDs(dsObj)) {
-        dsObj.options = {
-          ...dsObj.options,
-          ...params.args,
-        };
-
         const parsedDs = plainToClass(ConcordiumRuntimeDataSourceImpl, dsObj);
 
         const errors = validateSync(parsedDs, {

packages/node/src/indexer/fetch.service.ts

@@ -8,7 +8,6 @@ import { SchedulerRegistry } from '@nestjs/schedule';
 import {
   isCustomDs,
   SubqlConcordiumHandlerKind,
-  SubqlConcordiumProcessorOptions,
   ConcordiumTransactionFilter,
 } from '@subql/common-concordium';
 import {
@@ -27,7 +26,7 @@ import {
   DictionaryQueryCondition,
   DictionaryQueryEntry,
 } from '@subql/types-core';
-import { groupBy, partition, setWith, sortBy, uniqBy } from 'lodash';
+import { setWith, sortBy, uniqBy } from 'lodash';
 import { ConcordiumApi } from '../concordium';
 import { calcInterval } from '../concordium/utils.concordium';
 import { SubqueryProject } from '../configure/SubqueryProject';
@@ -140,12 +139,8 @@ export function speicalEventFilterToQueryEntry(
   };
 }
 
-type GroupedConcordiumProjectDs = SubqlDatasource & {
-  groupedOptions?: SubqlConcordiumProcessorOptions[];
-};
-
 export function buildDictionaryQueryEntries(
-  dataSources: GroupedConcordiumProjectDs[],
+  dataSources: SubqlDatasource[],
 ): DictionaryQueryEntry[] {
   const queryEntries: DictionaryQueryEntry[] = [];
 
@@ -246,31 +241,7 @@ export class FetchService extends BaseFetchService<
   protected buildDictionaryQueryEntries(
     dataSources: (SubqlDatasource & { name?: string })[],
   ): DictionaryQueryEntry[] {
-    const [normalDataSources, templateDataSources] = partition(
-      dataSources,
-      (ds) => !ds.name,
-    );
-
-    // Group templ
-    const groupedDataSources = Object.values(
-      groupBy(templateDataSources, (ds) => ds.name),
-    ).map((grouped) => {
-      if (grouped.length === 1) {
-        return grouped[0];
-      }
-
-      const options = grouped.map((ds) => ds.options);
-      const ref = grouped[0];
-
-      return {
-        ...ref,
-        groupedOptions: options,
-      };
-    });
-
-    const filteredDs = [...normalDataSources, ...groupedDataSources];
-
-    return buildDictionaryQueryEntries(filteredDs);
+    return buildDictionaryQueryEntries(dataSources);
   }
 
   protected async getFinalizedHeight(): Promise<number> {