From fddfea6145b123371fa102d908a972a256feca61 Mon Sep 17 00:00:00 2001 From: incrypto32 Date: Sun, 5 Jan 2025 21:58:50 +0530 Subject: [PATCH 1/6] Fix subgraphs without abi field failing to build --- packages/cli/src/compiler/index.ts | 58 ++++++++++++++++++++---------- 1 file changed, 39 insertions(+), 19 deletions(-) diff --git a/packages/cli/src/compiler/index.ts b/packages/cli/src/compiler/index.ts index 6dbe4911d..143b5f7e3 100644 --- a/packages/cli/src/compiler/index.ts +++ b/packages/cli/src/compiler/index.ts @@ -508,6 +508,9 @@ export default class Compiler { `Failed to write compiled subgraph to ${displayDir}`, `Warnings while writing compiled subgraph to ${displayDir}`, async spinner => { + // Add debug log for initial subgraph state + compilerDebug('Initial subgraph state:', subgraph.toJS()); + // Copy schema and update its path subgraph = subgraph.updateIn(['schema', 'file'], schemaFile => { const schemaFilePath = path.resolve(this.sourceDir, schemaFile as string); @@ -518,32 +521,49 @@ export default class Compiler { return path.relative(this.options.outputDir, targetFile); }); + // Add debug log before processing data sources + compilerDebug('Processing dataSources:', subgraph.get('dataSources').toJS()); + // Copy data source files and update their paths subgraph = subgraph.update('dataSources', (dataSources: any[]) => dataSources.map(dataSource => { + // Add debug log for each data source + compilerDebug('Processing dataSource:', dataSource.toJS()); + let updatedDataSource = dataSource; if (this.protocol.hasABIs()) { - updatedDataSource = updatedDataSource - // Write data source ABIs to the output directory - .updateIn(['mapping', 'abis'], (abis: any[]) => - abis.map((abi: any) => - abi.update('file', (abiFile: string) => { - abiFile = path.resolve(this.sourceDir, abiFile); - const abiData = this.ABI.load(abi.get('name'), abiFile); - return path.relative( - this.options.outputDir, - this._writeSubgraphFile( - abiFile, - JSON.stringify(abiData.data.toJS(), null, 2), - this.sourceDir, - this.subgraphDir(this.options.outputDir, dataSource), - spinner, - ), - ); - }), - ), + // Add debug log for ABIs + compilerDebug( + 'Processing ABIs for dataSource:', + dataSource.getIn(['mapping', 'abis'])?.toJS() || 'undefined', + ); + + updatedDataSource = updatedDataSource.updateIn(['mapping', 'abis'], (abis: any[]) => { + compilerDebug('ABIs value:', Array.isArray(abis) ? abis : 'undefined'); + + if (!abis) { + compilerDebug('No ABIs found for dataSource'); + return immutable.List(); + } + + return abis.map((abi: any) => + abi.update('file', (abiFile: string) => { + abiFile = path.resolve(this.sourceDir, abiFile); + const abiData = this.ABI.load(abi.get('name'), abiFile); + return path.relative( + this.options.outputDir, + this._writeSubgraphFile( + abiFile, + JSON.stringify(abiData.data.toJS(), null, 2), + this.sourceDir, + this.subgraphDir(this.options.outputDir, dataSource), + spinner, + ), + ); + }), ); + }); } if (protocol.name == 'substreams' || protocol.name == 'substreams/triggers') { From 6b9f04b11dc0efd9e931aba755e52e8d084bcc80 Mon Sep 17 00:00:00 2001 From: incrypto32 Date: Thu, 23 Jan 2025 09:51:23 +0400 Subject: [PATCH 2/6] Fix graph init for composed subgraphs --- packages/cli/src/commands/init.ts | 47 ++++++++++++++++++--- packages/cli/src/scaffold/index.ts | 60 +++++++++++++++++++++------ packages/cli/src/utils.ts | 66 ++++++++++++++++++++++++++++++ 3 files changed, 154 insertions(+), 19 deletions(-) diff --git a/packages/cli/src/commands/init.ts b/packages/cli/src/commands/init.ts index 731091be0..08f1d4232 100644 --- a/packages/cli/src/commands/init.ts +++ b/packages/cli/src/commands/init.ts @@ -23,7 +23,11 @@ import EthereumABI from '../protocols/ethereum/abi.js'; import Protocol, { ProtocolName } from '../protocols/index.js'; import { abiEvents } from '../scaffold/schema.js'; import Schema from '../schema.js'; -import { createIpfsClient, loadSubgraphSchemaFromIPFS } from '../utils.js'; +import { + createIpfsClient, + loadSubgraphSchemaFromIPFS, + validateSubgraphNetworkMatch, +} from '../utils.js'; import { validateContract } from '../validation/index.js'; import AddCommand from './add.js'; @@ -515,7 +519,7 @@ async function processInitForm( value: 'contract', }, { message: 'Substreams', name: 'substreams', value: 'substreams' }, - // { message: 'Subgraph', name: 'subgraph', value: 'subgraph' }, + { message: 'Subgraph', name: 'subgraph', value: 'subgraph' }, ].filter(({ name }) => name), }); @@ -596,9 +600,17 @@ async function processInitForm( isSubstreams || (!protocolInstance.hasContract() && !isComposedSubgraph), initial: initContract, - validate: async (value: string) => { + validate: async (value: string): Promise => { if (isComposedSubgraph) { - return value.startsWith('Qm') ? true : 'Subgraph deployment ID must start with Qm'; + if (!ipfsNode) { + return true; // Skip validation if no IPFS node is available + } + const ipfs = createIpfsClient(ipfsNode); + const { valid, error } = await validateSubgraphNetworkMatch(ipfs, value, network.id); + if (!valid) { + return error || 'Invalid subgraph network match'; + } + return true; } if (initFromExample !== undefined || !protocolInstance.hasContract()) { return true; @@ -731,7 +743,7 @@ async function processInitForm( isSubstreams || !!initAbiPath || isComposedSubgraph, - validate: async (value: string) => { + validate: async (value: string): Promise => { if ( initFromExample || abiFromApi || @@ -822,6 +834,22 @@ async function processInitForm( await promptManager.executeInteractive(); + // If loading from IPFS, validate network matches + if (ipfsNode && subgraphName.startsWith('Qm')) { + const ipfs = createIpfsClient(ipfsNode); + try { + const { valid, error } = await validateSubgraphNetworkMatch(ipfs, subgraphName, network.id); + if (!valid) { + throw new Error(error || 'Invalid subgraph network match'); + } + } catch (e) { + if (e instanceof Error) { + print.error(`Failed to validate subgraph network: ${e.message}`); + } + throw e; + } + } + return { abi: (abiFromApi || abiFromFile)!, protocolInstance, @@ -1188,8 +1216,9 @@ async function initSubgraphFromContract( } if ( - !protocolInstance.isComposedSubgraph() && + !isComposedSubgraph && protocolInstance.hasABIs() && + abi && // Add check for abi existence (abiEvents(abi).size === 0 || // @ts-expect-error TODO: the abiEvents result is expected to be a List, how's it an array? abiEvents(abi).length === 0) @@ -1204,6 +1233,12 @@ async function initSubgraphFromContract( `Failed to create subgraph scaffold`, `Warnings while creating subgraph scaffold`, async spinner => { + initDebugger('Generating scaffold with ABI:', abi); + initDebugger('ABI data:', abi?.data); + if (abi) { + initDebugger('ABI events:', abiEvents(abi)); + } + const scaffold = await generateScaffold( { protocolInstance, diff --git a/packages/cli/src/scaffold/index.ts b/packages/cli/src/scaffold/index.ts index 66a14cee0..d80c780c6 100644 --- a/packages/cli/src/scaffold/index.ts +++ b/packages/cli/src/scaffold/index.ts @@ -1,3 +1,4 @@ +import debugFactory from 'debug'; import fs from 'fs-extra'; import { strings } from 'gluegun'; import prettier from 'prettier'; @@ -11,6 +12,8 @@ import { generateEventIndexingHandlers } from './mapping.js'; import { abiEvents, generateEventType, generateExampleEntityType } from './schema.js'; import { generateTestsFiles } from './tests.js'; +const scaffoldDebugger = debugFactory('graph-cli:scaffold'); + const GRAPH_CLI_VERSION = process.env.GRAPH_CLI_TESTS ? // JSON.stringify should remove this key, we will install the local // graph-cli for the tests using `npm link` instead of fetching from npm. @@ -47,18 +50,34 @@ export default class Scaffold { spkgPath?: string; entities?: string[]; - constructor(options: ScaffoldOptions) { - this.protocol = options.protocol; - this.abi = options.abi; - this.indexEvents = options.indexEvents; - this.contract = options.contract; - this.network = options.network; - this.contractName = options.contractName; - this.subgraphName = options.subgraphName; - this.startBlock = options.startBlock; - this.node = options.node; - this.spkgPath = options.spkgPath; - this.entities = options.entities; + constructor({ + protocol, + abi, + contract, + network, + contractName, + startBlock, + subgraphName, + node, + spkgPath, + indexEvents, + entities, + }: ScaffoldOptions) { + this.protocol = protocol; + this.abi = abi; + this.contract = contract; + this.network = network; + this.contractName = contractName; + this.startBlock = startBlock; + this.subgraphName = subgraphName; + this.node = node; + this.spkgPath = spkgPath; + this.indexEvents = indexEvents; + this.entities = entities; + + scaffoldDebugger('Scaffold constructor called with ABI:', abi); + scaffoldDebugger('ABI data:', abi?.data); + scaffoldDebugger('ABI file:', abi?.file); } async generatePackageJson() { @@ -203,9 +222,24 @@ dataSources: } async generateABIs() { + scaffoldDebugger('Generating ABIs...'); + scaffoldDebugger('Protocol has ABIs:', this.protocol.hasABIs()); + scaffoldDebugger('ABI data:', this.abi?.data); + scaffoldDebugger('ABI file:', this.abi?.file); + + if (!this.protocol.hasABIs()) { + scaffoldDebugger('Protocol does not have ABIs, skipping ABI generation'); + return; + } + + if (!this.abi?.data) { + scaffoldDebugger('ABI data is undefined, skipping ABI generation'); + return; + } + return this.protocol.hasABIs() ? { - [`${this.contractName}.json`]: await prettier.format(JSON.stringify(this.abi?.data), { + [`${this.contractName}.json`]: await prettier.format(JSON.stringify(this.abi.data), { parser: 'json', }), } diff --git a/packages/cli/src/utils.ts b/packages/cli/src/utils.ts index 5b8223d20..4dcbb9d42 100644 --- a/packages/cli/src/utils.ts +++ b/packages/cli/src/utils.ts @@ -32,3 +32,69 @@ export async function loadSubgraphSchemaFromIPFS(ipfsClient: any, manifest: stri throw Error(`Failed to load schema from IPFS ${manifest}`); } } + +export async function loadManifestFromIPFS(ipfsClient: any, manifest: string) { + try { + const manifestBuffer = ipfsClient.cat(manifest); + let manifestFile = ''; + for await (const chunk of manifestBuffer) { + manifestFile += Buffer.from(chunk).toString('utf8'); + } + return manifestFile; + } catch (e) { + utilsDebug.extend('loadManifestFromIPFS')(`Failed to load manifest from IPFS ${manifest}`); + utilsDebug.extend('loadManifestFromIPFS')(e); + throw Error(`Failed to load manifest from IPFS ${manifest}`); + } +} + +/** + * Validates that the network of a source subgraph matches the target network + * @param ipfsClient IPFS client instance + * @param sourceSubgraphId IPFS hash of the source subgraph + * @param targetNetwork Network of the target subgraph being created + * @returns Object containing validation result and error message if any + */ +export async function validateSubgraphNetworkMatch( + ipfsClient: any, + sourceSubgraphId: string, + targetNetwork: string, +): Promise<{ valid: boolean; error?: string }> { + try { + const manifestFile = await loadManifestFromIPFS(ipfsClient, sourceSubgraphId); + const manifestYaml = yaml.load(manifestFile) as any; + + // Extract network from data sources + const dataSources = manifestYaml.dataSources || []; + const templates = manifestYaml.templates || []; + const allSources = [...dataSources, ...templates]; + + if (allSources.length === 0) { + return { valid: true }; // No data sources to validate + } + + // Get network from first data source + const sourceNetwork = allSources[0].network; + if (!sourceNetwork) { + return { valid: true }; // Network not specified in source, skip validation + } + + const normalizedSourceNetwork = sourceNetwork.toLowerCase(); + const normalizedTargetNetwork = targetNetwork.toLowerCase(); + + if (normalizedSourceNetwork !== normalizedTargetNetwork) { + return { + valid: false, + error: `Network mismatch: The source subgraph is indexing the '${sourceNetwork}' network, but you're creating a subgraph for '${targetNetwork}' network. When composing subgraphs, they must index the same network.`, + }; + } + + return { valid: true }; + } catch (e) { + utilsDebug.extend('validateSubgraphNetworkMatch')(`Failed to validate network match: ${e}`); + return { + valid: false, + error: e instanceof Error ? e.message : 'Failed to validate subgraph network', + }; + } +} From b065646a692a63311fca8de318e075d6e515bb7c Mon Sep 17 00:00:00 2001 From: incrypto32 Date: Thu, 23 Jan 2025 11:10:19 +0400 Subject: [PATCH 3/6] Add changeset --- .changeset/curly-buses-hang.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/curly-buses-hang.md diff --git a/.changeset/curly-buses-hang.md b/.changeset/curly-buses-hang.md new file mode 100644 index 000000000..ef7b1341e --- /dev/null +++ b/.changeset/curly-buses-hang.md @@ -0,0 +1,5 @@ +--- +'@graphprotocol/graph-cli': minor +--- + +Add support for subgraph datasource in `graph init` From 735b56572041ad2dd19b6e6630e88ff1abdc704a Mon Sep 17 00:00:00 2001 From: incrypto32 Date: Fri, 24 Jan 2025 15:38:41 +0400 Subject: [PATCH 4/6] Fix validation not working --- packages/cli/src/commands/init.ts | 34 +++++++++++++++---------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/packages/cli/src/commands/init.ts b/packages/cli/src/commands/init.ts index 08f1d4232..4c324a2fa 100644 --- a/packages/cli/src/commands/init.ts +++ b/packages/cli/src/commands/init.ts @@ -588,6 +588,19 @@ async function processInitForm( }, }); + promptManager.addStep({ + type: 'input', + name: 'ipfs', + message: `IPFS node to use for fetching subgraph manifest`, + initial: ipfsUrl, + skip: () => !isComposedSubgraph, + result: value => { + ipfsNode = value; + initDebugger.extend('processInitForm')('ipfs: %O', value); + return value; + }, + }); + promptManager.addStep({ type: 'input', name: 'source', @@ -603,7 +616,7 @@ async function processInitForm( validate: async (value: string): Promise => { if (isComposedSubgraph) { if (!ipfsNode) { - return true; // Skip validation if no IPFS node is available + return true; } const ipfs = createIpfsClient(ipfsNode); const { valid, error } = await validateSubgraphNetworkMatch(ipfs, value, network.id); @@ -691,19 +704,6 @@ async function processInitForm( }, }); - promptManager.addStep({ - type: 'input', - name: 'ipfs', - message: `IPFS node to use for fetching subgraph manifest`, - initial: ipfsUrl, - skip: () => !isComposedSubgraph, - result: value => { - ipfsNode = value; - initDebugger.extend('processInitForm')('ipfs: %O', value); - return value; - }, - }); - promptManager.addStep({ type: 'input', name: 'spkg', @@ -834,11 +834,11 @@ async function processInitForm( await promptManager.executeInteractive(); - // If loading from IPFS, validate network matches - if (ipfsNode && subgraphName.startsWith('Qm')) { + // Validate network matches if loading from IPFS + if (ipfsNode && source && source.startsWith('Qm')) { const ipfs = createIpfsClient(ipfsNode); try { - const { valid, error } = await validateSubgraphNetworkMatch(ipfs, subgraphName, network.id); + const { valid, error } = await validateSubgraphNetworkMatch(ipfs, source!, network.id); if (!valid) { throw new Error(error || 'Invalid subgraph network match'); } From 600280d88ce16f759b7c1a2c23ff6e54d419823e Mon Sep 17 00:00:00 2001 From: incrypto32 Date: Fri, 24 Jan 2025 15:38:52 +0400 Subject: [PATCH 5/6] Support declared calls in manifest --- packages/cli/src/protocols/subgraph/manifest.graphql | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/cli/src/protocols/subgraph/manifest.graphql b/packages/cli/src/protocols/subgraph/manifest.graphql index f97621368..35731e2b7 100644 --- a/packages/cli/src/protocols/subgraph/manifest.graphql +++ b/packages/cli/src/protocols/subgraph/manifest.graphql @@ -63,6 +63,7 @@ type ContractABI { type EntityHandler { handler: String! entity: String! + calls: JSON } type Graft { From 3a75a7fd2231fb7cd2ed22c8020b1dd5539efa70 Mon Sep 17 00:00:00 2001 From: incrypto32 Date: Mon, 27 Jan 2025 16:14:32 +0400 Subject: [PATCH 6/6] Lint fix --- packages/cli/src/commands/init.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/cli/src/commands/init.ts b/packages/cli/src/commands/init.ts index 4c324a2fa..864682774 100644 --- a/packages/cli/src/commands/init.ts +++ b/packages/cli/src/commands/init.ts @@ -835,7 +835,7 @@ async function processInitForm( await promptManager.executeInteractive(); // Validate network matches if loading from IPFS - if (ipfsNode && source && source.startsWith('Qm')) { + if (ipfsNode && source?.startsWith('Qm')) { const ipfs = createIpfsClient(ipfsNode); try { const { valid, error } = await validateSubgraphNetworkMatch(ipfs, source!, network.id); @@ -844,7 +844,7 @@ async function processInitForm( } } catch (e) { if (e instanceof Error) { - print.error(`Failed to validate subgraph network: ${e.message}`); + print.error(`Failed to validate subgraph network: ${e?.message}`); } throw e; }