diff --git a/.changeset/@graphprotocol_graph-cli-1920-dependencies.md b/.changeset/@graphprotocol_graph-cli-1920-dependencies.md new file mode 100644 index 000000000..5f845092a --- /dev/null +++ b/.changeset/@graphprotocol_graph-cli-1920-dependencies.md @@ -0,0 +1,6 @@ +--- +"@graphprotocol/graph-cli": patch +--- +dependencies updates: + - Updated dependency [`@oclif/core@4.2.6` ↗︎](https://www.npmjs.com/package/@oclif/core/v/4.2.6) (from `4.2.4`, in `dependencies`) + - Updated dependency [`semver@7.7.1` ↗︎](https://www.npmjs.com/package/semver/v/7.7.1) (from `7.6.3`, in `dependencies`) diff --git a/.changeset/curly-buses-hang.md b/.changeset/curly-buses-hang.md new file mode 100644 index 000000000..ef7b1341e --- /dev/null +++ b/.changeset/curly-buses-hang.md @@ -0,0 +1,5 @@ +--- +'@graphprotocol/graph-cli': minor +--- + +Add support for subgraph datasource in `graph init` diff --git a/packages/cli/src/commands/codegen.ts b/packages/cli/src/commands/codegen.ts index fe76b985d..75bbbe5d6 100644 --- a/packages/cli/src/commands/codegen.ts +++ b/packages/cli/src/commands/codegen.ts @@ -43,7 +43,6 @@ export default class CodegenCommand extends Command { summary: 'IPFS node to use for fetching subgraph data.', char: 'i', default: DEFAULT_IPFS_URL, - hidden: true, }), 'uncrashable-config': Flags.file({ summary: 'Directory for uncrashable config.', diff --git a/packages/cli/src/commands/init.ts b/packages/cli/src/commands/init.ts index 731091be0..581958a50 100644 --- a/packages/cli/src/commands/init.ts +++ b/packages/cli/src/commands/init.ts @@ -23,7 +23,13 @@ import EthereumABI from '../protocols/ethereum/abi.js'; import Protocol, { ProtocolName } from '../protocols/index.js'; import { abiEvents } from '../scaffold/schema.js'; import Schema from '../schema.js'; -import { createIpfsClient, loadSubgraphSchemaFromIPFS } from '../utils.js'; +import { + createIpfsClient, + getMinStartBlock, + loadManifestYaml, + loadSubgraphSchemaFromIPFS, + validateSubgraphNetworkMatch, +} from '../utils.js'; import { validateContract } from '../validation/index.js'; import AddCommand from './add.js'; @@ -54,6 +60,10 @@ export default class InitCommand extends Command { summary: 'Graph node for which to initialize.', char: 'g', }), + 'from-subgraph': Flags.string({ + description: 'Creates a scaffold based on an existing subgraph.', + exclusive: ['from-example', 'from-contract'], + }), 'from-contract': Flags.string({ description: 'Creates a scaffold based on an existing contract.', exclusive: ['from-example'], @@ -88,7 +98,6 @@ export default class InitCommand extends Command { description: 'Block number to start indexing from.', // TODO: using a default sets the value and therefore requires --from-contract // default: '0', - dependsOn: ['from-contract'], }), abi: Flags.string({ @@ -110,7 +119,6 @@ export default class InitCommand extends Command { summary: 'IPFS node to use for fetching subgraph data.', char: 'i', default: DEFAULT_IPFS_URL, - hidden: true, }), }; @@ -124,6 +132,7 @@ export default class InitCommand extends Command { protocol, node: nodeFlag, 'from-contract': fromContract, + 'from-subgraph': fromSubgraph, 'contract-name': contractName, 'from-example': fromExample, 'index-events': indexEvents, @@ -138,11 +147,20 @@ export default class InitCommand extends Command { initDebugger('Flags: %O', flags); + if (startBlock && !(fromContract || fromSubgraph)) { + this.error('--start-block can only be used with --from-contract or --from-subgraph'); + } + + if (fromContract && fromSubgraph) { + this.error('Cannot use both --from-contract and --from-subgraph at the same time'); + } + if (skipGit) { this.warn( 'The --skip-git flag will be removed in the next major version. By default we will stop initializing a Git repository.', ); } + if ((fromContract || spkgPath) && !network && !fromExample) { this.error('--network is required when using --from-contract or --spkg'); } @@ -196,16 +214,15 @@ export default class InitCommand extends Command { let abi!: EthereumABI; // If all parameters are provided from the command-line, - // go straight to creating the subgraph from an existing contract - if ((fromContract || spkgPath) && protocol && subgraphName && directory && network && node) { - const registry = await loadRegistry(); - const contractService = new ContractService(registry); - const sourcifyContractInfo = await contractService.getFromSourcify( - EthereumABI, - network, - fromContract!, - ); - + // go straight to creating the subgraph from an existing contract or source subgraph + if ( + (fromContract || spkgPath || fromSubgraph) && + protocol && + subgraphName && + directory && + network && + node + ) { if (!protocolChoices.includes(protocol as ProtocolName)) { this.error( `Protocol '${protocol}' is not supported, choose from these options: ${protocolChoices.join( @@ -217,7 +234,31 @@ export default class InitCommand extends Command { const protocolInstance = new Protocol(protocol as ProtocolName); - if (protocolInstance.hasABIs()) { + if (fromSubgraph && !protocolInstance.isComposedSubgraph()) { + this.error('--protocol can only be subgraph when using --from-subgraph'); + } + + if ( + fromContract && + (protocolInstance.isComposedSubgraph() || protocolInstance.isSubstreams()) + ) { + this.error('--protocol cannot be subgraph or substreams when using --from-contract'); + } + + if (spkgPath && !protocolInstance.isSubstreams()) { + this.error('--protocol can only be substreams when using --spkg'); + } + + // Only fetch contract info and ABI for non-source-subgraph cases + if (!fromSubgraph && protocolInstance.hasABIs()) { + const registry = await loadRegistry(); + const contractService = new ContractService(registry); + const sourcifyContractInfo = await contractService.getFromSourcify( + EthereumABI, + network, + fromContract!, + ); + const ABI = protocolInstance.getABI(); if (abiPath) { try { @@ -241,7 +282,7 @@ export default class InitCommand extends Command { protocolInstance, abi, directory, - source: fromContract!, + source: fromSubgraph || fromContract!, indexEvents, network, subgraphName, @@ -285,7 +326,7 @@ export default class InitCommand extends Command { abi, abiPath, directory, - source: fromContract, + source: fromContract || fromSubgraph, indexEvents, fromExample, subgraphName, @@ -515,7 +556,7 @@ async function processInitForm( value: 'contract', }, { message: 'Substreams', name: 'substreams', value: 'substreams' }, - // { message: 'Subgraph', name: 'subgraph', value: 'subgraph' }, + { message: 'Subgraph', name: 'subgraph', value: 'subgraph' }, ].filter(({ name }) => name), }); @@ -584,6 +625,30 @@ async function processInitForm( }, }); + promptManager.addStep({ + type: 'input', + name: 'ipfs', + message: `IPFS node to use for fetching subgraph manifest`, + initial: ipfsUrl, + skip: () => !isComposedSubgraph, + validate: value => { + if (!value) { + return 'IPFS node URL cannot be empty'; + } + try { + new URL(value); + return true; + } catch { + return 'Please enter a valid URL'; + } + }, + result: value => { + ipfsNode = value; + initDebugger.extend('processInitForm')('ipfs: %O', value); + return value; + }, + }); + promptManager.addStep({ type: 'input', name: 'source', @@ -596,9 +661,16 @@ async function processInitForm( isSubstreams || (!protocolInstance.hasContract() && !isComposedSubgraph), initial: initContract, - validate: async (value: string) => { + validate: async (value: string): Promise => { if (isComposedSubgraph) { - return value.startsWith('Qm') ? true : 'Subgraph deployment ID must start with Qm'; + const ipfs = createIpfsClient(ipfsNode); + const manifestYaml = await loadManifestYaml(ipfs, value); + const { valid, error } = validateSubgraphNetworkMatch(manifestYaml, network.id); + if (!valid) { + return error || 'Invalid subgraph network match'; + } + startBlock ||= getMinStartBlock(manifestYaml)?.toString(); + return true; } if (initFromExample !== undefined || !protocolInstance.hasContract()) { return true; @@ -648,6 +720,7 @@ async function processInitForm( } else { abiFromApi = initAbi; } + // If startBlock is not provided, try to fetch it from Etherscan API if (!initStartBlock) { startBlock = await retryWithPrompt(() => @@ -679,19 +752,6 @@ async function processInitForm( }, }); - promptManager.addStep({ - type: 'input', - name: 'ipfs', - message: `IPFS node to use for fetching subgraph manifest`, - initial: ipfsUrl, - skip: () => !isComposedSubgraph, - result: value => { - ipfsNode = value; - initDebugger.extend('processInitForm')('ipfs: %O', value); - return value; - }, - }); - promptManager.addStep({ type: 'input', name: 'spkg', @@ -731,7 +791,7 @@ async function processInitForm( isSubstreams || !!initAbiPath || isComposedSubgraph, - validate: async (value: string) => { + validate: async (value: string): Promise => { if ( initFromExample || abiFromApi || @@ -1179,17 +1239,29 @@ async function initSubgraphFromContract( }, }); + // Validate network match first + const manifestYaml = await loadManifestYaml(ipfsClient, source); + const { valid, error } = validateSubgraphNetworkMatch(manifestYaml, network); + if (!valid) { + throw new Error(error || 'Invalid subgraph network match'); + } + + startBlock ||= getMinStartBlock(manifestYaml)?.toString(); const schemaString = await loadSubgraphSchemaFromIPFS(ipfsClient, source); const schema = await Schema.loadFromString(schemaString); entities = schema.getEntityNames(); } catch (e) { + if (e instanceof Error) { + print.error(`Failed to validate subgraph: ${e?.message}`); + } this.error(`Failed to load and parse subgraph schema: ${e.message}`, { exit: 1 }); } } if ( - !protocolInstance.isComposedSubgraph() && + !isComposedSubgraph && protocolInstance.hasABIs() && + abi && // Add check for abi existence (abiEvents(abi).size === 0 || // @ts-expect-error TODO: the abiEvents result is expected to be a List, how's it an array? abiEvents(abi).length === 0) @@ -1204,6 +1276,12 @@ async function initSubgraphFromContract( `Failed to create subgraph scaffold`, `Warnings while creating subgraph scaffold`, async spinner => { + initDebugger('Generating scaffold with ABI:', abi); + initDebugger('ABI data:', abi?.data); + if (abi) { + initDebugger('ABI events:', abiEvents(abi)); + } + const scaffold = await generateScaffold( { protocolInstance, @@ -1260,7 +1338,7 @@ async function initSubgraphFromContract( this.exit(1); } - while (addContract) { + while (addContract && !isComposedSubgraph) { addContract = await addAnotherContract .bind(this)({ protocolInstance, diff --git a/packages/cli/src/compiler/index.ts b/packages/cli/src/compiler/index.ts index 6dbe4911d..143b5f7e3 100644 --- a/packages/cli/src/compiler/index.ts +++ b/packages/cli/src/compiler/index.ts @@ -508,6 +508,9 @@ export default class Compiler { `Failed to write compiled subgraph to ${displayDir}`, `Warnings while writing compiled subgraph to ${displayDir}`, async spinner => { + // Add debug log for initial subgraph state + compilerDebug('Initial subgraph state:', subgraph.toJS()); + // Copy schema and update its path subgraph = subgraph.updateIn(['schema', 'file'], schemaFile => { const schemaFilePath = path.resolve(this.sourceDir, schemaFile as string); @@ -518,32 +521,49 @@ export default class Compiler { return path.relative(this.options.outputDir, targetFile); }); + // Add debug log before processing data sources + compilerDebug('Processing dataSources:', subgraph.get('dataSources').toJS()); + // Copy data source files and update their paths subgraph = subgraph.update('dataSources', (dataSources: any[]) => dataSources.map(dataSource => { + // Add debug log for each data source + compilerDebug('Processing dataSource:', dataSource.toJS()); + let updatedDataSource = dataSource; if (this.protocol.hasABIs()) { - updatedDataSource = updatedDataSource - // Write data source ABIs to the output directory - .updateIn(['mapping', 'abis'], (abis: any[]) => - abis.map((abi: any) => - abi.update('file', (abiFile: string) => { - abiFile = path.resolve(this.sourceDir, abiFile); - const abiData = this.ABI.load(abi.get('name'), abiFile); - return path.relative( - this.options.outputDir, - this._writeSubgraphFile( - abiFile, - JSON.stringify(abiData.data.toJS(), null, 2), - this.sourceDir, - this.subgraphDir(this.options.outputDir, dataSource), - spinner, - ), - ); - }), - ), + // Add debug log for ABIs + compilerDebug( + 'Processing ABIs for dataSource:', + dataSource.getIn(['mapping', 'abis'])?.toJS() || 'undefined', + ); + + updatedDataSource = updatedDataSource.updateIn(['mapping', 'abis'], (abis: any[]) => { + compilerDebug('ABIs value:', Array.isArray(abis) ? abis : 'undefined'); + + if (!abis) { + compilerDebug('No ABIs found for dataSource'); + return immutable.List(); + } + + return abis.map((abi: any) => + abi.update('file', (abiFile: string) => { + abiFile = path.resolve(this.sourceDir, abiFile); + const abiData = this.ABI.load(abi.get('name'), abiFile); + return path.relative( + this.options.outputDir, + this._writeSubgraphFile( + abiFile, + JSON.stringify(abiData.data.toJS(), null, 2), + this.sourceDir, + this.subgraphDir(this.options.outputDir, dataSource), + spinner, + ), + ); + }), ); + }); } if (protocol.name == 'substreams' || protocol.name == 'substreams/triggers') { diff --git a/packages/cli/src/protocols/subgraph/manifest.graphql b/packages/cli/src/protocols/subgraph/manifest.graphql index f97621368..35731e2b7 100644 --- a/packages/cli/src/protocols/subgraph/manifest.graphql +++ b/packages/cli/src/protocols/subgraph/manifest.graphql @@ -63,6 +63,7 @@ type ContractABI { type EntityHandler { handler: String! entity: String! + calls: JSON } type Graft { diff --git a/packages/cli/src/scaffold/index.ts b/packages/cli/src/scaffold/index.ts index 66a14cee0..d80c780c6 100644 --- a/packages/cli/src/scaffold/index.ts +++ b/packages/cli/src/scaffold/index.ts @@ -1,3 +1,4 @@ +import debugFactory from 'debug'; import fs from 'fs-extra'; import { strings } from 'gluegun'; import prettier from 'prettier'; @@ -11,6 +12,8 @@ import { generateEventIndexingHandlers } from './mapping.js'; import { abiEvents, generateEventType, generateExampleEntityType } from './schema.js'; import { generateTestsFiles } from './tests.js'; +const scaffoldDebugger = debugFactory('graph-cli:scaffold'); + const GRAPH_CLI_VERSION = process.env.GRAPH_CLI_TESTS ? // JSON.stringify should remove this key, we will install the local // graph-cli for the tests using `npm link` instead of fetching from npm. @@ -47,18 +50,34 @@ export default class Scaffold { spkgPath?: string; entities?: string[]; - constructor(options: ScaffoldOptions) { - this.protocol = options.protocol; - this.abi = options.abi; - this.indexEvents = options.indexEvents; - this.contract = options.contract; - this.network = options.network; - this.contractName = options.contractName; - this.subgraphName = options.subgraphName; - this.startBlock = options.startBlock; - this.node = options.node; - this.spkgPath = options.spkgPath; - this.entities = options.entities; + constructor({ + protocol, + abi, + contract, + network, + contractName, + startBlock, + subgraphName, + node, + spkgPath, + indexEvents, + entities, + }: ScaffoldOptions) { + this.protocol = protocol; + this.abi = abi; + this.contract = contract; + this.network = network; + this.contractName = contractName; + this.startBlock = startBlock; + this.subgraphName = subgraphName; + this.node = node; + this.spkgPath = spkgPath; + this.indexEvents = indexEvents; + this.entities = entities; + + scaffoldDebugger('Scaffold constructor called with ABI:', abi); + scaffoldDebugger('ABI data:', abi?.data); + scaffoldDebugger('ABI file:', abi?.file); } async generatePackageJson() { @@ -203,9 +222,24 @@ dataSources: } async generateABIs() { + scaffoldDebugger('Generating ABIs...'); + scaffoldDebugger('Protocol has ABIs:', this.protocol.hasABIs()); + scaffoldDebugger('ABI data:', this.abi?.data); + scaffoldDebugger('ABI file:', this.abi?.file); + + if (!this.protocol.hasABIs()) { + scaffoldDebugger('Protocol does not have ABIs, skipping ABI generation'); + return; + } + + if (!this.abi?.data) { + scaffoldDebugger('ABI data is undefined, skipping ABI generation'); + return; + } + return this.protocol.hasABIs() ? { - [`${this.contractName}.json`]: await prettier.format(JSON.stringify(this.abi?.data), { + [`${this.contractName}.json`]: await prettier.format(JSON.stringify(this.abi.data), { parser: 'json', }), } diff --git a/packages/cli/src/utils.ts b/packages/cli/src/utils.ts index 5b8223d20..39d3689b9 100644 --- a/packages/cli/src/utils.ts +++ b/packages/cli/src/utils.ts @@ -32,3 +32,70 @@ export async function loadSubgraphSchemaFromIPFS(ipfsClient: any, manifest: stri throw Error(`Failed to load schema from IPFS ${manifest}`); } } + +export async function loadManifestFromIPFS(ipfsClient: any, manifest: string) { + try { + const manifestBuffer = ipfsClient.cat(manifest); + let manifestFile = ''; + for await (const chunk of manifestBuffer) { + manifestFile += Buffer.from(chunk).toString('utf8'); + } + return manifestFile; + } catch (e) { + utilsDebug.extend('loadManifestFromIPFS')(`Failed to load manifest from IPFS ${manifest}`); + utilsDebug.extend('loadManifestFromIPFS')(e); + throw Error(`Failed to load manifest from IPFS ${manifest}`); + } +} + +export async function loadManifestYaml(ipfsClient: any, manifest: string): Promise { + const manifestFile = await loadManifestFromIPFS(ipfsClient, manifest); + return yaml.load(manifestFile) as any; +} + +/** + * Validates that the network of a source subgraph matches the target network + * @param manifestYaml Parsed manifest YAML + * @param targetNetwork Network of the target subgraph being created + * @returns Object containing validation result and error message if any + */ +export function validateSubgraphNetworkMatch( + manifestYaml: any, + targetNetwork: string, +): { valid: boolean; error?: string } { + // Extract network from data sources + const dataSources = manifestYaml.dataSources || []; + const templates = manifestYaml.templates || []; + const allSources = [...dataSources, ...templates]; + + if (allSources.length === 0) { + return { valid: true }; // No data sources to validate + } + + // Get network from first data source + const sourceNetwork = allSources[0].network; + + if (sourceNetwork !== targetNetwork) { + return { + valid: false, + error: `Network mismatch: The source subgraph is indexing the '${sourceNetwork}' network, but you're creating a subgraph for '${targetNetwork}' network. When composing subgraphs, they must index the same network.`, + }; + } + + return { valid: true }; +} + +/** + * Gets the minimum startBlock from all dataSources in the manifest + * @param manifestYaml Parsed manifest YAML + * @returns The minimum startBlock or undefined if no startBlock is found + */ +export function getMinStartBlock(manifestYaml: any): number | undefined { + const dataSources = manifestYaml.dataSources || []; + + const startBlocks = dataSources + .map((ds: { source?: { startBlock?: number } }) => ds.source?.startBlock) + .filter((block: unknown): block is number => typeof block === 'number'); + + return startBlocks.length > 0 ? Math.min(...startBlocks) : undefined; +}