Subgraph Composition : Fix graph init for composed subgraphs

.changeset/curly-buses-hang.md

@@ -0,0 +1,5 @@
+---
+'@graphprotocol/graph-cli': minor
+---
+
+Add support for subgraph datasource in `graph init`

packages/cli/src/commands/init.ts

@@ -23,7 +23,11 @@ import EthereumABI from '../protocols/ethereum/abi.js';
 import Protocol, { ProtocolName } from '../protocols/index.js';
 import { abiEvents } from '../scaffold/schema.js';
 import Schema from '../schema.js';
-import { createIpfsClient, loadSubgraphSchemaFromIPFS } from '../utils.js';
+import {
+  createIpfsClient,
+  loadSubgraphSchemaFromIPFS,
+  validateSubgraphNetworkMatch,
+} from '../utils.js';
 import { validateContract } from '../validation/index.js';
 import AddCommand from './add.js';
 
@@ -515,7 +519,7 @@ async function processInitForm(
               value: 'contract',
             },
             { message: 'Substreams', name: 'substreams', value: 'substreams' },
-            // { message: 'Subgraph', name: 'subgraph', value: 'subgraph' },
+            { message: 'Subgraph', name: 'subgraph', value: 'subgraph' },
           ].filter(({ name }) => name),
         });
 
@@ -584,6 +588,19 @@ async function processInitForm(
       },
     });
 
+    promptManager.addStep({
+      type: 'input',
+      name: 'ipfs',
+      message: `IPFS node to use for fetching subgraph manifest`,
+      initial: ipfsUrl,
+      skip: () => !isComposedSubgraph,
+      result: value => {
+        ipfsNode = value;
+        initDebugger.extend('processInitForm')('ipfs: %O', value);
+        return value;
+      },
+    });
+
     promptManager.addStep({
       type: 'input',
       name: 'source',
@@ -596,9 +613,17 @@ async function processInitForm(
         isSubstreams ||
         (!protocolInstance.hasContract() && !isComposedSubgraph),
       initial: initContract,
-      validate: async (value: string) => {
+      validate: async (value: string): Promise<string | boolean> => {
         if (isComposedSubgraph) {
-          return value.startsWith('Qm') ? true : 'Subgraph deployment ID must start with Qm';
+          if (!ipfsNode) {
+            return true;
+          }
+          const ipfs = createIpfsClient(ipfsNode);
+          const { valid, error } = await validateSubgraphNetworkMatch(ipfs, value, network.id);
+          if (!valid) {
+            return error || 'Invalid subgraph network match';
+          }
+          return true;
         }
         if (initFromExample !== undefined || !protocolInstance.hasContract()) {
           return true;
@@ -679,19 +704,6 @@ async function processInitForm(
       },
     });
 
-    promptManager.addStep({
-      type: 'input',
-      name: 'ipfs',
-      message: `IPFS node to use for fetching subgraph manifest`,
-      initial: ipfsUrl,
-      skip: () => !isComposedSubgraph,
-      result: value => {
-        ipfsNode = value;
-        initDebugger.extend('processInitForm')('ipfs: %O', value);
-        return value;
-      },
-    });
-
     promptManager.addStep({
       type: 'input',
       name: 'spkg',
@@ -731,7 +743,7 @@ async function processInitForm(
         isSubstreams ||
         !!initAbiPath ||
         isComposedSubgraph,
-      validate: async (value: string) => {
+      validate: async (value: string): Promise<string | boolean> => {
         if (
           initFromExample ||
           abiFromApi ||
@@ -822,6 +834,22 @@ async function processInitForm(
 
     await promptManager.executeInteractive();
 
+    // Validate network matches if loading from IPFS
+    if (ipfsNode && source?.startsWith('Qm')) {
+      const ipfs = createIpfsClient(ipfsNode);
+      try {
+        const { valid, error } = await validateSubgraphNetworkMatch(ipfs, source!, network.id);
+        if (!valid) {
+          throw new Error(error || 'Invalid subgraph network match');
+        }
+      } catch (e) {
+        if (e instanceof Error) {
+          print.error(`Failed to validate subgraph network: ${e?.message}`);
+        }
+        throw e;
+      }
+    }
+
     return {
       abi: (abiFromApi || abiFromFile)!,
       protocolInstance,
@@ -1188,8 +1216,9 @@ async function initSubgraphFromContract(
   }
 
   if (
-    !protocolInstance.isComposedSubgraph() &&
+    !isComposedSubgraph &&
     protocolInstance.hasABIs() &&
+    abi && // Add check for abi existence
     (abiEvents(abi).size === 0 ||
       // @ts-expect-error TODO: the abiEvents result is expected to be a List, how's it an array?
       abiEvents(abi).length === 0)
@@ -1204,6 +1233,12 @@ async function initSubgraphFromContract(
     `Failed to create subgraph scaffold`,
     `Warnings while creating subgraph scaffold`,
     async spinner => {
+      initDebugger('Generating scaffold with ABI:', abi);
+      initDebugger('ABI data:', abi?.data);
+      if (abi) {
+        initDebugger('ABI events:', abiEvents(abi));
+      }
+
       const scaffold = await generateScaffold(
         {
           protocolInstance,

packages/cli/src/compiler/index.ts

@@ -508,6 +508,9 @@ export default class Compiler {
       `Failed to write compiled subgraph to ${displayDir}`,
       `Warnings while writing compiled subgraph to ${displayDir}`,
       async spinner => {
+        // Add debug log for initial subgraph state
+        compilerDebug('Initial subgraph state:', subgraph.toJS());
+
         // Copy schema and update its path
         subgraph = subgraph.updateIn(['schema', 'file'], schemaFile => {
           const schemaFilePath = path.resolve(this.sourceDir, schemaFile as string);
@@ -518,32 +521,49 @@ export default class Compiler {
           return path.relative(this.options.outputDir, targetFile);
         });
 
+        // Add debug log before processing data sources
+        compilerDebug('Processing dataSources:', subgraph.get('dataSources').toJS());
+
         // Copy data source files and update their paths
         subgraph = subgraph.update('dataSources', (dataSources: any[]) =>
           dataSources.map(dataSource => {
+            // Add debug log for each data source
+            compilerDebug('Processing dataSource:', dataSource.toJS());
+
             let updatedDataSource = dataSource;
 
             if (this.protocol.hasABIs()) {
-              updatedDataSource = updatedDataSource
-                // Write data source ABIs to the output directory
-                .updateIn(['mapping', 'abis'], (abis: any[]) =>
-                  abis.map((abi: any) =>
-                    abi.update('file', (abiFile: string) => {
-                      abiFile = path.resolve(this.sourceDir, abiFile);
-                      const abiData = this.ABI.load(abi.get('name'), abiFile);
-                      return path.relative(
-                        this.options.outputDir,
-                        this._writeSubgraphFile(
-                          abiFile,
-                          JSON.stringify(abiData.data.toJS(), null, 2),
-                          this.sourceDir,
-                          this.subgraphDir(this.options.outputDir, dataSource),
-                          spinner,
-                        ),
-                      );
-                    }),
-                  ),
+              // Add debug log for ABIs
+              compilerDebug(
+                'Processing ABIs for dataSource:',
+                dataSource.getIn(['mapping', 'abis'])?.toJS() || 'undefined',
+              );
+
+              updatedDataSource = updatedDataSource.updateIn(['mapping', 'abis'], (abis: any[]) => {
+                compilerDebug('ABIs value:', Array.isArray(abis) ? abis : 'undefined');
+
+                if (!abis) {
+                  compilerDebug('No ABIs found for dataSource');
+                  return immutable.List();
+                }
+
+                return abis.map((abi: any) =>
+                  abi.update('file', (abiFile: string) => {
+                    abiFile = path.resolve(this.sourceDir, abiFile);
+                    const abiData = this.ABI.load(abi.get('name'), abiFile);
+                    return path.relative(
+                      this.options.outputDir,
+                      this._writeSubgraphFile(
+                        abiFile,
+                        JSON.stringify(abiData.data.toJS(), null, 2),
+                        this.sourceDir,
+                        this.subgraphDir(this.options.outputDir, dataSource),
+                        spinner,
+                      ),
+                    );
+                  }),
                 );
+              });
             }
 
             if (protocol.name == 'substreams' || protocol.name == 'substreams/triggers') {

packages/cli/src/protocols/subgraph/manifest.graphql

@@ -63,6 +63,7 @@ type ContractABI {
 type EntityHandler {
   handler: String!
   entity: String!
+  calls: JSON
 }
 
 type Graft {

packages/cli/src/scaffold/index.ts

@@ -1,3 +1,4 @@
+import debugFactory from 'debug';
 import fs from 'fs-extra';
 import { strings } from 'gluegun';
 import prettier from 'prettier';
@@ -11,6 +12,8 @@ import { generateEventIndexingHandlers } from './mapping.js';
 import { abiEvents, generateEventType, generateExampleEntityType } from './schema.js';
 import { generateTestsFiles } from './tests.js';
 
+const scaffoldDebugger = debugFactory('graph-cli:scaffold');
+
 const GRAPH_CLI_VERSION = process.env.GRAPH_CLI_TESTS
   ? // JSON.stringify should remove this key, we will install the local
     // graph-cli for the tests using `npm link` instead of fetching from npm.
@@ -47,18 +50,34 @@ export default class Scaffold {
   spkgPath?: string;
   entities?: string[];
 
-  constructor(options: ScaffoldOptions) {
-    this.protocol = options.protocol;
-    this.abi = options.abi;
-    this.indexEvents = options.indexEvents;
-    this.contract = options.contract;
-    this.network = options.network;
-    this.contractName = options.contractName;
-    this.subgraphName = options.subgraphName;
-    this.startBlock = options.startBlock;
-    this.node = options.node;
-    this.spkgPath = options.spkgPath;
-    this.entities = options.entities;
+  constructor({
+    protocol,
+    abi,
+    contract,
+    network,
+    contractName,
+    startBlock,
+    subgraphName,
+    node,
+    spkgPath,
+    indexEvents,
+    entities,
+  }: ScaffoldOptions) {
+    this.protocol = protocol;
+    this.abi = abi;
+    this.contract = contract;
+    this.network = network;
+    this.contractName = contractName;
+    this.startBlock = startBlock;
+    this.subgraphName = subgraphName;
+    this.node = node;
+    this.spkgPath = spkgPath;
+    this.indexEvents = indexEvents;
+    this.entities = entities;
+
+    scaffoldDebugger('Scaffold constructor called with ABI:', abi);
+    scaffoldDebugger('ABI data:', abi?.data);
+    scaffoldDebugger('ABI file:', abi?.file);
   }
 
   async generatePackageJson() {
@@ -203,9 +222,24 @@ dataSources:
   }
 
   async generateABIs() {
+    scaffoldDebugger('Generating ABIs...');
+    scaffoldDebugger('Protocol has ABIs:', this.protocol.hasABIs());
+    scaffoldDebugger('ABI data:', this.abi?.data);
+    scaffoldDebugger('ABI file:', this.abi?.file);
+
+    if (!this.protocol.hasABIs()) {
+      scaffoldDebugger('Protocol does not have ABIs, skipping ABI generation');
+      return;
+    }
+
+    if (!this.abi?.data) {
+      scaffoldDebugger('ABI data is undefined, skipping ABI generation');
+      return;
+    }
+
     return this.protocol.hasABIs()
       ? {
-          [`${this.contractName}.json`]: await prettier.format(JSON.stringify(this.abi?.data), {
+          [`${this.contractName}.json`]: await prettier.format(JSON.stringify(this.abi.data), {
             parser: 'json',
           }),
         }

packages/cli/src/utils.ts

@@ -32,3 +32,69 @@ export async function loadSubgraphSchemaFromIPFS(ipfsClient: any, manifest: stri
     throw Error(`Failed to load schema from IPFS ${manifest}`);
   }
 }
+
+export async function loadManifestFromIPFS(ipfsClient: any, manifest: string) {
+  try {
+    const manifestBuffer = ipfsClient.cat(manifest);
+    let manifestFile = '';
+    for await (const chunk of manifestBuffer) {
+      manifestFile += Buffer.from(chunk).toString('utf8');
+    }
+    return manifestFile;
+  } catch (e) {
+    utilsDebug.extend('loadManifestFromIPFS')(`Failed to load manifest from IPFS ${manifest}`);
+    utilsDebug.extend('loadManifestFromIPFS')(e);
+    throw Error(`Failed to load manifest from IPFS ${manifest}`);
+  }
+}
+
+/**
+ * Validates that the network of a source subgraph matches the target network
+ * @param ipfsClient IPFS client instance
+ * @param sourceSubgraphId IPFS hash of the source subgraph
+ * @param targetNetwork Network of the target subgraph being created
+ * @returns Object containing validation result and error message if any
+ */
+export async function validateSubgraphNetworkMatch(
+  ipfsClient: any,
+  sourceSubgraphId: string,
+  targetNetwork: string,
+): Promise<{ valid: boolean; error?: string }> {
+  try {
+    const manifestFile = await loadManifestFromIPFS(ipfsClient, sourceSubgraphId);
+    const manifestYaml = yaml.load(manifestFile) as any;
+
+    // Extract network from data sources
+    const dataSources = manifestYaml.dataSources || [];
+    const templates = manifestYaml.templates || [];
+    const allSources = [...dataSources, ...templates];
+
+    if (allSources.length === 0) {
+      return { valid: true }; // No data sources to validate
+    }
+
+    // Get network from first data source
+    const sourceNetwork = allSources[0].network;
+    if (!sourceNetwork) {
+      return { valid: true }; // Network not specified in source, skip validation
+    }
+
+    const normalizedSourceNetwork = sourceNetwork.toLowerCase();
+    const normalizedTargetNetwork = targetNetwork.toLowerCase();
+
+    if (normalizedSourceNetwork !== normalizedTargetNetwork) {
+      return {
+        valid: false,
+        error: `Network mismatch: The source subgraph is indexing the '${sourceNetwork}' network, but you're creating a subgraph for '${targetNetwork}' network. When composing subgraphs, they must index the same network.`,
+      };
+    }
+
+    return { valid: true };
+  } catch (e) {
+    utilsDebug.extend('validateSubgraphNetworkMatch')(`Failed to validate network match: ${e}`);
+    return {
+      valid: false,
+      error: e instanceof Error ? e.message : 'Failed to validate subgraph network',
+    };
+  }
+}