Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: validate subgraph fetches #124

Draft
wants to merge 1 commit into
base: main
Choose a base branch
from
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
89 changes: 83 additions & 6 deletions src/commands/audit.js
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import { diff } from 'jest-diff';
import { serializeQueryPlan as serializeQueryPlan1 } from '@apollo/query-planner-1';
import { serializeQueryPlan as serializeQueryPlan2 } from '@apollo/query-planner';
import ora from 'ora';
import { buildSubgraphSchema } from '@apollo/subgraph';
import {
chooseVariant,
getConfig,
Expand Down Expand Up @@ -144,11 +145,50 @@ function generateReport(result, graphRef) {
'='.repeat(title.length),
`https://studio.apollographql.com/graph/${graph}/operations?query=${result.queryId}&queryName=${result.queryName}&variant=${variant}`,
'',

result.twoFromOneFetchErrors.length
? '💣 Federation 1 supergraph query plan has subgraph fetch nodes that will fail'
: '🎉 Federation 1 supergraph query plan is valid',
result.twoFetchErrors.length
? '💣 Federation 2 supergraph query plan has subgraph fetch nodes that will fail'
: '🎉 Federation 2 supergraph query plan is valid',
result.queryPlansMatch
? '🎉 No difference in query plans'
: '💣 Query plans differ',
: '👀 Query plans differ',

'',
...(result.twoFromOneFetchErrors.length
? [
'Subgraph fetch errors (Federation 1 supergraph)',
'-----------------------------------------------',
...result.twoFromOneFetchErrors.flatMap((err) => [
`* [${err.extensions.subgraph}] ${err.toString()}`,
' ```graphql',
...err.extensions.operation
.split('\n')
.map((line) => ` ${line}`),
' ```',
]),
]
: []),

'',
...(result.twoFetchErrors.length
? [
'Subgraph fetch errors (Federation 2 supergraph)',
'-----------------------------------------------',
...result.twoFetchErrors.flatMap((err) => [
`* [${err.extensions.subgraph}] ${err.toString()}`,
' ```graphql',
...err.extensions.operation
.split('\n')
.map((line) => ` ${line}`),
' ```',
]),
]
: []),

'',
...(!result.planner1MatchesPlanner2
? [
'Before and After Migration Diff',
Expand Down Expand Up @@ -308,9 +348,14 @@ export default class AuditCommand extends Command {
Performs a series of tasks on a federated graph:
1. Ensures that it composes using Federation 1.
2. Ensures that it composes using Federation 2.
3. Fetches recent operations from Apollo Studio.
4. Generates query plans for each operation using both the Federation 1
and Federation 2 schemas and compares the results.
3. Ensures that both supergraphs load correctly in Gateway 2.0
4. Fetches recent operations from Apollo Studio.
5. Generates query plans using:
1. The Federation 1 supergraph with Gateway 0.x.
2. The Federation 1 supergraph with Gateway 2.0.
3. The Federation 2 supergraph with Gateway 2.0.
6. Validates that fetch nodes in each 2.0 query plan (using supergraphs
from Fed 1 and Fed2) are valid against subgraphs.
`,
examples: [
['Get basic details on composition and query plans', '$0'],
Expand Down Expand Up @@ -454,10 +499,17 @@ export default class AuditCommand extends Command {
spinner.text = `Generating query plans for ${validOperations.length} operations`;
spinner.start();

const subgraphSchemas = new Map(
Object.entries(resolvedConfig.subgraphs).map(
([name, { schema: sdl }]) => [name, buildSubgraphSchema(parse(sdl))],
),
);

const results = await queryPlanAudit({
fed1Schema: fed1,
fed2Schema: fed2,
operations: validOperations,
subgraphSchemas,
});

spinner.stop();
Expand All @@ -467,13 +519,32 @@ export default class AuditCommand extends Command {
(r) => r.type === 'SUCCESS' && r.queryPlansMatch,
).length;

const twoFetchErrors = results.filter(
(r) => r.type === 'SUCCESS' && r.twoFetchErrors.length,
);
const twoFromOneFetchErrors = results.filter(
(r) => r.type === 'SUCCESS' && r.twoFromOneFetchErrors.length,
);

this.context.stdout.write('-----------------------------------\n');
this.context.stdout.write(`✅ Operations audited: ${total}\n`);
this.context.stdout.write(`🏆 Operations that match: ${matched}\n`);

if (matched < total) {
this.context.stdout.write(
`❌ Operations with differences: ${total - matched}\n`,
`👀 Operations with differences: ${total - matched}\n`,
);
}

if (twoFromOneFetchErrors.length) {
this.context.stdout.write(
`❌ Query plans with invalid subgraph fetches (Fed 1 Supergraph): ${twoFromOneFetchErrors.length}\n`,
);
}

if (twoFetchErrors.length) {
this.context.stdout.write(
`❌ Query plans with invalid subgraph fetches (Fed 2 Supergraph): ${twoFetchErrors.length}\n`,
);
}

Expand Down Expand Up @@ -514,7 +585,13 @@ export default class AuditCommand extends Command {
}
}

if (result.type === 'SUCCESS' && !result.queryPlansMatch) {
const successReport =
result.type === 'SUCCESS' &&
(!result.queryPlansMatch ||
result.twoFetchErrors.length ||
result.twoFromOneFetchErrors.length);

if (successReport) {
// eslint-disable-next-line no-await-in-loop
await writeFile(
path,
Expand Down
101 changes: 99 additions & 2 deletions src/federation/query-plan-audit.js
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import { GraphQLError, parse, validate, print } from 'graphql';
import { queryPlan as queryPlan1 } from './one.js';
import { queryPlan as queryPlan2, queryPlanWithFed1Schema } from './two.js';
import { diffQueryPlans } from './diff.js';
Expand All @@ -21,10 +22,98 @@ function assert(condition, message) {
*/

/**
* @param {{ fed1Schema: import('./one.js').CompositionResult; fed2Schema: import('@apollo/composition').CompositionSuccess; operations: Operation[] }} options
* @param {import('@apollo/query-planner').QueryPlan} queryPlan
*/
function allFetchNodes(queryPlan) {
/** @type {import('@apollo/query-planner').FetchNode[]} */
const fetchNodes = [];

/**
* @param {import('@apollo/query-planner').PlanNode} node
*/
function recurse(node) {
switch (node.kind) {
case 'Fetch':
fetchNodes.push(node);
break;
case 'Flatten':
recurse(node.node);
break;
case 'Parallel':
case 'Sequence':
node.nodes.map((n) => recurse(n));
node.nodes.map((n) => recurse(n));
break;
default:
throw new Error('not possible');
}
}

if (queryPlan.node) {
recurse(queryPlan.node);
}

return fetchNodes;
}

/**
* @param {import('@apollo/query-planner').FetchNode} fetchNode
* @param {Map<string, import("graphql").GraphQLSchema>} subgraphSchemas
*/
function validateFetchNode(fetchNode, subgraphSchemas) {
const subgraphSchema = subgraphSchemas.get(fetchNode.serviceName);
if (!subgraphSchema) {
return [
new GraphQLError(
`fetch node calls missing subgraph ${fetchNode.serviceName}`,
{
extensions: {
subgraph: fetchNode.serviceName,
},
},
),
];
}

const errors = validate(subgraphSchema, parse(fetchNode.operation));
return errors.map(
(err) =>
new GraphQLError(err.message, {
...err,
extensions: {
...err.extensions,
subgraph: fetchNode.serviceName,
operation: print(parse(fetchNode.operation)),
},
}),
);
}

/**
* @param {import('@apollo/query-planner').QueryPlan} queryPlan
* @param {Map<string, import("graphql").GraphQLSchema>} subgraphSchemas
*/
function validateQueryPlan(queryPlan, subgraphSchemas) {
return allFetchNodes(queryPlan).flatMap((fetchNode) =>
validateFetchNode(fetchNode, subgraphSchemas),
);
}

/**
* @param {{
* fed1Schema: import('./one.js').CompositionResult;
* fed2Schema: import('@apollo/composition').CompositionSuccess;
* operations: Operation[];
* subgraphSchemas: Map<string, import("graphql").GraphQLSchema>;
* }} options
* @returns {Promise<AuditResult[]>}
*/
export async function queryPlanAudit({ fed1Schema, fed2Schema, operations }) {
export async function queryPlanAudit({
fed1Schema,
fed2Schema,
operations,
subgraphSchemas,
}) {
return Promise.all(
operations.map(async (op) => {
assert(fed1Schema.schema, 'federation 1 composition unsuccessful');
Expand Down Expand Up @@ -80,6 +169,12 @@ export async function queryPlanAudit({ fed1Schema, fed2Schema, operations }) {
normalizedTwo,
);

const twoFetchErrors = validateQueryPlan(two, subgraphSchemas);
const twoFromOneFetchErrors = validateQueryPlan(
twoFromOne,
subgraphSchemas,
);

return {
type: 'SUCCESS',
queryPlansMatch:
Expand All @@ -92,6 +187,8 @@ export async function queryPlanAudit({ fed1Schema, fed2Schema, operations }) {
normalizedOne,
normalizedTwo,
normalizedTwoFromOne,
twoFetchErrors,
twoFromOneFetchErrors,
...op,
};
}
Expand Down
3 changes: 3 additions & 0 deletions src/typings.d.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import {
} from '@apollo/query-planner-1';
import { QueryPlan as QueryPlan_1 } from '@apollo/query-planner-1';
import { QueryPlan as QueryPlan_2 } from '@apollo/query-planner';
import { GraphQLError } from 'graphql';

interface Operation {
queryId: string;
Expand All @@ -34,6 +35,8 @@ type AuditResult =
queryId: string;
queryName: string | undefined;
querySignature: string;
twoFetchErrors: GraphQLError[];
twoFromOneFetchErrors: GraphQLError[];
}
| {
type: 'FAILURE';
Expand Down