AI Suggestions (#215)

* AI suggestions

* Fix scrolling UI and improve prompt

.vscode/settings.json

@@ -2,7 +2,7 @@
   "typescript.tsdk": "node_modules/typescript/lib",
   "eslint.format.enable": true,
   "editor.codeActionsOnSave": {
-    "source.fixAll.eslint": true
+    "source.fixAll.eslint": "explicit"
   },
   "[typescriptreact]": {
     "editor.defaultFormatter": "esbenp.prettier-vscode"

apps/api/src/routes/ai-fix.ts

@@ -0,0 +1,245 @@
+import { type FastifyPluginAsync } from "fastify";
+import { getStorageBucket } from "../services/storage.js";
+import path from "path";
+import { tmpdir } from "os";
+import { readFile } from "fs/promises";
+import { load } from "js-yaml";
+import { getOpenAIClient } from "../services/openai.js";
+import OpenAI from "openai";
+export class ReportGenerationError extends Error {}
+
+type Issue = {
+  code: string | number;
+  message: string;
+  severity: number;
+  path: (string | number)[];
+  range: {
+    start: {
+      line: number;
+      character: number;
+    };
+    end: {
+      line: number;
+      character: number;
+    };
+  };
+};
+
+async function getOpenAiResponse(
+  messages: OpenAI.Chat.Completions.ChatCompletionMessageParam[],
+): Promise<string | null> {
+  try {
+    const response = await getOpenAIClient()?.chat.completions.create({
+      model: "gpt-3.5-turbo-1106",
+      messages,
+      temperature: 0,
+      max_tokens: 1000,
+      top_p: 1,
+      frequency_penalty: 0,
+      presence_penalty: 0,
+    });
+    return response
+      ? response.choices[0].message.content
+      : "Placeholder OpenAI response";
+  } catch (err) {
+    throw new ReportGenerationError(`Could not get OpenAI response: ${err}`, {
+      cause: err,
+    });
+  }
+}
+
+/**
+ * Parse out JSON references from a JSON object
+ */
+function getAllReferences(issueProperty: Record<string, unknown>): string[] {
+  const regex = /"\$ref":\s*"(#[^"]+)"/g;
+  const issuePropertyString = JSON.stringify(issueProperty);
+  let matches: RegExpExecArray | null;
+  const extractedRefs: string[] = [];
+
+  while ((matches = regex.exec(issuePropertyString)) !== null) {
+    extractedRefs.push(matches[1]);
+  }
+  return extractedRefs;
+}
+
+/**
+ * Given a reference and a corpus, resolve it to a value
+ */
+function resolveAllReferences(
+  reference: string,
+  openApiSpec: unknown,
+): unknown {
+  let referenceValue = openApiSpec;
+  const references = reference.replace("#/", "").split("/");
+  for (const refFragment of references) {
+    if (referenceValue == null || typeof referenceValue !== "object") {
+      break;
+    }
+    referenceValue = (referenceValue as Record<string, unknown>)[
+      refFragment as string
+    ];
+  }
+  return { reference, referenceValue };
+}
+
+export const aiFixRoute: FastifyPluginAsync = async function (server) {
+  server.route({
+    method: "POST",
+    schema: {
+      params: {
+        type: "object",
+        required: ["reportName"],
+        properties: {
+          reportId: { type: "string" },
+        },
+      },
+      body: {
+        type: "object",
+        properties: {
+          issue: {
+            type: "object",
+            properties: {
+              code: { type: "string" },
+              path: { type: "array", items: { type: "string" } },
+              severity: { type: "number" },
+              source: { type: "string" },
+              range: {
+                type: "object",
+                properties: {
+                  start: {
+                    type: "object",
+                    properties: {
+                      line: { type: "number" },
+                      character: { type: "number" },
+                    },
+                  },
+                  end: {
+                    type: "object",
+                    properties: {
+                      line: { type: "number" },
+                      character: { type: "number" },
+                    },
+                  },
+                },
+              },
+            },
+          },
+        },
+      },
+      response: {
+        200: {
+          type: "string",
+        },
+      },
+    },
+    url: "/ai-fix/:reportName",
+    handler: async (request) => {
+      const { reportName } = request.params as {
+        reportName: string;
+        operationId: string;
+      };
+      const issue = (request.body as { issue: Issue }).issue;
+
+      // Grab the OpenAPI file from storage
+      const tempPath = path.join(tmpdir(), reportName);
+      try {
+        await getStorageBucket().file(reportName).download({
+          destination: tempPath,
+        });
+      } catch (err) {
+        throw new ReportGenerationError(
+          `Could not download file from storage`,
+          {
+            cause: err,
+          },
+        );
+      }
+      const rawContent = await readFile(tempPath, "utf-8");
+      let openApiSpec: unknown;
+
+      // We need to know the file extension to parse the file correctly
+      const isJson = reportName.endsWith(".json");
+      if (!isJson) {
+        try {
+          // It's really difficult to operate on a YAML file as a string, so we
+          // parse it into a JSON object
+          openApiSpec = load(rawContent);
+        } catch (err) {
+          throw new ReportGenerationError(`Could not parse file from storage`, {
+            cause: err,
+          });
+        }
+      } else {
+        try {
+          openApiSpec = JSON.parse(rawContent);
+        } catch (err) {
+          throw new ReportGenerationError(`Could not parse file from storage`, {
+            cause: err,
+          });
+        }
+      }
+
+      const { path: issuePath } = issue;
+      // We want to get the property that the issue is referring to
+      let issueProperty = openApiSpec;
+      for (let i = 0; i < issuePath.length; i++) {
+        const pathFragment = issuePath[i];
+        if (issueProperty == null || typeof issueProperty !== "object") {
+          break;
+        }
+        issueProperty = (issueProperty as Record<string, unknown>)[
+          pathFragment as string
+        ];
+        if (i >= 1 && issuePath[i - 1] === "paths") {
+          // We want the the AI to have the full context of the path
+          issueProperty = {
+            [pathFragment]: issueProperty,
+          };
+          break;
+        }
+        if (i >= 2 && issuePath[i - 2] === "components") {
+          // We preserve the component name as it is relevant for certain rules
+          issueProperty = {
+            [pathFragment]: issueProperty,
+          };
+          break;
+        }
+      }
+
+      // We parse out the values of all references to give the AI more context
+      const references = getAllReferences(
+        issueProperty as Record<string, unknown>,
+      )
+        .map((ref) => resolveAllReferences(ref, openApiSpec))
+        .map((reference) => JSON.stringify(reference, null, 2))
+        .join(", ");
+
+      const prompt = `Given the following OpenAPI spec sample, and an issue found with that sample, please provide a suggested fix for the issue. The Sample: ${JSON.stringify(
+        issueProperty,
+        null,
+        2,
+      )}\n\n The Issue: ${JSON.stringify(
+        issue,
+        null,
+        2,
+      )}. If your suggestion is a change to the OpenAPI spec, the suggestion should be in ${
+        isJson ? "JSON" : "YAML"
+      } format. Leave inline comments explaining the changes you made. Any code blocks should use the markdown codeblock syntax. If the issue has to do with a component being orphaned, you should suggest deleting that component from the spec. ${
+        references
+          ? `Here are some components that are referenced within the OpenAPI spec sample: ${references}`
+          : ""
+      } 
+      `;
+
+      const response = await getOpenAiResponse([
+        {
+          role: "user",
+          content: prompt,
+        },
+      ]);
+
+      return response;
+    },
+  });
+};

apps/api/src/server.ts

@@ -22,6 +22,7 @@ import { fileRoute } from "./routes/file.js";
 import healthRoute from "./routes/health.js";
 import { reportRoute } from "./routes/report.js";
 import uploadRoute from "./routes/upload.js";
+import { aiFixRoute } from "./routes/ai-fix.js";
 
 const fastify = Fastify({
   logger: createNewLogger(),
@@ -45,6 +46,7 @@ async function build() {
   await fastify.register(uploadRoute);
   await fastify.register(reportRoute);
   await fastify.register(fileRoute);
+  await fastify.register(aiFixRoute);
 }
 
 const start = async () => {

apps/web/package.json

@@ -14,6 +14,7 @@
     "@monaco-editor/react": "^4.6.0",
     "@rate-my-openapi/core": "*",
     "@sentry/nextjs": "^7.81.0",
+    "@tailwindcss/typography": "^0.5.10",
     "@typeform/embed-react": "^3.8.0",
     "@vercel/og": "^0.5.20",
     "classnames": "^2.3.2",
@@ -23,6 +24,7 @@
     "react": "18.2.0",
     "react-dom": "18.2.0",
     "react-intersection-observer": "^9.5.3",
+    "react-markdown": "^9.0.1",
     "react-modal-hook": "^3.0.2",
     "typescript": "5.2.2"
   },

apps/web/src/app/report/[id]/full-report.tsx

@@ -92,6 +92,7 @@ export const FullReport = ({
             score={report?.docsScore}
             issues={report?.docsIssues}
             openapi={openapi}
+            reportName={`${reportId}.${fileExtension}`}
             fileExtension={fileExtension}
           />
         ) : (
@@ -103,6 +104,7 @@ export const FullReport = ({
             score={report?.completenessScore}
             issues={report?.completenessIssues}
             openapi={openapi}
+            reportName={`${reportId}.${fileExtension}`}
             fileExtension={fileExtension}
           />
         )}
@@ -112,6 +114,7 @@ export const FullReport = ({
             score={report?.sdkGenerationScore}
             issues={report?.sdkGenerationIssues}
             openapi={openapi}
+            reportName={`${reportId}.${fileExtension}`}
             fileExtension={fileExtension}
           />
         )}
@@ -121,6 +124,7 @@ export const FullReport = ({
             score={report?.securityScore}
             issues={report?.securityIssues}
             openapi={openapi}
+            reportName={`${reportId}.${fileExtension}`}
             fileExtension={fileExtension}
           />
         )}

apps/web/src/components/DetailedScoreSection/index.tsx

@@ -12,6 +12,7 @@ export type Issue = {
   code: string | number;
   message: string;
   severity: number;
+  path: (string | number)[];
   range: {
     start: {
       line: number;
@@ -46,12 +47,14 @@ const DetailedScoreSection = ({
   title,
   score,
   issues,
+  reportName,
   openapi,
   fileExtension,
 }: {
   title: string;
   score: number;
   issues: Issue[];
+  reportName: string;
   openapi: string;
   fileExtension: "json" | "yaml";
 }) => {
@@ -75,6 +78,7 @@ const DetailedScoreSection = ({
         onClose={() => {
           hideModal();
         }}
+        reportName={reportName}
         issue={issueToView!}
       />
     );