explorer: add sqlite format for answer API response (#1685)

packages/api-server/package.json

@@ -60,7 +60,9 @@
     "pino-pretty": "^9.1.1",
     "pino-sentry-transport": "^1.0.4",
     "prom-client": "^14.1.0",
+    "promised-sqlite3": "^2.1.0",
     "socket.io": "^4.5.3",
+    "sqlite3": "^5.1.6",
     "yaml": "^2.1.3"
   },
   "devDependencies": {

packages/api-server/src/routes/explorer/answer/index.ts

@@ -1,8 +1,11 @@
 import {FastifyPluginAsyncJsonSchemaToTs} from "@fastify/type-provider-json-schema-to-ts";
+import fs, {mkdirSync} from "fs";
 import {DateTime} from "luxon";
+import path from "path";
 import {Question, QuestionStatus} from "../../../plugins/services/explorer-service/types";
 import {APIError} from "../../../utils/error";
 import sleep from "../../../utils/sleep";
+import {saveQueryResultToSqlite} from "../../../utils/sqlite";
 
 const schema = {
   summary: 'Get answer of a new question',
@@ -12,13 +15,23 @@ const schema = {
     type: 'object',
     properties: {
       question: { type: 'string' },
+      format: {
+        type: 'string',
+        enum: ['json', 'sqlite'],
+      },
       ignoreCache: { type: 'boolean', default: false }
     }
   } as const
 };
 
+export enum AnswerFormat {
+  JSON = 'json',
+  SQLITE = 'sqlite',
+}
+
 export interface IBody {
   question: string;
+  format: AnswerFormat;
   ignoreCache: boolean;
 }
 
@@ -28,7 +41,7 @@ export const newQuestionHandler: FastifyPluginAsyncJsonSchemaToTs = async (app):
   }>('/', {
     schema
   }, async function (req, reply) {
-    const { question: questionTitle, ignoreCache } = req.body;
+    const { question: questionTitle, ignoreCache, format = AnswerFormat.JSON } = req.body;
 
     // Create question.
     const question = await app.explorerService.newQuestion(0, questionTitle, ignoreCache, true, false, null, 10);
@@ -69,11 +82,28 @@ export const newQuestionHandler: FastifyPluginAsyncJsonSchemaToTs = async (app):
       throw new APIError(500, 'The question has been canceled.', undefined, convertQuestionToPayload(resolvedQuestion));
     }
 
-    reply.status(200).send(convertQuestionToPayload(resolvedQuestion));
+    if (format === AnswerFormat.SQLITE) {
+      try {
+        const dir = path.resolve(process.cwd(), '../../temp');
+        const dbName = `answer-${question.id}.db`;
+        const dbFile = path.resolve(dir, dbName) ;
+        mkdirSync(dir, { recursive: true });
+
+        await saveQueryResultToSqlite(resolvedQuestion.result!, dbFile);
+        const buffer = fs.readFileSync(dbFile);
+
+        reply.type('application/octet-stream');
+        reply.header('Content-Disposition', `attachment; filename="${dbName}"`);
+        reply.send(buffer)
+      } catch (err: any) {
+        throw new APIError(500, `Failed to save query result to sqlite file: ${err.message}`, err, convertQuestionToPayload(resolvedQuestion));
+      }
+    } else {
+      reply.status(200).send(convertQuestionToPayload(resolvedQuestion));
+    }
   });
 }
 
-
 export function convertQuestionToPayload(question: Question) {
   return {
     question: {
@@ -93,4 +123,5 @@ export function convertQuestionToPayload(question: Question) {
   };
 }
 
+
 export default newQuestionHandler;

packages/api-server/src/utils/sqlite.ts

@@ -0,0 +1,66 @@
+import { AsyncDatabase } from "promised-sqlite3";
+import {QuestionSQLResult} from "../plugins/services/explorer-service/types";
+
+const typeMaps: Record<number, string> = {
+  0: "DECIMAL",
+  1: "TINY",
+  2: "SHORT",
+  3: "LONG",
+  4: "FLOAT",
+  5: "DOUBLE",
+  6: "NULL",
+  7: "TIMESTAMP",
+  8: "LONGLONG",
+  9: "INT24",
+  10: "DATE",
+  11: "TIME",
+  12: "DATETIME",
+  13: "YEAR",
+  14: "NEWDATE",
+  15: "VARCHAR",
+  16: "BIT",
+  245: "JSON",
+  246: "NEWDECIMAL",
+  247: "ENUM",
+  248: "SET",
+  249: "TINY_BLOB",
+  250: "MEDIUM_BLOB",
+  251: "LONG_BLOB",
+  252: "BLOB",
+  253: "VAR_STRING",
+  254: "STRING",
+  255: "GEOMETRY"
+};
+
+export async function saveQueryResultToSqlite(queryResult: QuestionSQLResult, dbFile: string) {
+  const { fields = [], rows = [] } = queryResult;
+  const db = await AsyncDatabase.open(dbFile);
+  await db.run('DROP TABLE IF EXISTS result;');
+  await db.run(`
+    CREATE TABLE IF NOT EXISTS result (
+      ${fields.map(field => `${field.name} ${mapMySQLTypeID2SqliteType(field.columnType)}`).join(',')}
+    );
+  `);
+
+  const columns = fields.map(field => field.name);
+  const stmt = await db.prepare(`
+    INSERT INTO result (${columns.join(', ')}) VALUES (
+        ${columns.map(_ => '?').join(', ')}
+    );
+  `);
+  for (const row of rows) {
+    const values = columns.map(column => row[column]);
+    await stmt.run(...values);
+  }
+  await stmt.finalize();
+  await db.close();
+}
+
+function mapMySQLTypeID2SqliteType(columnType: number): string {
+  const type = typeMaps[columnType] as any;
+  if (type === undefined) {
+    return 'VARCHAR(255)';
+  } else {
+    return type;
+  }
+}