Initial support for files table in Dexie and DuckDB (#789)

* Initial support for files table in Dexie and DuckDB

* Remove expires from files

* Clean-up file import logic

* /duck should show duckdb and dexie files

src/hooks/use-file-import.tsx

@@ -5,6 +5,7 @@ import { ChatCraftHumanMessage } from "../lib/ChatCraftMessage";
 import { compressImageToBase64, formatAsCodeBlock } from "../lib/utils";
 import { getSettings } from "../lib/settings";
 import { JinaAIProvider, type JinaAiReaderResponse } from "../lib/providers/JinaAIProvider";
+import { ChatCraftFile } from "../lib/ChatCraftFile";
 
 function readTextFile(file: File) {
   return new Promise<string>((resolve, reject) => {
@@ -180,21 +181,25 @@ export function useFileImport({ chat, onImageImport }: UseFileImportOptions) {
   const settings = getSettings();
 
   const importFile = useCallback(
-    (file: File, contents: string | JinaAiReaderResponse) => {
-      if (file.type.startsWith("image/")) {
-        const base64 = contents as string;
-        onImageImport(base64);
-      } else if (file.type === "application/pdf") {
-        const document = (contents as JinaAiReaderResponse).data;
-        chat.addMessage(new ChatCraftHumanMessage({ text: `${document.content}\n` }));
-      } else if (
-        file.type === "application/vnd.openxmlformats-officedocument.wordprocessingml.document"
-      ) {
-        const document = contents as string;
-        chat.addMessage(new ChatCraftHumanMessage({ text: `${document}\n` }));
-      } else {
-        const document = contents as string;
-        chat.addMessage(new ChatCraftHumanMessage({ text: `${document}\n` }));
+    async (file: File, contents: string | JinaAiReaderResponse) => {
+      const isImage = file.type.startsWith("image/");
+      const isPDF = file.type === "application/pdf";
+      const isWordDoc =
+        file.type === "application/vnd.openxmlformats-officedocument.wordprocessingml.document";
+
+      // Extract text based on file type
+      const text = isPDF ? (contents as JinaAiReaderResponse).data.content : (contents as string);
+
+      // Create or find the ChatCraftFile in the files table and add to the chat
+      const chatCraftFile = await ChatCraftFile.findOrCreate(file, { text });
+      await chat.addFile(chatCraftFile);
+
+      // Add the file's contents to the chat as a message
+      if (isImage) {
+        onImageImport(text);
+      } else if (isPDF || isWordDoc || !isImage) {
+        // Add the content as a human message for non-image files
+        await chat.addMessage(new ChatCraftHumanMessage({ text: `${text}\n` }));
       }
     },
     [chat, onImageImport]

src/lib/ChatCraftChat.ts

@@ -9,13 +9,14 @@ import {
   ChatCraftSystemMessage,
   type SerializedChatCraftMessage,
 } from "./ChatCraftMessage";
-import db, { type ChatCraftChatTable, type ChatCraftMessageTable } from "./db";
+import db, { ChatCraftFileTable, type ChatCraftChatTable, type ChatCraftMessageTable } from "./db";
 import summarize from "./summarize";
 import { createSystemMessage } from "./system-prompt";
 import { createDataShareUrl, createShare } from "./share";
 import { SharedChatCraftChat } from "./SharedChatCraftChat";
 import { countTokensInMessages } from "./ai";
 import { parseFunctionNames, loadFunctions } from "./ChatCraftFunction";
+import { ChatCraftFile } from "./ChatCraftFile";
 
 export type SerializedChatCraftChat = {
   id: string;
@@ -44,23 +45,27 @@ export class ChatCraftChat {
   date: Date;
   private _summary?: string;
   private _messages: ChatCraftMessage[];
+  private _files?: ChatCraftFile[];
   readonly: boolean;
 
   constructor({
     id,
     date,
     summary,
     messages,
+    files,
     readonly,
   }: {
     id?: string;
     date?: Date;
     summary?: string;
     messages?: ChatCraftMessage[];
+    files?: ChatCraftFile[];
     readonly?: boolean;
   } = {}) {
     this.id = id ?? nanoid();
     this._messages = messages ?? [createSystemMessage()];
+    this._files = files;
     this.date = date ?? new Date();
     // If the user provides a summary, use it, otherwise we'll generate something
     this._summary = summary;
@@ -114,6 +119,10 @@ export class ChatCraftChat {
     });
   }
 
+  files() {
+    return this._files || [];
+  }
+
   // Get a list of functions mentioned via @fn or fn-url from db or remote servers
   async functions(onError?: (err: Error) => void) {
     // We scan the entire set of human and system messages in the chat for functions
@@ -167,6 +176,27 @@ export class ChatCraftChat {
     return this.save();
   }
 
+  async addFile(file: ChatCraftFile) {
+    if (this.readonly) {
+      return;
+    }
+
+    // Filter out any existing file with the same ID, then add the new one
+    this._files = [...this.files().filter((f) => f.id !== file.id), file];
+    return this.save();
+  }
+
+  async removeFile(id: string) {
+    if (this.readonly) {
+      return;
+    }
+
+    if (this._files) {
+      this._files = this._files.filter((file) => file.id !== id);
+      return this.save();
+    }
+  }
+
   // Remove all messages in the chat *before* the message with the given id,
   // keeping only the system message.
   async removeMessagesBefore(id: string) {
@@ -250,8 +280,14 @@ export class ChatCraftChat {
     // Rehydrate the messages from their IDs
     const messages = await db.messages.bulkGet(chat.messageIds);
 
-    // Return a new ChatCraftChat object for this chat/messages, skipping any
-    // that were not found (e.g., user deleted)
+    // Rehydrate the files from their IDs
+    if (chat.fileIds) {
+      const files = await db.files.bulkGet(chat.fileIds);
+      if (files) {
+        return ChatCraftChat.fromDB(chat, messages, files);
+      }
+    }
+
     return ChatCraftChat.fromDB(chat, messages);
   }
 
@@ -318,6 +354,7 @@ export class ChatCraftChat {
       messages: this.messages({ includeAppMessages: false, includeSystemMessages: true }).map(
         (message) => message.toJSON()
       ),
+      // We don't attempt to serialize files in JSON
     };
   }
 
@@ -332,6 +369,8 @@ export class ChatCraftChat {
       summary: this.summary,
       // In the DB, we store the app messages, since that's what we show in the UI
       messageIds: this._messages.map(({ id }) => id),
+      // In the DB, we store the files associated with a chat
+      fileIds: this._files?.map(({ id }) => id),
     };
   }
 
@@ -419,6 +458,7 @@ export class ChatCraftChat {
       messages: messages.map((message) => ChatCraftMessage.fromJSON(message)),
       // We can't modify a chat loaded outside the db
       readonly: true,
+      // We don't attempt to deserialize files in JSON
     });
   }
 
@@ -429,12 +469,19 @@ export class ChatCraftChat {
   // Parse from db representation, where chat and messages are separate.
   // Assumes all messages have already been obtained for messageIds, but
   // deals with any that are missing (undefined)
-  static fromDB(chat: ChatCraftChatTable, messages: (ChatCraftMessageTable | undefined)[]) {
+  static fromDB(
+    chat: ChatCraftChatTable,
+    messages: (ChatCraftMessageTable | undefined)[],
+    files?: (ChatCraftFileTable | undefined)[]
+  ) {
     return new ChatCraftChat({
       ...chat,
       messages: messages
         .filter((message): message is ChatCraftMessageTable => !!message)
         .map((message) => ChatCraftMessage.fromDB(message)),
+      files: files
+        ?.filter((file): file is ChatCraftFileTable => !!file)
+        .map((file) => ChatCraftFile.fromDB(file)),
     });
   }
 }