change tokeniser to tiktoken

package.json

@@ -9,6 +9,7 @@
     "preview": "vite preview"
   },
   "dependencies": {
+    "@dqbd/tiktoken": "^1.0.2",
     "html2canvas": "^1.4.1",
     "i18next": "^22.4.11",
     "i18next-browser-languagedetector": "^7.0.1",
@@ -29,7 +30,6 @@
   },
   "devDependencies": {
     "@tailwindcss/typography": "^0.5.9",
-    "@types/dompurify": "^2.4.0",
     "@types/react": "^18.0.27",
     "@types/react-dom": "^18.0.10",
     "@types/react-scroll-to-bottom": "^4.2.0",
@@ -39,6 +39,8 @@
     "postcss": "^8.4.21",
     "tailwindcss": "^3.2.7",
     "typescript": "^4.9.3",
-    "vite": "^4.1.0"
+    "vite": "^4.1.0",
+    "vite-plugin-top-level-await": "^1.3.0",
+    "vite-plugin-wasm": "^3.2.2"
   }
 }

src/components/TokenCount/TokenCount.tsx

@@ -2,7 +2,7 @@ import React, { useEffect, useState } from 'react';
 import useStore from '@store/store';
 import { shallow } from 'zustand/shallow';
 
-import { countMessagesToken } from '@utils/messageUtils';
+import countTokens from '@utils/messageUtils';
 
 const TokenCount = React.memo(() => {
   const [tokenCount, setTokenCount] = useState<number>(0);
@@ -13,8 +13,14 @@ const TokenCount = React.memo(() => {
     shallow
   );
 
+  const model = useStore((state) =>
+    state.chats
+      ? state.chats[state.currentChatIndex].config.model
+      : 'gpt-3.5-turbo'
+  );
+
   useEffect(() => {
-    if (!generating) setTokenCount(countMessagesToken(messages));
+    if (!generating) setTokenCount(countTokens(messages, model));
   }, [messages, generating]);
 
   return (

src/constants/chat.ts

@@ -16,10 +16,10 @@ Current date: ${dateString}`;
 
 export const modelOptions: ModelOptions[] = [
   'gpt-3.5-turbo',
-  // 'gpt-3.5-turbo-0301',
   'gpt-4',
-  // 'gpt-4-0314',
   'gpt-4-32k',
+  // 'gpt-3.5-turbo-0301',
+  // 'gpt-4-0314',
   // 'gpt-4-32k-0314',
 ];
 

src/hooks/useSubmit.ts

@@ -55,9 +55,13 @@ const useSubmit = () => {
 
     try {
       let stream;
+      if (chats[currentChatIndex].messages.length === 0)
+        throw new Error('No messages submitted!');
+
       const messages = limitMessageTokens(
         chats[currentChatIndex].messages,
-        chats[currentChatIndex].config.max_tokens
+        chats[currentChatIndex].config.max_tokens,
+        chats[currentChatIndex].config.model
       );
       if (messages.length === 0) throw new Error('Message exceed max token!');
 

src/types/chat.ts

@@ -25,13 +25,10 @@ export interface ConfigInterface {
   frequency_penalty: number;
 }
 
-export type ModelOptions =
-  | 'gpt-4'
-  | 'gpt-4-0314'
-  | 'gpt-4-32k'
-  | 'gpt-4-32k-0314'
-  | 'gpt-3.5-turbo'
-  | 'gpt-3.5-turbo-0301';
+export type ModelOptions = 'gpt-4' | 'gpt-4-32k' | 'gpt-3.5-turbo';
+// | 'gpt-3.5-turbo-0301';
+// | 'gpt-4-0314'
+// | 'gpt-4-32k-0314'
 
 export interface LocalStorageInterfaceV0ToV1 {
   chats: ChatInterface[];