diff --git a/package.json b/package.json index 6a00982..f747c67 100644 --- a/package.json +++ b/package.json @@ -17,6 +17,8 @@ "@sveltejs/adapter-node": "^2.0.0", "@sveltejs/kit": "^2.0.0", "@sveltejs/vite-plugin-svelte": "^3.0.0", + "@types/markdown-it": "^13.0.7", + "@types/sanitize-html": "^2.9.5", "@typescript-eslint/eslint-plugin": "^6.15.0", "@typescript-eslint/parser": "^6.15.0", "autoprefixer": "^10.4.16", @@ -40,6 +42,8 @@ "@steeze-ui/icons": "^2.1.0", "@steeze-ui/svelte-icon": "^1.5.0", "langchain": "^0.0.198", + "markdown-it": "^14.0.0", + "sanitize-html": "^2.11.0", "tiktoken": "^1.0.11" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 885cfb5..58e5bf1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -20,6 +20,12 @@ dependencies: langchain: specifier: ^0.0.198 version: 0.0.198(@qdrant/js-client-rest@1.7.0) + markdown-it: + specifier: ^14.0.0 + version: 14.0.0 + sanitize-html: + specifier: ^2.11.0 + version: 2.11.0 tiktoken: specifier: ^1.0.11 version: 1.0.11 @@ -37,6 +43,12 @@ devDependencies: '@sveltejs/vite-plugin-svelte': specifier: ^3.0.0 version: 3.0.1(svelte@4.2.8)(vite@5.0.10) + '@types/markdown-it': + specifier: ^13.0.7 + version: 13.0.7 + '@types/sanitize-html': + specifier: ^2.9.5 + version: 2.9.5 '@typescript-eslint/eslint-plugin': specifier: ^6.15.0 version: 6.15.0(@typescript-eslint/parser@6.15.0)(eslint@8.56.0)(typescript@5.3.3) @@ -754,6 +766,21 @@ packages: resolution: {integrity: sha512-5+fP8P8MFNC+AyZCDxrB2pkZFPGzqQWUzpSeuuVLvm8VMcorNYavBqoFcxK8bQz4Qsbn4oUEEem4wDLfcysGHA==} dev: true + /@types/linkify-it@3.0.5: + resolution: {integrity: sha512-yg6E+u0/+Zjva+buc3EIb+29XEg4wltq7cSmd4Uc2EE/1nUVmxyzpX6gUXD0V8jIrG0r7YeOGVIbYRkxeooCtw==} + dev: true + + /@types/markdown-it@13.0.7: + resolution: {integrity: sha512-U/CBi2YUUcTHBt5tjO2r5QV/x0Po6nsYwQU4Y04fBS6vfoImaiZ6f8bi3CjTCxBPQSO1LMyUqkByzi8AidyxfA==} + dependencies: + '@types/linkify-it': 3.0.5 + '@types/mdurl': 1.0.5 + dev: true + + /@types/mdurl@1.0.5: + resolution: {integrity: sha512-6L6VymKTzYSrEf4Nev4Xa1LCHKrlTlYCBMTlQKFuddo1CvQcE52I0mwfOJayueUC7MJuXOeHTcIU683lzd0cUA==} + dev: true + /@types/node-fetch@2.6.9: resolution: {integrity: sha512-bQVlnMLFJ2d35DkPNjEPmd9ueO/rh5EiaZt2bhqiSarPjZIuIV6bPQVqcrEyvNo+AfTrRGVazle1tl597w3gfA==} dependencies: @@ -779,6 +806,12 @@ packages: resolution: {integrity: sha512-wWKOClTTiizcZhXnPY4wikVAwmdYHp8q6DmC+EJUzAMsycb7HB32Kh9RN4+0gExjmPmZSAQjgURXIGATPegAvA==} dev: false + /@types/sanitize-html@2.9.5: + resolution: {integrity: sha512-2Sr1vd8Dw+ypsg/oDDfZ57OMSG2Befs+l2CMyCC5bVSK3CpE7lTB2aNlbbWzazgVA+Qqfuholwom6x/mWd1qmw==} + dependencies: + htmlparser2: 8.0.2 + dev: true + /@types/semver@7.5.6: resolution: {integrity: sha512-dn1l8LaMea/IjDoHNd9J52uBbInB796CDffS6VdIxvqYCPSG0V0DzHp76GpaWnlhg88uYyPbXCDIowa86ybd5A==} dev: true @@ -1235,7 +1268,6 @@ packages: /deepmerge@4.3.1: resolution: {integrity: sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==} engines: {node: '>=0.10.0'} - dev: true /delayed-stream@1.0.0: resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} @@ -1284,10 +1316,37 @@ packages: esutils: 2.0.3 dev: true + /dom-serializer@2.0.0: + resolution: {integrity: sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==} + dependencies: + domelementtype: 2.3.0 + domhandler: 5.0.3 + entities: 4.5.0 + + /domelementtype@2.3.0: + resolution: {integrity: sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==} + + /domhandler@5.0.3: + resolution: {integrity: sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==} + engines: {node: '>= 4'} + dependencies: + domelementtype: 2.3.0 + + /domutils@3.1.0: + resolution: {integrity: sha512-H78uMmQtI2AhgDJjWeQmHwJJ2bLPD3GMmO7Zja/ZZh84wkm+4ut+IUnUdRa8uCGX88DiVx1j6FRe1XfxEgjEZA==} + dependencies: + dom-serializer: 2.0.0 + domelementtype: 2.3.0 + domhandler: 5.0.3 + /electron-to-chromium@1.4.615: resolution: {integrity: sha512-/bKPPcgZVUziECqDc+0HkT87+0zhaWSZHNXqF8FLd2lQcptpmUFwoCSWjCdOng9Gdq+afKArPdEg/0ZW461Eng==} dev: true + /entities@4.5.0: + resolution: {integrity: sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==} + engines: {node: '>=0.12'} + /es6-promise@3.3.1: resolution: {integrity: sha512-SOp9Phqvqn7jtEUxPWdWfWoLmyt2VaJ6MpvP9Comy1MceMXqE6bxvaTu4iaxpYYPzhny28Lc+M87/c2cPK6lDg==} dev: true @@ -1331,7 +1390,6 @@ packages: /escape-string-regexp@4.0.0: resolution: {integrity: sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==} engines: {node: '>=10'} - dev: true /eslint-compat-utils@0.1.2(eslint@8.56.0): resolution: {integrity: sha512-Jia4JDldWnFNIru1Ehx1H5s9/yxiRHY/TimCuUc0jNexew3cF1gI6CYZil1ociakfWO3rRqFjl1mskBblB3RYg==} @@ -1709,6 +1767,14 @@ packages: void-elements: 3.1.0 dev: false + /htmlparser2@8.0.2: + resolution: {integrity: sha512-GYdjWKDkbRLkZ5geuHs5NY1puJ+PXwP7+fHPRz06Eirsb9ugf6d8kkXav6ADhcODhFFPMIXyxkxSuMf3D6NCFA==} + dependencies: + domelementtype: 2.3.0 + domhandler: 5.0.3 + domutils: 3.1.0 + entities: 4.5.0 + /humanize-ms@1.2.1: resolution: {integrity: sha512-Fl70vYtsAFb/C06PTS9dZBo7ihau+Tu/DNCk/OyHhea07S+aeMWpFFkUaXRa8fI+ScZbEI8dfSxwY7gxZ9SAVQ==} dependencies: @@ -1802,6 +1868,11 @@ packages: engines: {node: '>=8'} dev: true + /is-plain-object@5.0.0: + resolution: {integrity: sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q==} + engines: {node: '>=0.10.0'} + dev: false + /is-reference@1.2.1: resolution: {integrity: sha512-U82MsXXiFIrjCK4otLT+o2NA2Cd2g5MLoOVXUZjIOhLurrRxpEXzI8O0KZHr3IjLvlAH1kTPYSuqer5T9ZVBKQ==} dependencies: @@ -2232,6 +2303,12 @@ packages: resolution: {integrity: sha512-7ylylesZQ/PV29jhEDl3Ufjo6ZX7gCqJr5F7PKrqc93v7fzSymt1BpwEU8nAUXs8qzzvqhbjhK5QZg6Mt/HkBg==} dev: true + /linkify-it@5.0.0: + resolution: {integrity: sha512-5aHCbzQRADcdP+ATqnDuhhJ/MRIqDkZX5pyjFHRRysS8vZ5AbqGEoFIb6pYHPZ+L/OC2Lc+xT8uHVVR5CAK/wQ==} + dependencies: + uc.micro: 2.0.0 + dev: false + /locate-character@3.0.0: resolution: {integrity: sha512-SW13ws7BjaeJ6p7Q6CO2nchbYEc3X3J6WrmTTDto7yMPqVSZTUyY5Tjbid+Ab8gLnATtygYtiDIJGQRRn2ZOiA==} @@ -2259,6 +2336,18 @@ packages: dependencies: '@jridgewell/sourcemap-codec': 1.4.15 + /markdown-it@14.0.0: + resolution: {integrity: sha512-seFjF0FIcPt4P9U39Bq1JYblX0KZCjDLFFQPHpL5AzHpqPEKtosxmdq/LTVZnjfH7tjt9BxStm+wXcDBNuYmzw==} + hasBin: true + dependencies: + argparse: 2.0.1 + entities: 4.5.0 + linkify-it: 5.0.0 + mdurl: 2.0.0 + punycode.js: 2.3.1 + uc.micro: 2.0.0 + dev: false + /md5@2.3.0: resolution: {integrity: sha512-T1GITYmFaKuO91vxyoQMFETst+O71VUPEU3ze5GNzDm0OWdP8v1ziTaAEPUr/3kLsY3Sftgz242A1SetQiDL7g==} dependencies: @@ -2270,6 +2359,10 @@ packages: /mdn-data@2.0.30: resolution: {integrity: sha512-GaqWWShW4kv/G9IEucWScBx9G1/vsFZZJUO+tD26M8J8z3Kw5RDQjaoZe03YAClgeS/SWPOcb4nkFBTEi5DUEA==} + /mdurl@2.0.0: + resolution: {integrity: sha512-Lf+9+2r+Tdp5wXDXC4PcIBjTDtq4UKjCPMQhKIuzpJNW0b96kVqSwW0bT7FhRSfmAiFYgP+SCRvdrDozfh0U5w==} + dev: false + /merge2@1.4.1: resolution: {integrity: sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==} engines: {node: '>= 8'} @@ -2385,7 +2478,6 @@ packages: resolution: {integrity: sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g==} engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} hasBin: true - dev: true /natural-compare@1.4.0: resolution: {integrity: sha512-OWND8ei3VtNC9h7V60qff3SVobHr996CTwgxubgyQYEpg290h9J0buyECNNJexkFm5sOajh5G116RYA1c8ZMSw==} @@ -2525,6 +2617,10 @@ packages: callsites: 3.1.0 dev: true + /parse-srcset@1.0.2: + resolution: {integrity: sha512-/2qh0lav6CmI15FzA3i/2Bzk2zCgQhGMkvhOhKNcBVQ1ldgpbfiNTVslmooUmWJcADi1f1kIeynbDRVzNlfR6Q==} + dev: false + /path-exists@4.0.0: resolution: {integrity: sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w==} engines: {node: '>=8'} @@ -2558,7 +2654,6 @@ packages: /picocolors@1.0.0: resolution: {integrity: sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==} - dev: true /picomatch@2.3.1: resolution: {integrity: sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==} @@ -2678,7 +2773,6 @@ packages: nanoid: 3.3.7 picocolors: 1.0.0 source-map-js: 1.0.2 - dev: true /prelude-ls@1.2.1: resolution: {integrity: sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==} @@ -2701,6 +2795,11 @@ packages: hasBin: true dev: true + /punycode.js@2.3.1: + resolution: {integrity: sha512-uxFIHU0YlHYhDQtV4R9J6a52SLx28BCjT+4ieh7IGbgwVJWO+km431c4yRlREUAsAmt/uMjQUyQHNEPf0M39CA==} + engines: {node: '>=6'} + dev: false + /punycode@2.3.1: resolution: {integrity: sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg==} engines: {node: '>=6'} @@ -2804,6 +2903,17 @@ packages: rimraf: 2.7.1 dev: true + /sanitize-html@2.11.0: + resolution: {integrity: sha512-BG68EDHRaGKqlsNjJ2xUB7gpInPA8gVx/mvjO743hZaeMCZ2DwzW7xvsqZ+KNU4QKwj86HJ3uu2liISf2qBBUA==} + dependencies: + deepmerge: 4.3.1 + escape-string-regexp: 4.0.0 + htmlparser2: 8.0.2 + is-plain-object: 5.0.0 + parse-srcset: 1.0.2 + postcss: 8.4.32 + dev: false + /semver@7.5.4: resolution: {integrity: sha512-1bCSESV6Pv+i21Hvpxp3Dx+pSD8lIPt8uVjRrxAUt/nbswYc+tK6Y2btiULjd4+fnq15PX+nqQDC7Oft7WkwcA==} engines: {node: '>=10'} @@ -3129,6 +3239,10 @@ packages: engines: {node: '>=14.17'} hasBin: true + /uc.micro@2.0.0: + resolution: {integrity: sha512-DffL94LsNOccVn4hyfRe5rdKa273swqeA5DJpMOeFmEn1wCDc7nAbbB0gXlgBCL7TNzeTv6G7XVWzan7iJtfig==} + dev: false + /undici-types@5.26.5: resolution: {integrity: sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==} dev: false diff --git a/src/lib/completion.ts b/src/lib/completion.ts index 29403ae..a622650 100644 --- a/src/lib/completion.ts +++ b/src/lib/completion.ts @@ -8,6 +8,7 @@ import { ChatHistoryType, type ChatHistory } from '$lib/history'; import { BytesOutputParser } from 'langchain/schema/output_parser'; import { QdrantClient } from '@qdrant/js-client-rest'; +import sanitizeHtml from 'sanitize-html'; const DEFAULT_MODEL = 'gpt-3.5-turbo'; const DEFAULT_COLLECTION = 'default'; @@ -98,6 +99,7 @@ export class ChatbotCompletion { // eslint-disable-next-line @typescript-eslint/no-explicit-any public async query(history: ChatHistory[]): Promise { + history[history.length - 1].content = sanitizeHtml(history[history.length - 1].content); const vector_response = await this.get_vector_response(history[history.length - 1].content); const context = vector_response.map((content: string) => { diff --git a/src/routes/+page.svelte b/src/routes/+page.svelte index e6f55b9..55f2844 100644 --- a/src/routes/+page.svelte +++ b/src/routes/+page.svelte @@ -10,7 +10,9 @@ import { PaperAirplane, Trash, Stop } from '@steeze-ui/heroicons'; import { writable } from 'svelte/store'; + import markdownit from 'markdown-it'; import { setContext } from 'svelte'; + import sanitizeHtml from 'sanitize-html'; let history: ChatHistory[] = []; @@ -71,6 +73,9 @@ break; } } + const md = markdownit('commonmark'); + history[history.length - 1].content = sanitizeHtml(history[history.length - 1].content); + history[history.length - 1].content = md.render(history[history.length - 1].content); completionState.set(CompletionState.Completed); } diff --git a/src/routes/Message.svelte b/src/routes/Message.svelte index d682d29..45229c1 100644 --- a/src/routes/Message.svelte +++ b/src/routes/Message.svelte @@ -27,7 +27,7 @@

🤖

{/if} -

{content}

+

{@html content}

{:else}