diff --git a/src/tokenizers.js b/src/tokenizers.js index ce60f7d1e..8cf1325aa 100644 --- a/src/tokenizers.js +++ b/src/tokenizers.js @@ -2986,7 +2986,7 @@ export class PreTrainedTokenizer extends Callable { * ```javascript * import { AutoTokenizer } from "@xenova/transformers"; * - * const tokenizer = await AutoTokenizer.from_pretrained("mistralai/Mistral-7B-Instruct-v0.1"); + * const tokenizer = await AutoTokenizer.from_pretrained("Xenova/mistral-tokenizer-v1"); * * const chat = [ * { "role": "user", "content": "Hello, how are you?" }, diff --git a/tests/generate_tests.py b/tests/generate_tests.py index c449d34be..b36387963 100644 --- a/tests/generate_tests.py +++ b/tests/generate_tests.py @@ -229,7 +229,7 @@ 'basic', ], - 'mistralai/Mistral-7B-Instruct-v0.1': [ + 'Xenova/mistral-tokenizer-v1': [ 'basic', ], diff --git a/tests/tokenizers.test.js b/tests/tokenizers.test.js index 8b92c6702..cfbe7c663 100644 --- a/tests/tokenizers.test.js +++ b/tests/tokenizers.test.js @@ -334,7 +334,7 @@ describe('Extra decoding tests', () => { describe('Chat templates', () => { it('should generate a chat template', async () => { - const tokenizer = await AutoTokenizer.from_pretrained("mistralai/Mistral-7B-Instruct-v0.1"); + const tokenizer = await AutoTokenizer.from_pretrained("Xenova/mistral-tokenizer-v1"); const chat = [ { "role": "user", "content": "Hello, how are you?" },