update tests to accomodate tokenizers move

pytorch · Jan 21, 2025 · e451834 · e451834
1 parent fc2c9e0
commit e451834
Show file tree

Hide file tree

Showing 4 changed files with 3 additions and 1 deletion.
diff --git a/.../modules/tokenizers/test_sentencepiece.py → ...ansforms/tokenizers/test_sentencepiece.py b/.../modules/tokenizers/test_sentencepiece.py → ...ansforms/tokenizers/test_sentencepiece.py
diff --git a/...htune/modules/tokenizers/test_tiktoken.py → ...es/transforms/tokenizers/test_tiktoken.py b/...htune/modules/tokenizers/test_tiktoken.py → ...es/transforms/tokenizers/test_tiktoken.py
diff --git a/...orchtune/modules/tokenizers/test_utils.py → ...dules/transforms/tokenizers/test_utils.py b/...orchtune/modules/tokenizers/test_utils.py → ...dules/transforms/tokenizers/test_utils.py
diff --git a/torchtune/models/t5/_tokenizer.py b/torchtune/models/t5/_tokenizer.py
@@ -5,7 +5,9 @@
 # LICENSE file in the root directory of this source tree.
 from typing import Any, Dict, List
 
-from torchtune.modules.tokenizers._sentencepiece import SentencePieceBaseTokenizer
+from torchtune.modules.transforms.tokenizers._sentencepiece import (
+    SentencePieceBaseTokenizer,
+)
 
 
 class T5Tokenizer(SentencePieceBaseTokenizer):