modified: tests/test_tokenizer/test_tokenizer.py

bigdata-ustc · Mar 12, 2024 · 05172b4 · 05172b4
1 parent f02ccce
commit 05172b4
Showing 1 changed file with 2 additions and 10 deletions.
diff --git a/tests/test_tokenizer/test_tokenizer.py b/tests/test_tokenizer/test_tokenizer.py
@@ -71,7 +71,7 @@ def test_TokenizerSpacy():
               some, there are still 4 packs left, 25 each, how many are sold?"]
     ans = [
         'The', 'stationery', 'store', 'has', '600', 'exercise',
-        'books', 'and', 'after', 'selling', '              ', 'some', 'there', 'are', 'still',
+        'books', 'and', 'after', 'selling', '               ', 'some', 'there', 'are', 'still',
         '4', 'packs', 'left', '25', 'each', 'how', 'many', 'are', 'sold'
     ]
     tokenizer = get_tokenizer("pure_text",
@@ -84,15 +84,7 @@ def test_TokenizerSpacy():
 def test_TokenizerBPE():
     items = ['The stationery store has $600$ exercise books, and after selling some,\
         there are still $4$ packs left, $25$ each, how many are sold?']
-    ans = [
-        'h', 'e', ' ', 'st', 'at', 'io', 'n', 'er', 'y', ' ', 'st', 'o', 're', ' ',
-         'h', 'as', ' $', '6', '00', '$ ', 'e', 'x', 'er', 'ci', 's', 'e', ' b', 'o',
-         'o', 'k', 's', ', ', 'an', 'd', ' a', 'ft', 'er', ' ', 's', 'e', 'l', 'l',
-         'in', 'g', ' ', 's', 'ome', ', ', 't', 'h', 'e', 're', ' ', 'are', ' ',
-         'st', 'i', 'l', 'l', ' $', '4', '$ ', 'p', 'a', 'c', 'k', 's', ' ', 'left',
-         ', ', '$', '25', '$ ', 'e', 'a', 'c', 'h', ', ', 'h', 'ow', ' m', 'an', 'y',
-         ' ', 'are', ' ', 's', 'o', 'l', 'd'
-    ]
+    ans = ['h', '600', ' ', '^', '4', '^', ' ', '25', ' ']
     data_path = path_append(abs_current_dir(__file__),
                             "../../static/test_data/standard_luna_data.json", to_str=True)
     tokenizer = get_tokenizer("pure_text", text_params={"tokenizer": 'bpe', "stopwords": set(",?"),