modified: EduNLP/SIF/tokenization/text/tokenization.py

modified: tests/test_tokenizer/test_tokenizer.py
bigdata-ustc · Mar 12, 2024 · f02ccce · f02ccce
1 parent 1476f8a
commit f02ccce
Show file tree

Hide file tree

Showing 2 changed files with 24 additions and 10 deletions.
diff --git a/EduNLP/SIF/tokenization/text/tokenization.py b/EduNLP/SIF/tokenization/text/tokenization.py
@@ -101,7 +101,7 @@ def tokenize(text,
     elif (tokenizer == 'bpe'):
         try:
             tokenizer = HGTokenizer.from_file('bpeTokenizer.json')
-        except OSError:
+        except :
             tokenizer = huggingface_tokenizer.Tokenizer(
                 huggingface_tokenizer.models.BPE())
             if (bpe_trainfile is None):

diff --git a/tests/test_tokenizer/test_tokenizer.py b/tests/test_tokenizer/test_tokenizer.py
@@ -51,20 +51,34 @@ def test_CharTokenizer():
     assert ret == ans
 
 
-def test_Tokenizer():
-    items = ["""The stationery store has 600 exercise books, and after selling
-              some, there are still 4 packs left, 25 each, how many are sold?"""]
+def test_TokenizerNLTK():
+    items = ["The stationery store has 600 exercise books, and after selling\
+              some, there are still 4 packs left, 25 each, how many are sold?"]
     ans = [
         'The', 'stationery', 'store', 'has', '600', 'exercise',
         'books', 'and', 'after', 'selling', 'some', 'there', 'are', 'still',
         '4', 'packs', 'left', '25', 'each', 'how', 'many', 'are', 'sold'
     ]
-    for tok in ['nltk', 'spacy']:
-        tokenizer = get_tokenizer("pure_text",
-                                  text_params={"tokenizer": tok, "stopwords": set(",?")})
-        tokens = tokenizer(items)
-        ret = next(tokens)
-        assert ret == ans
+    tokenizer = get_tokenizer("pure_text",
+                              text_params={"tokenizer": 'nltk', "stopwords": set(",?")})
+    tokens = tokenizer(items)
+    ret = next(tokens)
+    assert ret == ans
+
+
+def test_TokenizerSpacy():
+    items = ["The stationery store has 600 exercise books, and after selling\
+              some, there are still 4 packs left, 25 each, how many are sold?"]
+    ans = [
+        'The', 'stationery', 'store', 'has', '600', 'exercise',
+        'books', 'and', 'after', 'selling', '              ', 'some', 'there', 'are', 'still',
+        '4', 'packs', 'left', '25', 'each', 'how', 'many', 'are', 'sold'
+    ]
+    tokenizer = get_tokenizer("pure_text",
+                              text_params={"tokenizer": 'spacy', "stopwords": set(",?")})
+    tokens = tokenizer(items)
+    ret = next(tokens)
+    assert ret == ans
 
 
 def test_TokenizerBPE():