Debug ValueError by Voikko analyzer

annif/analyzer/analyzer.py

@@ -5,6 +5,8 @@
 import functools
 import unicodedata
 
+from annif.exception import OperationFailedException
+
 _KEY_TOKEN_MIN_LENGTH = "token_min_length"
 
 
@@ -44,11 +46,16 @@ def tokenize_words(self, text: str, filter: bool = True) -> list[str]:
 
         import nltk.tokenize
 
-        return [
-            self._normalize_word(word)
-            for word in nltk.tokenize.word_tokenize(text)
-            if (not filter or self.is_valid_token(word))
-        ]
+        try:
+            return [
+                self._normalize_word(word)
+                for word in nltk.tokenize.word_tokenize(text)
+                if (not filter or self.is_valid_token(word))
+            ]
+        except OperationFailedException as err:
+            raise OperationFailedException(
+                f"Error in tokenization of text '{text}'"
+            ) from err
 
     def _normalize_word(self, word):
         """Normalize (stem or lemmatize) a word form into a normal form."""

annif/analyzer/voikko.py

@@ -5,6 +5,8 @@
 
 import voikko.libvoikko
 
+from annif.exception import OperationFailedException
+
 from . import analyzer
 
 
@@ -27,7 +29,13 @@ def __getstate__(self) -> dict[str, str | None]:
     def _normalize_word(self, word: str) -> str:
         if self.voikko is None:
             self.voikko = voikko.libvoikko.Voikko(self.param)
-        result = self.voikko.analyze(word)
+        try:
+            result = self.voikko.analyze(word)
+        except ValueError as err:
+            raise OperationFailedException(
+                f"Voikko error in analysis of word '{word}'"
+            ) from err
+
         if len(result) > 0 and "BASEFORM" in result[0]:
             return result[0]["BASEFORM"]
         return word

tests/test_analyzer.py

@@ -1,8 +1,11 @@
 """Unit tests for analyzers in Annif"""
+import importlib.util
+from unittest import mock
 
 import pytest
 
 import annif.analyzer
+from annif.exception import OperationFailedException
 
 
 def test_get_analyzer_nonexistent():
@@ -60,6 +63,24 @@ def test_english_tokenize_words_no_filter():
     assert len(words) == 23
 
 
+@pytest.mark.skipif(
+    importlib.util.find_spec("voikko") is None,
+    reason="test requires that Voikko is installed",
+)
+def test_tokenize_words_operationfailed():
+    analyzer = annif.analyzer.get_analyzer("voikko(fi)")
+    text = "An error producing sentence."
+    with mock.patch(
+        "voikko.libvoikko.Voikko.analyze",
+        side_effect=ValueError,
+    ):
+        with pytest.raises(
+            OperationFailedException,
+            match="Error in tokenization of text 'An error producing sentence.'",
+        ):
+            analyzer.tokenize_words(text)
+
+
 def test_english_filter_words_min_token():
     analyzer = annif.analyzer.get_analyzer("snowball(english,token_min_length=2)")
     text = """Since 2000, a 3D printer can be used to print

tests/test_analyzer_voikko.py

@@ -1,8 +1,11 @@
 """Unit tests for voikko analyzer in Annif"""
 
+from unittest import mock
+
 import pytest
 
 import annif.analyzer
+from annif.exception import OperationFailedException
 
 voikko = pytest.importorskip("annif.analyzer.voikko")
 
@@ -18,3 +21,15 @@ def test_voikko_finnish_analyzer_normalize_word():
     assert analyzer._normalize_word("xyzzy") == "xyzzy"
     assert analyzer._normalize_word("vanhat") == "vanha"
     assert analyzer._normalize_word("koirien") == "koira"
+
+
+def test_voikko_analyze_valueerror():
+    analyzer = annif.analyzer.get_analyzer("voikko(fi)")
+    with mock.patch(
+        "voikko.libvoikko.Voikko.analyze",
+        side_effect=ValueError,
+    ):
+        with pytest.raises(
+            OperationFailedException, match="Voikko error in analysis of word 'kissa'"
+        ):
+            assert analyzer._normalize_word("kissa")