remove asserts in Lexer

norminette/lexer/lexer.py

@@ -92,7 +92,6 @@ def __init__(self, file: File):
         self.__line_pos = self.__line = 1
 
     def raw_peek(self, *, offset: int = 0, collect: int = 1):
-        assert collect > 0 and offset >= 0
         if (pos := self.__pos + offset) < len(self.file.source):
             return ''.join(self.file.source[pos:pos+collect])
         return None
@@ -122,7 +121,6 @@ def pop(
         use_spaces: bool = False,
         use_escape: bool = False,
     ) -> str:
-        assert times > 0
         result = ""
         for _ in range(times):
             for _ in range(100):
@@ -147,8 +145,7 @@ def pop(
                             # BUG It is just considering one `byte` (0x0 to 0xFF), so it not works correctly
                             # with prefixed strings like `L"\0x1234"`.
                             peek = self.raw_peek(offset=size, collect=2)
-                            assert peek  # TODO Replace it to `raise UnexpectedEOF`
-                            if peek[0] not in hexadecimal_digits:
+                            if peek is None or peek[0] not in hexadecimal_digits:
                                 error = Error.from_name("NO_HEX_DIGITS", level="Notice")
                                 error.add_highlight(self.__line, self.__line_pos + size - 1, length=1)
                                 self.file.errors.add(error)
@@ -380,9 +377,9 @@ def parse_line_comment(self) -> Optional[Token]:
             return
         pos = self.line_pos()
         val = self.pop(times=2)
-        while self.peek():
-            char, _ = self.peek()
-            if char in ('\n', None):
+        while result := self.peek():
+            char, _ = result
+            if char == '\n':
                 break
             try:
                 val += self.pop()

tests/test_lexer.py

@@ -48,7 +48,7 @@ def test_lexer_raw_peek(source: str, parameters: Dict[str, Any], expected: Optio
     "Newline": ['\n', {}, ('\n', 1)],
     "Escaped newline": ["\\\n", {}, ('\\', 1)],
     "Times with exact chars": ["abc", {"times": 3}, ("abc", 3)],
-    "Times with trigraphs": [r"??<a??<b", {"times": 4}, ("{a{b", 6)],
+    "Times with trigraphs": [r"??<a??<b", {"times": 4}, ("{a{b", 8)],
     "Times with trigraphs 2": [r"??<a??<b", {"times": 4}, ("{a{b", 8)],
     "Offset with large source": ["heello!s", {"offset": 6}, ('!', 1)],
     # "Offset with large source with trigraphs": ["he??<el??/lo!s", {"offset": 8}, ('!', 1)],  # teoric BUG
@@ -98,6 +98,10 @@ def test_lexer_peek(source: str, parameters: Dict[str, Any], expected: Optional[
     "Multiples escaped newlines": ["\\\n\\\na\n", {}, 'a'],
     "Multiples escaped newlines with trigraphs": ["??/\n??/\na\n", {}, 'a'],
     "Bla": ["\\\na\n", {}, 'a'],
+    "Escape hex followed by EOF": [r"\x", {}, '\\'],
+    "Escape hex followed by EOF with use_escape": [r"\x", {"use_escape": True}, r"\x"],
+    "Bad escape hex with use_escape and times": [r"\x\x", {"use_escape": True, "times": 2}, r"\x\x"],
+    "Ok escape hex followed by a bad with use_escape and times": [r"\xF\x", {"use_escape": True, "times": 2}, r"\xF\x"],
 }))
 def test_lexer_pop(source: str, parameters: Dict[str, Any], expected: Optional[str]):
     lexer = lexer_from_source(source)