🐛 Use pre-line file offset

okfde · Dec 20, 2022 · 102b0b5 · 102b0b5
1 parent 00ab021
commit 102b0b5
Show file tree

Hide file tree

Showing 5 changed files with 24 additions and 12 deletions.
diff --git a/froide/helper/email_log_parsing.py b/froide/helper/email_log_parsing.py
@@ -166,11 +166,11 @@ def iteration_done(self):
         if not self._msg_log:
             self.logfile_reader.update_offset_file()
         else:
-            first_logoffset = sorted(x["offset"] for x in self._msg_log.values())[0]
+            first_logoffset = min(x["offset"] for x in self._msg_log.values())
             self.logfile_reader.write_offset_to_file(first_logoffset)
 
     def __next__(self):
-        for line, offset in self.logfile_reader.with_offsets():
+        for line, offset in self.logfile_reader.with_offsets(offset_position="pre"):
             parsed_line = self._parse_line(line)
             if parsed_line is None:
                 continue

diff --git a/requirements-test.txt b/requirements-test.txt
@@ -1,11 +1,13 @@
 #
-# This file is autogenerated by pip-compile with python 3.10
+# This file is autogenerated by pip-compile with python 3.9
 # To update, run:
 #
 #    pip-compile requirements-test.in
 #
 -e git+https://github.com/okfde/django-filingcabinet.git@main#egg=django-filingcabinet
     # via -r requirements.in
+-e git+https://github.com/okfde/pygtail.git@3be17b00f55781caba62d589bcf8aceb0805a58c#egg=pygtail
+    # via -r requirements.in
 aiohttp==3.8.3
     # via geoip2
 aiosignal==1.3.1
@@ -174,7 +176,7 @@ djangorestframework-jsonp==1.0.2
     # via -r requirements.in
 dnspython==2.2.1
     # via pyisemail
-easy-thumbnails==2.8.3
+easy-thumbnails==2.8.4
     # via -r requirements.in
 elasticsearch==7.17.8
     # via
@@ -228,6 +230,8 @@ idna==3.4
     # via
     #   requests
     #   yarl
+importlib-metadata==5.2.0
+    # via markdown
 inflection==0.5.1
     # via pytest-factoryboy
 iniconfig==1.1.1
@@ -330,8 +334,6 @@ pyflakes==3.0.1
     # via
     #   -r requirements-test.in
     #   flake8
-pygtail==0.14.0
-    # via -r requirements.in
 pyisemail==2.0.1
     # via -r requirements.in
 pyotp==2.8.0
@@ -444,11 +446,13 @@ types-urllib3==1.26.25.4
     # via types-requests
 typing-extensions==4.4.0
     # via
+    #   black
     #   django-stubs
     #   django-stubs-ext
     #   libcst
     #   mypy
     #   pyee
+    #   pypdf2
     #   pytest-factoryboy
     #   typing-inspect
 typing-inspect==0.8.0
@@ -491,6 +495,8 @@ wrapt==1.14.1
     # via deprecated
 yarl==1.8.2
     # via aiohttp
+zipp==3.11.0
+    # via importlib-metadata
 zopfli==0.2.2
     # via fonttools
 

diff --git a/requirements.in b/requirements.in
@@ -34,7 +34,6 @@ phonenumbers
 Pillow>=9.0.0
 psycopg2-binary
 pyisemail
-pygtail>=0.14.0
 pypdf2>=2
 python-magic
 python-mimeparse
@@ -46,3 +45,4 @@ WeasyPrint
 websockets
 
 -e git+https://github.com/okfde/django-filingcabinet.git@main#egg=django-filingcabinet
+-e git+https://github.com/okfde/pygtail.git@3be17b00f55781caba62d589bcf8aceb0805a58c#egg=pygtail
diff --git a/requirements.txt b/requirements.txt
@@ -1,11 +1,13 @@
 #
-# This file is autogenerated by pip-compile with python 3.10
+# This file is autogenerated by pip-compile with python 3.9
 # To update, run:
 #
 #    pip-compile requirements.in
 #
 -e git+https://github.com/okfde/django-filingcabinet.git@main#egg=django-filingcabinet
     # via -r requirements.in
+-e git+https://github.com/okfde/pygtail.git@3be17b00f55781caba62d589bcf8aceb0805a58c#egg=pygtail
+    # via -r requirements.in
 aiohttp==3.8.3
     # via geoip2
 aiosignal==1.3.1
@@ -153,7 +155,7 @@ djangorestframework-jsonp==1.0.2
     # via -r requirements.in
 dnspython==2.2.1
     # via pyisemail
-easy-thumbnails==2.8.3
+easy-thumbnails==2.8.4
     # via -r requirements.in
 elasticsearch==7.17.8
     # via
@@ -185,6 +187,8 @@ idna==3.4
     # via
     #   requests
     #   yarl
+importlib-metadata==5.2.0
+    # via markdown
 itypes==1.2.0
     # via coreapi
 jinja2==3.1.2
@@ -238,8 +242,6 @@ pycryptodome==3.16.0
     # via django-filingcabinet
 pydyf==0.5.0
     # via weasyprint
-pygtail==0.14.0
-    # via -r requirements.in
 pyisemail==2.0.1
     # via -r requirements.in
 pyotp==2.8.0
@@ -301,6 +303,8 @@ tinycss2==1.2.1
     # via
     #   cssselect2
     #   weasyprint
+typing-extensions==4.4.0
+    # via pypdf2
 tzdata==2022.7
     # via django-celery-beat
 unicodecsv==0.14.1
@@ -337,5 +341,7 @@ wrapt==1.14.1
     # via deprecated
 yarl==1.8.2
     # via aiohttp
+zipp==3.11.0
+    # via importlib-metadata
 zopfli==0.2.2
     # via fonttools
diff --git a/setup.py b/setup.py
@@ -55,7 +55,7 @@ def find_version(*file_paths):
         "dj-database-url",
         "django-filter",
         "phonenumbers",
-        "pygtail>=0.14.0",
+        "pygtail @ git+https://github.com/okfde/pygtail.git@3be17b00f55781caba62d589bcf8aceb0805a58c#egg=pygtail",
         "django-filingcabinet",
         "icalendar",
         "easy-thumbnails",