Skip to content

Commit

Permalink
♻️ Move test_mail_log_parsing to pytest
Browse files Browse the repository at this point in the history
  • Loading branch information
@pajowu
pajowu committed Dec 20, 2022
1 parent 2540a01 commit 00ab021
Showing 1 changed file with 162 additions and 183 deletions.
345 changes: 162 additions & 183 deletions froide/helper/tests/test_email_log_parsing.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
import os.path
import tempfile

from django.test import TestCase
import pytest

from ..email_log_parsing import (
PostfixLogfileParser,
Expand All @@ -18,204 +18,183 @@ def p(path):
return os.path.join(TEST_DATA_ROOT, path)


class MaillogParseTest(TestCase):
DEMO_FIELDS = [
" to=<[email protected]>",
" relay=mx1.bund.de[77.87.224.131]:25",
" delay=0.54",
" delays=0.09/0.01/0.28/0.16",
" dsn=2.0.0",
" status=sent (250 2.0.0 Ok: queued as QUEUEUEUEUEU)",
]

MAIL_1_ID = "<foirequest/[email protected]>"
MAIL_1_LOG = [
"Jan 1 01:02:04 mail postfix/smtpd[1234567]: 163CB3EE5E7F: client=localhost.localdomain[127.0.0.1], sasl_method=PLAIN, [email protected]\n",
"Jan 1 01:02:04 mail postfix/cleanup[2345678]: 163CB3EE5E7F: message-id=<foirequest/[email protected]>\n",
"Jan 1 01:02:05 mail postfix/qmgr[2345678]: 163CB3EE5E7F: from=<bounce+asdadasd=+132kl12j31n12d1190+a.foo=example.com@example.com>, size=12321, nrcpt=1 (queue active)\n",
"Jan 1 01:02:05 mail postfix/smtp[2345678]: 163CB3EE5E7F: to=<[email protected]>, relay=example.com[123.456.789.123]:25, delay=0.67, delays=0.1/0.37/0.13/0.07, dsn=2.0.0, status=sent (250 Requested mail action okay, completed: id=aaaaa-bbbbb-ccccc-ddddd\n",
"Jan 1 01:02:05 mail postfix/qmgr[2345678]: 163CB3EE5E7F: removed\n",
]
MAIL_2_ID = "<[email protected]>"
MAIL_2_LOG = [
"Jan 1 01:02:04 mail postfix/smtpd[2345678]: 065AB16D682C: client=localhost.localdomain[127.0.0.1], sasl_method=PLAIN, [email protected]\n",
"Jan 1 01:02:04 mail postfix/cleanup[2345678]: 065AB16D682C: message-id=<[email protected]>\n",
"Jan 1 01:02:04 mail postfix/qmgr[2345678]: 065AB16D682C: from=<[email protected]>, size=5608, nrcpt=1 (queue active)\n",
"Jan 1 01:02:05 mail postfix/smtp[2345678]: 065AB16D682C: to=<[email protected]>, relay=example.org[234.456.321.123]:25, delay=5.2, delays=0.09/0.02/0.14/4.9, dsn=2.0.0, status=sent (250 2.0.0 Ok: queued as ABCDEF)\n",
"Jan 1 01:02:05 mail postfix/qmgr[2345678]: 065AB16D682C: removed\n",
]
MAIL_2_DATA = {
"message-id": "<[email protected]>",
"from": "[email protected]",
"to": "[email protected]",
DEMO_FIELDS = [
" to=<[email protected]>",
" relay=mx1.bund.de[77.87.224.131]:25",
" delay=0.54",
" delays=0.09/0.01/0.28/0.16",
" dsn=2.0.0",
" status=sent (250 2.0.0 Ok: queued as QUEUEUEUEUEU)",
]

MAIL_1_ID = "<foirequest/[email protected]>"
MAIL_1_LOG = [
"Jan 1 01:02:04 mail postfix/smtpd[1234567]: 163CB3EE5E7F: client=localhost.localdomain[127.0.0.1], sasl_method=PLAIN, [email protected]\n",
"Jan 1 01:02:04 mail postfix/cleanup[2345678]: 163CB3EE5E7F: message-id=<foirequest/[email protected]>\n",
"Jan 1 01:02:05 mail postfix/qmgr[2345678]: 163CB3EE5E7F: from=<bounce+asdadasd=+132kl12j31n12d1190+a.foo=example.com@example.com>, size=12321, nrcpt=1 (queue active)\n",
"Jan 1 01:02:05 mail postfix/smtp[2345678]: 163CB3EE5E7F: to=<[email protected]>, relay=example.com[123.456.789.123]:25, delay=0.67, delays=0.1/0.37/0.13/0.07, dsn=2.0.0, status=sent (250 Requested mail action okay, completed: id=aaaaa-bbbbb-ccccc-ddddd\n",
"Jan 1 01:02:05 mail postfix/qmgr[2345678]: 163CB3EE5E7F: removed\n",
]
MAIL_2_ID = "<[email protected]>"
MAIL_2_LOG = [
"Jan 1 01:02:04 mail postfix/smtpd[2345678]: 065AB16D682C: client=localhost.localdomain[127.0.0.1], sasl_method=PLAIN, [email protected]\n",
"Jan 1 01:02:04 mail postfix/cleanup[2345678]: 065AB16D682C: message-id=<[email protected]>\n",
"Jan 1 01:02:04 mail postfix/qmgr[2345678]: 065AB16D682C: from=<[email protected]>, size=5608, nrcpt=1 (queue active)\n",
"Jan 1 01:02:05 mail postfix/smtp[2345678]: 065AB16D682C: to=<[email protected]>, relay=example.org[234.456.321.123]:25, delay=5.2, delays=0.09/0.02/0.14/4.9, dsn=2.0.0, status=sent (250 2.0.0 Ok: queued as ABCDEF)\n",
"Jan 1 01:02:05 mail postfix/qmgr[2345678]: 065AB16D682C: removed\n",
]
MAIL_2_DATA = {
"message-id": "<[email protected]>",
"from": "[email protected]",
"to": "[email protected]",
"status": "sent",
"removed": "",
}


def test_parse_field_minimal():
assert PostfixLogfileParser._parse_fields(DEMO_FIELDS) == {
"to": "[email protected]",
"relay": "mx1.bund.de[77.87.224.131]:25",
"delay": "0.54",
"delays": "0.09/0.01/0.28/0.16",
"dsn": "2.0.0",
"status": "sent",
}


def test_parse_fields_limit():
assert PostfixLogfileParser._parse_fields(DEMO_FIELDS, ["to", "status"]) == {
"to": "[email protected]",
"status": "sent",
}

assert PostfixLogfileParser._parse_fields(DEMO_FIELDS, {"to", "status"}) == {
"to": "[email protected]",
"status": "sent",
}


def test_parse_without_value():
assert PostfixLogfileParser._parse_fields([" removed"]) == {
"removed": "",
}

def test_parse_field_minimal(self):
self.assertEqual(
PostfixLogfileParser._parse_fields(self.DEMO_FIELDS),
{
"to": "[email protected]",
"relay": "mx1.bund.de[77.87.224.131]:25",
"delay": "0.54",
"delays": "0.09/0.01/0.28/0.16",
"dsn": "2.0.0",
"status": "sent",
},
)

def test_parse_fields_limit(self):
self.assertEqual(
PostfixLogfileParser._parse_fields(self.DEMO_FIELDS, ["to", "status"]),
{
"to": "[email protected]",
"status": "sent",
},
)
def test_parse_empty_file():
parser = PostfixLogfileParser(io.StringIO(""))
assert next(parser, None) is None

self.assertEqual(
PostfixLogfileParser._parse_fields(self.DEMO_FIELDS, {"to", "status"}),
{
"to": "[email protected]",
"status": "sent",
},
)

def test_parse_without_value(self):
self.assertEqual(
PostfixLogfileParser._parse_fields([" removed"]),
{
"removed": "",
},
def test_parse_line():
parser = PostfixLogfileParser(io.StringIO(""), relevant_fields={"field"})
assert (
parser._parse_line(
"Jan 1 01:02:03 mail postfix/something[1234566]: ABCDEF1234: field=value"
)
) == PostfixLogLine("Jan 1 01:02:03", "ABCDEF1234", {"field": "value"})

def test_parse_empty_file(self):
parser = PostfixLogfileParser(io.StringIO(""))
self.assertEqual(next(parser, None), None)

def test_parse_line(self):
parser = PostfixLogfileParser(io.StringIO(""), relevant_fields={"field"})
self.assertEqual(
parser._parse_line(
"Jan 1 01:02:03 mail postfix/something[1234566]: ABCDEF1234: field=value"
),
PostfixLogLine("Jan 1 01:02:03", "ABCDEF1234", {"field": "value"}),
assert (
parser._parse_line(
"Jan 1 01:02:03 mail postfix/something[1234566]: ABCDEF1234: field=value, field2"
)
) == PostfixLogLine("Jan 1 01:02:03", "ABCDEF1234", {"field": "value"})

self.assertEqual(
parser._parse_line(
"Jan 1 01:02:03 mail postfix/something[1234566]: ABCDEF1234: field=value, field2"
),
PostfixLogLine("Jan 1 01:02:03", "ABCDEF1234", {"field": "value"}),
)

def test_parse_file(self):
with open(p("maillog_001.txt")) as f:
parser = PostfixLogfileParser(f)
msg1 = next(parser)
self.assertEqual(msg1["data"]["message-id"], self.MAIL_1_ID)
self.assertEqual(len(msg1["log"]), 5)

self.assertEqual(len(parser._msg_log), 1)

msg2 = next(parser)
self.assertEqual(
msg2["data"],
self.MAIL_2_DATA,
)
self.assertEqual(
msg2["log"],
self.MAIL_2_LOG,
)
self.assertEqual(next(parser, None), None)

self.assertEqual(len(parser._msg_log), 0)

def test_parse_file_with_partial_log(self):
with open(p("maillog_002.txt")) as f:
parser = PostfixLogfileParser(f)
self.assertEqual(next(parser, None), None)
self.assertEqual(len(parser._msg_log), 1)

def test_email_signal(self):
invocations = []

def callback(**kwargs):
invocations.append(kwargs)

email_left_queue.connect(callback)
self.assertEqual(len(invocations), 0)
with tempfile.TemporaryDirectory() as dir:
check_delivery_from_log(p("maillog_001.txt"), str(dir + "/mail_log.offset"))
self.assertEqual(len(invocations), 2)
self.assertEqual(invocations[0]["message_id"], self.MAIL_1_ID)
self.assertEqual(invocations[1]["message_id"], self.MAIL_2_ID)
self.assertEqual(
invocations[0]["log"],
self.MAIL_1_LOG,
)
self.assertEqual(
invocations[1]["log"],
self.MAIL_2_LOG,
)
def test_parse_file():
with open(p("maillog_001.txt")) as f:
parser = PostfixLogfileParser(f)
msg1 = next(parser)
assert msg1["data"]["message-id"] == MAIL_1_ID
assert len(msg1["log"]) == 5

def test_pygtail_log_append(self):
invocations = []

def callback(**kwargs):
invocations.append(kwargs)

email_left_queue.connect(callback)
self.assertEqual(len(invocations), 0)

with tempfile.TemporaryDirectory() as tmpdir:
logfile_path = str(tmpdir + "/mail_log")
offset_file_path = str(tmpdir + "/mail_log.offset")
with open(logfile_path, "w") as logfile:
check_delivery_from_log(logfile_path, offset_file_path)
self.assertEqual(len(invocations), 0)
with open(p("maillog_003.txt")) as fixture:
logfile.write(fixture.read())
logfile.flush()
check_delivery_from_log(logfile_path, offset_file_path)
self.assertEqual(len(invocations), 1)

with open(p("maillog_004.txt")) as fixture:
logfile.write(fixture.read())
logfile.flush()
check_delivery_from_log(logfile_path, offset_file_path)
self.assertEqual(len(invocations), 2)

check_delivery_from_log(logfile_path, offset_file_path)
self.assertEqual(len(invocations), 2)

self.assertEqual(invocations[0]["message_id"], self.MAIL_2_ID)
self.assertEqual(invocations[1]["message_id"], self.MAIL_1_ID)
self.assertEqual(
invocations[0]["log"],
self.MAIL_2_LOG,
)
assert len(parser._msg_log) == 1

self.assertEqual(invocations[1]["log"], self.MAIL_1_LOG)
msg2 = next(parser)
assert msg2["data"] == MAIL_2_DATA
assert msg2["log"] == MAIL_2_LOG
assert next(parser, None) is None

def test_multiple_partial(self):
invocations = []
assert len(parser._msg_log) == 0

def callback(**kwargs):
invocations.append(kwargs)

email_left_queue.connect(callback)
self.assertEqual(len(invocations), 0)
with tempfile.TemporaryDirectory() as dir:
check_delivery_from_log(p("maillog_005.txt"), str(dir + "/mail_log.offset"))
self.assertEqual(len(invocations), 1)
def test_parse_file_with_partial_log():
with open(p("maillog_002.txt")) as f:
parser = PostfixLogfileParser(f)
assert next(parser, None) is None
assert len(parser._msg_log) == 1

check_delivery_from_log(p("maillog_005.txt"), str(dir + "/mail_log.offset"))
self.assertEqual(len(invocations), 1)

self.assertEqual(invocations[0]["message_id"], self.MAIL_1_ID)
self.assertEqual(
invocations[0]["log"],
self.MAIL_1_LOG,
)
@pytest.mark.django_db
def test_email_signal():
invocations = []

def callback(**kwargs):
invocations.append(kwargs)

email_left_queue.connect(callback)
assert len(invocations) == 0
with tempfile.TemporaryDirectory() as dir:
check_delivery_from_log(p("maillog_001.txt"), str(dir + "/mail_log.offset"))
assert len(invocations) == 2
assert invocations[0]["message_id"] == MAIL_1_ID
assert invocations[1]["message_id"] == MAIL_2_ID
assert invocations[0]["log"] == MAIL_1_LOG
assert invocations[1]["log"] == MAIL_2_LOG


@pytest.mark.django_db
def test_pygtail_log_append():
invocations = []

def callback(**kwargs):
invocations.append(kwargs)

email_left_queue.connect(callback)
assert len(invocations) == 0

with tempfile.TemporaryDirectory() as tmpdir:
logfile_path = str(tmpdir + "/mail_log")
offset_file_path = str(tmpdir + "/mail_log.offset")
with open(logfile_path, "w") as logfile:
check_delivery_from_log(logfile_path, offset_file_path)
assert len(invocations) == 0
with open(p("maillog_003.txt")) as fixture:
logfile.write(fixture.read())
logfile.flush()
check_delivery_from_log(logfile_path, offset_file_path)
assert len(invocations) == 1

with open(p("maillog_004.txt")) as fixture:
logfile.write(fixture.read())
logfile.flush()
check_delivery_from_log(logfile_path, offset_file_path)
assert len(invocations) == 2

check_delivery_from_log(logfile_path, offset_file_path)
assert len(invocations) == 2

assert invocations[0]["message_id"] == MAIL_2_ID
assert invocations[1]["message_id"] == MAIL_1_ID
assert invocations[0]["log"] == MAIL_2_LOG

assert invocations[1]["log"] == MAIL_1_LOG


def test_multiple_partial():
invocations = []

def callback(**kwargs):
invocations.append(kwargs)

email_left_queue.connect(callback)
assert len(invocations) == 0
with tempfile.TemporaryDirectory() as dir:
check_delivery_from_log(p("maillog_005.txt"), str(dir + "/mail_log.offset"))
assert len(invocations) == 1

check_delivery_from_log(p("maillog_005.txt"), str(dir + "/mail_log.offset"))
assert len(invocations) == 1

assert invocations[0]["message_id"] == MAIL_1_ID
assert invocations[0]["log"] == MAIL_1_LOG

0 comments on commit 00ab021

Please sign in to comment.