Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

mozpsl fixes from @morkrost #24

Open
wants to merge 5 commits into
base: main
Choose a base branch
from
Open

mozpsl fixes from @morkrost #24

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
d7ac0a4
Order is important for longest-match... Also, typing.Self not in 3.9.
Nov 19, 2024
34f70a2
Fixed tests
Nov 19, 2024
08e0e5c
Add dots to FQDNs
Nov 19, 2024
e10497f
Tests, tests... now properly fixed, I hope.
Nov 19, 2024
962d8bf
we want typing
jschlyter Nov 20, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
40 changes: 27 additions & 13 deletions dnstapir/dns/mozpsl.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import io
from typing import Self

import httpx

Expand All @@ -10,7 +9,7 @@ class TrieNode:
def __init__(self) -> None:
self.count = 0
self.icann: bool | None = None
self.children: dict[str, Self] = {}
self.children: dict[str, TrieNode] = {}


class Trie:
Expand Down Expand Up @@ -47,17 +46,21 @@ def search(self, key: list[str]) -> tuple[int, int]:
pcore = 0
current = self.root
for label in key:
if current.icann is True:
core = current.count
elif current.icann is False:
pcore = current.count
# # If current.icann is None, do not update core or pcore
if label not in current.children:
# if '*' in current.children:
# current = current.children['*']
if current.count != 0:
break
else:
raise KeyError
current = current.children[label]
else:
current = current.children[label]

# If current.icann is None, do not update core or pcore
if current.icann is True:
core = current.count
elif current.icann is False:
pcore = current.count
if pcore == core:
pcore = 0
return (core, pcore)
Expand Down Expand Up @@ -128,12 +131,23 @@ def coredomain(self, domain: str) -> tuple[str, str]:
raise ValueError from exc
lbls = domain.split(".")
lbls.reverse()

c, p = self.trie.search(lbls)
core = lbls[0:c]
core.reverse()
pcore = lbls[0:p]
pcore.reverse()
return (".".join(core), ".".join(pcore))
if c != 0:
core = lbls[0:c]
core.reverse()
core_txt = ".".join(core) + "."
else:
core_txt = ""

if p != 0:
pcore = lbls[0:p]
pcore.reverse()
pcore_txt = ".".join(pcore) + "."
else:
pcore_txt = ""

return (core_txt, pcore_txt)

def rdomain(self, rdomain: str) -> tuple[str, str]:
"""Find ICANN and private name cut-off for domain, reverse order process"""
Expand Down
14 changes: 7 additions & 7 deletions tests/test_dns_mozpsl.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,13 +9,13 @@ def test_mozpsl():
psl = PublicSuffixList()
psl.load_psl_url(url=MOZ_PSL)

assert psl.coredomain("www.ck.") == ("www.ck", "")
assert psl.coredomain("www.something.gov.ck.") == ("something.gov.ck", "")
assert psl.coredomain("www.something.or.other.microsoft.com.") == ("microsoft.com", "")
assert psl.coredomain("www.something.or.other.microsoft.com.br.") == ("microsoft.com.br", "")
assert psl.coredomain("www.ck.") == ("ck.", "")
assert psl.coredomain("www.something.gov.ck.") == ("something.gov.ck.", "")
assert psl.coredomain("www.something.or.other.microsoft.com.") == ("microsoft.com.", "")
assert psl.coredomain("www.something.or.other.microsoft.com.br.") == ("microsoft.com.br.", "")
assert psl.coredomain("www.something.emrstudio-prod.us-gov-east-1.amazonaws.com.") == (
"amazonaws.com",
"something.emrstudio-prod.us-gov-east-1.amazonaws.com",
"amazonaws.com.",
"something.emrstudio-prod.us-gov-east-1.amazonaws.com.",
)
assert psl.rdomain("com.amazonaws.us-gov-east-1.emrstudio-prod.www.something.emrstudio-prod") == (
"com.amazonaws",
Expand All @@ -26,7 +26,7 @@ def test_mozpsl():
psl.coredomain("local.")

# IDN test
assert psl.coredomain("www.xn--mnchen-3ya.de.") == ("xn--mnchen-3ya.de", "")
assert psl.coredomain("www.xn--mnchen-3ya.de.") == ("xn--mnchen-3ya.de.", "")

# Edge cases
with pytest.raises(ValueError):
Expand Down