Skip to content

Commit

Permalink
Codes refactoring
Browse files Browse the repository at this point in the history
  • Loading branch information
@Justin-APNIC
Justin-APNIC committed Apr 17, 2024
1 parent 880bdc8 commit 2d03d28
Show file tree
Hide file tree
Showing 3 changed files with 121 additions and 51 deletions.
24 changes: 5 additions & 19 deletions irrd/mirroring/nrtm_generator.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,10 @@
from datetime import datetime, timedelta, timezone
from typing import Optional

from irrd.conf import get_nrtm_response_dummy_object_class_for_source, get_setting
from irrd.conf import get_setting
from irrd.storage.database_handler import DatabaseHandler
from irrd.storage.queries import DatabaseStatusQuery, RPSLDatabaseJournalQuery
from irrd.utils.text import dummy_rpsl_object
from irrd.utils.text import dummify_object_text
from irrd.utils.text import remove_auth_hashes as remove_auth_hashes_func


Expand Down Expand Up @@ -121,23 +121,9 @@ def generate(
operation_str += " " + str(operation["serial_nrtm"])
text = operation["object_text"]

nrtm_response_dummy_object_class = get_nrtm_response_dummy_object_class_for_source(source)
if nrtm_response_dummy_object_class:
object_class = operation["object_class"]
pk = operation["rpsl_pk"]

if object_class in nrtm_response_dummy_object_class:
dummy_attributes = get_setting(f"sources.{source}.nrtm_response_dummy_attributes")

if get_setting(f"sources.{source}.nrtm_response_dummy_remarks"):
dummy_remarks = textwrap.indent(
get_setting(f"sources.{source}.nrtm_response_dummy_remarks"), "remarks:".ljust(16)
)
else:
dummy_remarks = None

if dummy_attributes:
text = dummy_rpsl_object(text, dummy_attributes, pk, dummy_remarks)
object_class = operation["object_class"]
pk = operation["rpsl_pk"]
text = dummify_object_text(text, object_class, source, pk)

if remove_auth_hashes:
text = remove_auth_hashes_func(text)
Expand Down
85 changes: 76 additions & 9 deletions irrd/utils/tests/test_text.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
from irrd.utils.rpsl_samples import SAMPLE_MNTNER

from ..text import (
dummy_rpsl_object,
dummify_object_text,
remove_auth_hashes,
remove_last_modified,
snake_to_camel_case,
Expand Down Expand Up @@ -67,15 +67,82 @@ def test_snake_to_camel_case():
assert snake_to_camel_case(["foo1_bar", "second_item"]) == ["foo1Bar", "secondItem"]


def test_dummy_rpsl_object():
assert dummy_rpsl_object("", {}, "", None) == ""
def test_dummify_object_text(config_override):
assert dummify_object_text("", "person", "TEST", "PERSON-TEST") == ""

config_override(
{
"sources": {
"TEST": {
"keep_journal": True,
"nrtm_response_dummy_object_class": "role",
}
}
}
)
assert (
dummify_object_text(
"person: Test person\naddress: address\nnic-hdl: PERSON-TEST\nphone: "
" +31 20 000 0000",
"person",
"TEST",
"PERSON-TEST",
)
== "person: Test person\naddress: address\nnic-hdl: PERSON-TEST\nphone: "
" +31 20 000 0000",
)

config_override(
{
"sources": {
"TEST": {
"keep_journal": True,
"nrtm_response_dummy_object_class": "person",
"nrtm_response_dummy_attributes": {
"person": "Dummy person for %s",
"address": "Dummy address",
"phone": "1234",
},
}
}
}
)
assert (
dummify_object_text(
"person: Test person\naddress: address\nnic-hdl: PERSON-TEST\nphone: "
" +31 20 000 0000",
"person",
"TEST",
"PERSON-TEST",
)
== "person: Dummy person for PERSON-TEST\naddress: Dummy address\nnic-hdl: "
" PERSON-TEST\nphone: 1234\n"
)

config_override(
{
"sources": {
"TEST": {
"keep_journal": True,
"nrtm_response_dummy_object_class": "person",
"nrtm_response_dummy_attributes": {
"person": "Dummy person for %s",
"address": "Dummy address",
"phone": "1234",
},
"nrtm_response_dummy_remarks": "Invalid object",
}
}
}
)
assert (
dummy_rpsl_object(
"person: Test person\nnic-hdl: PERSON-TEST\nphone: +31 20 000 0000",
{"person": "Dummy person for %s", "phone": 1234},
dummify_object_text(
"person: Test person\naddress: address\nnic-hdl: PERSON-TEST\nphone: "
" +31 20 000 0000",
"person",
"TEST",
"PERSON-TEST",
"remarks: Invalid object",
)
== "person: Dummy person for PERSON-TEST\nnic-hdl: PERSON-TEST\nphone: "
" 1234\nremarks: Invalid object\n"
== "person: Dummy person for PERSON-TEST\naddress: Dummy address\nnic-hdl: "
" PERSON-TEST\nphone: 1234\nremarks: Invalid object\n"
)
63 changes: 40 additions & 23 deletions irrd/utils/text.py
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,12 @@
import re
from typing import Dict, Iterator, List, Optional, Set, TextIO, Union

from irrd.conf import PASSWORD_HASH_DUMMY_VALUE
import textwrap
from typing import Iterator, List, Optional, Set, TextIO, Union

from irrd.conf import (
PASSWORD_HASH_DUMMY_VALUE,
get_nrtm_response_dummy_object_class_for_source,
get_setting,
)
from irrd.rpsl.auth import PASSWORD_HASHERS_ALL

re_remove_passwords = re.compile(r"(%s)[^\n]+" % "|".join(PASSWORD_HASHERS_ALL.keys()), flags=re.IGNORECASE)
Expand Down Expand Up @@ -100,32 +105,44 @@ def clean_ip_value_error(value_error):
return re.sub(re_clean_ip_error, "", str(value_error))


def dummy_rpsl_object(rpsl_text: str, dummy_attributes: Dict[str, str], pk: str, remarks: Optional[str]):
def dummify_object_text(rpsl_text: str, object_class: str, source: str, pk: str):
"""
Modify the value of attributes in an RPSL object.
"""

if not rpsl_text:
return rpsl_text

lines = rpsl_text.splitlines()

for index, line in enumerate(lines):
for key, value in dummy_attributes.items():
if "%s" in str(value):
value = str(value).replace("%s", pk)

if line.startswith(f"{key}:"):
format_key = f"{key}:".ljust(RPSL_ATTRIBUTE_TEXT_WIDTH)
if not isinstance(value, str):
value = str(value)
lines[index] = format_key + value

dummyfied_rpsl_object = "\n".join(lines) + "\n"

if rpsl_text != dummyfied_rpsl_object:
if remarks:
dummyfied_rpsl_object += remarks.strip() + "\n"
return dummyfied_rpsl_object
nrtm_response_dummy_object_class = get_nrtm_response_dummy_object_class_for_source(source)
if nrtm_response_dummy_object_class:
if object_class in nrtm_response_dummy_object_class:
dummy_attributes = get_setting(f"sources.{source}.nrtm_response_dummy_attributes")
if dummy_attributes:
if get_setting(f"sources.{source}.nrtm_response_dummy_remarks"):
dummy_remarks = textwrap.indent(
get_setting(f"sources.{source}.nrtm_response_dummy_remarks"), "remarks:".ljust(16)
)
else:
dummy_remarks = None

lines = rpsl_text.splitlines()

for index, line in enumerate(lines):
for key, value in dummy_attributes.items():
if "%s" in str(value):
value = str(value).replace("%s", pk)

if line.startswith(f"{key}:"):
format_key = f"{key}:".ljust(RPSL_ATTRIBUTE_TEXT_WIDTH)
if not isinstance(value, str):
value = str(value)
lines[index] = format_key + value

dummyfied_rpsl_object = "\n".join(lines) + "\n"

if rpsl_text != dummyfied_rpsl_object:
if dummy_remarks:
dummyfied_rpsl_object += dummy_remarks.strip() + "\n"
return dummyfied_rpsl_object

return rpsl_text

0 comments on commit 2d03d28

Please sign in to comment.