Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add entity address parser #37

Merged
merged 4 commits into from
Jul 30, 2024
Merged

Add entity address parser #37

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
6da04d8
add entity address parser
sam-kleiner Jul 30, 2024
33a45d4
update tests
sam-kleiner Jul 30, 2024
f3505df
cleanup
sam-kleiner Jul 30, 2024
b9d661f
fix python 3.8 list type
sam-kleiner Jul 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
167 changes: 151 additions & 16 deletions tests/test_parser.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,16 @@ def test_autnum_response_parser(self):
'type': 'entity',
'whois_server': 'whois.arin.net',
'name': 'Cloudflare, Inc.',
'rir': 'arin'
'rir': 'arin',
'address': {
'po_box': '',
'ext_address': '',
'street_address': '',
'locality': '',
'region': '',
'postal_code': '',
'country': '',
}
}
]
)
Expand All @@ -72,7 +81,16 @@ def test_autnum_response_parser(self):
'name': 'Abuse',
'email': '[email protected]',
'tel': '+1-650-319-8930',
'rir': 'arin'
'rir': 'arin',
'address': {
'po_box': '',
'ext_address': '',
'street_address': '',
'locality': '',
'region': '',
'postal_code': '',
'country': '',
}
}
]
)
Expand All @@ -86,7 +104,16 @@ def test_autnum_response_parser(self):
'name': 'NOC',
'email': '[email protected]',
'tel': '+1-650-319-8930',
'rir': 'arin'
'rir': 'arin',
'address': {
'po_box': '',
'ext_address': '',
'street_address': '',
'locality': '',
'region': '',
'postal_code': '',
'country': '',
}
}
]
)
Expand All @@ -100,7 +127,16 @@ def test_autnum_response_parser(self):
'name': 'Admin',
'email': '[email protected]',
'tel': '+1-650-319-8930',
'rir': 'arin'
'rir': 'arin',
'address': {
'po_box': '',
'ext_address': '',
'street_address': '',
'locality': '',
'region': '',
'postal_code': '',
'country': '',
}
}
]
)
Expand Down Expand Up @@ -176,7 +212,16 @@ def test_domain_response_parser(self):
'url': 'https://rdap.norid.no/entity/reg42-NORID',
'type': 'entity',
'name': 'Domeneshop AS',
'email': '[email protected]'
'email': '[email protected]',
'address': {
'po_box': '',
'ext_address': '',
'street_address': 'Christian Krohgs gate 16',
'locality': 'Oslo',
'region': '',
'postal_code': 'NO-0186',
'country': 'NORWAY',
}
}
]
)
Expand All @@ -187,7 +232,16 @@ def test_domain_response_parser(self):
'url': 'https://rdap.norid.no/entity/DH21326R-NORID',
'type': 'entity',
'name': 'Domeneshop Hostmaster',
'email': '[email protected]'
'email': '[email protected]',
'address': {
'po_box': '',
'ext_address': '',
'street_address': '',
'locality': '',
'region': '',
'postal_code': '',
'country': 'NORWAY',
}
}
]
)
Expand All @@ -210,7 +264,7 @@ def test_domain_response_parser(self):
{
'handle': '1647',
'type': 'entity',
'name': 'Hosting Concepts B.V. d/b/a Registrar.eu',
'name': 'Hosting Concepts B.V. d/b/a Registrar.eu'
}
]
)
Expand Down Expand Up @@ -255,7 +309,16 @@ def test_ip_response_parser(self):
'name': 'APNIC RESEARCH',
'email': '[email protected]',
'tel': '+61-7-3858-3199',
'rir': 'apnic'
'rir': 'apnic',
'address': {
'po_box': '',
'ext_address': '',
'street_address': '',
'locality': '',
'region': '',
'postal_code': '',
'country': '',
}
}
]
)
Expand Down Expand Up @@ -315,7 +378,16 @@ def test_ip_response_parser(self):
'name': 'APNIC RESEARCH',
'email': '[email protected]',
'tel': '+61-7-3858-3199',
'rir': 'apnic'
'rir': 'apnic',
'address': {
'po_box': '',
'ext_address': '',
'street_address': '',
'locality': '',
'region': '',
'postal_code': '',
'country': '',
}
}
]
)
Expand Down Expand Up @@ -348,7 +420,16 @@ def test_ip_response_parser(self):
'rir': 'arin',
'type': 'entity',
'url': 'https://rdap.arin.net/registry/entity/GOGL',
'whois_server': 'whois.arin.net'
'whois_server': 'whois.arin.net',
'address': {
'po_box': '',
'ext_address': '',
'street_address': '',
'locality': '',
'region': '',
'postal_code': '',
'country': '',
}
}
]
)
Expand All @@ -362,7 +443,16 @@ def test_ip_response_parser(self):
'type': 'entity',
'url': 'https://rdap.arin.net/registry/entity/ZG39-ARIN',
'tel': '+1-650-253-0000',
'whois_server': 'whois.arin.net'
'whois_server': 'whois.arin.net',
'address': {
'po_box': '',
'ext_address': '',
'street_address': '',
'locality': '',
'region': '',
'postal_code': '',
'country': '',
}
}
]
)
Expand All @@ -376,7 +466,16 @@ def test_ip_response_parser(self):
'type': 'entity',
'url': 'https://rdap.arin.net/registry/entity/ZG39-ARIN',
'tel': '+1-650-253-0000',
'whois_server': 'whois.arin.net'
'whois_server': 'whois.arin.net',
'address': {
'po_box': '',
'ext_address': '',
'street_address': '',
'locality': '',
'region': '',
'postal_code': '',
'country': '',
}
}
]
)
Expand All @@ -390,7 +489,16 @@ def test_ip_response_parser(self):
'name': 'Abuse',
'email': '[email protected]',
'tel': '+1-650-253-0000',
'rir': 'arin'
'rir': 'arin',
'address': {
'po_box': '',
'ext_address': '',
'street_address': '',
'locality': '',
'region': '',
'postal_code': '',
'country': '',
}
},
{
'handle': 'ZG39-ARIN',
Expand All @@ -400,7 +508,16 @@ def test_ip_response_parser(self):
'name': 'Google LLC',
'email': '[email protected]',
'tel': '+1-650-253-0000',
'rir': 'arin'
'rir': 'arin',
'address': {
'po_box': '',
'ext_address': '',
'street_address': '',
'locality': '',
'region': '',
'postal_code': '',
'country': '',
}
}
]
)
Expand Down Expand Up @@ -432,7 +549,16 @@ def test_ip_response_parser(self):
'rir': 'arin',
'type': 'entity',
'url': 'https://rdap.arin.net/registry/entity/GOGL',
'whois_server': 'whois.arin.net'
'whois_server': 'whois.arin.net',
'address': {
'po_box': '',
'ext_address': '',
'street_address': '',
'locality': '',
'region': '',
'postal_code': '',
'country': '',
}
}
]
)
Expand Down Expand Up @@ -464,7 +590,16 @@ def test_entity_response_parser(self):
'type': 'entity',
'url': 'https://rdap.arin.net/registry/entity/GTS7-ARIN',
'tel': '+1-519-254-5115',
'whois_server': 'whois.arin.net'
'whois_server': 'whois.arin.net',
'address': {
'po_box': '',
'ext_address': '',
'street_address': '',
'locality': '',
'region': '',
'postal_code': '',
'country': '',
}
}
]
)
22 changes: 22 additions & 0 deletions tests/test_public_interface.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,17 @@ def test_domain_interface_1(self):
)
self.assertTrue(isinstance(resp["entities"]["registrar"], list))
self.assertTrue(len(resp["entities"]["registrar"]) > 0)
self.assertTrue(isinstance(resp["entities"]["registrant"], list))
self.assertTrue(len(resp["entities"]["registrant"]) > 0)
self.assertEqual(resp["entities"]["registrant"][0]['address'], {
'country': "US",
'ext_address': '',
'locality': '',
'po_box': '',
'postal_code': '',
'region': 'CA',
'street_address': '',
})

@responses.activate
# @_recorder.record(file_path=RESPONSES / 'ip-v4-1.yaml')
Expand Down Expand Up @@ -314,6 +325,17 @@ async def test_domain_interface_1(self):
)
self.assertTrue(isinstance(resp["entities"]["registrar"], list))
self.assertTrue(len(resp["entities"]["registrar"]) > 0)
self.assertTrue(isinstance(resp["entities"]["registrant"], list))
self.assertTrue(len(resp["entities"]["registrant"]) > 0)
self.assertEqual(resp["entities"]["registrant"][0]['address'], {
'country': "US",
'ext_address': '',
'locality': '',
'po_box': '',
'postal_code': '',
'region': 'CA',
'street_address': '',
})

@pytest.mark.asyncio
@pytest.mark.usefixtures("mock_httpx")
Expand Down
33 changes: 32 additions & 1 deletion whoisit/parser.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
IPv4Network,
IPv6Network
)
from typing import Optional
from typing import Optional, List
from typing_extensions import TypedDict

from dateutil.parser import parse as dateutil_parse
Expand All @@ -25,10 +25,31 @@ def clean(s):
return s.strip()


def clean_address(a):
if a is None:
a = ''
if isinstance(a, list):
a = ' '.join(a)
if not isinstance(a, str):
a = str(a)
return a.strip()


class VCardArrayDataDict(TypedDict, total=False):
name: str
email: str
tel: str
address: List[str]


class VCardArrayAddressDataDict(TypedDict, total=False):
po_box: str
ext_address: str
street_address: str
locality: str
region: str
postal_code: str
country: str


class Parser:
Expand Down Expand Up @@ -95,6 +116,16 @@ def parse_vcard_array(self, vcard) -> Optional[VCardArrayDataDict]:
v_card_array_data_dict["email"] = clean(entry_label)
elif entry_field == 'tel':
v_card_array_data_dict["tel"] = clean(entry_label)
elif entry_field == 'adr' and isinstance(entry_label, list) and len(entry_label) == 7:
v_card_array_data_dict['address'] = VCardArrayAddressDataDict(
po_box=clean_address(entry_label[0]),
ext_address=clean_address(entry_label[1]),
street_address=clean_address(entry_label[2]),
locality=clean_address(entry_label[3]),
region=clean_address(entry_label[4]),
postal_code=clean_address(entry_label[5]),
country=clean_address(entry_label[6])
)

return v_card_array_data_dict or None

Expand Down