57 lines
1.7 KiB
Python
57 lines
1.7 KiB
Python
import http
|
|
import logging
|
|
import typing
|
|
|
|
import requests
|
|
import unidecode
|
|
|
|
from cool_domain.cache import Cache
|
|
from cool_domain.constants import BLACKLIST
|
|
|
|
|
|
class TLDError(Exception):
|
|
pass
|
|
|
|
|
|
class TLDList:
|
|
__slots__ = ["__cache", "__remote_uri", "logger", "tlds"]
|
|
|
|
def __init__(self, cache: Cache, remote_uri: str) -> None:
|
|
self.logger: logging.Logger = logging.getLogger(self.__class__.__name__)
|
|
self.tlds: list[str] | None = None
|
|
self.__cache = cache
|
|
self.__remote_uri = remote_uri
|
|
|
|
def __load_remote(self) -> list[str]:
|
|
response = requests.get(self.__remote_uri, timeout=5)
|
|
if response.status_code != http.HTTPStatus.OK:
|
|
msg = f"Failed to read {self.__remote_uri}."
|
|
raise TLDError(msg)
|
|
raw_lines = response.content.decode().split("\n")
|
|
out_lines = []
|
|
for line in raw_lines:
|
|
stripped_line = line.strip().lower()
|
|
if (
|
|
len(stripped_line)
|
|
and not stripped_line.startswith("//")
|
|
and not stripped_line.startswith("*")
|
|
and not stripped_line.startswith("#")
|
|
and not stripped_line.startswith("xn--")
|
|
and "." not in stripped_line
|
|
and unidecode.unidecode(stripped_line) == stripped_line
|
|
and stripped_line not in BLACKLIST
|
|
):
|
|
out_lines += [stripped_line]
|
|
return out_lines
|
|
|
|
def load(self) -> None:
|
|
self.tlds = self.__cache.get_or_set("tld_list", self.__load_remote)
|
|
self.logger.info("Loaded %d TLD entries", len(self.tlds))
|
|
|
|
def __enter__(self) -> typing.Self:
|
|
self.load()
|
|
return self
|
|
|
|
def __exit__(self, *_: object) -> None:
|
|
pass
|