import http import logging import typing import requests import unidecode if typing.TYPE_CHECKING: from .cache import Cache class TLDError(Exception): pass class TLDList: __slots__ = ["__cache", "__remote_uri", "logger", "tlds"] def __init__(self, cache: Cache, remote_uri: str) -> None: self.logger: logging.Logger = logging.getLogger(self.__class__.__name__) self.tlds: list[str] | None = None self.__cache = cache self.__remote_uri = remote_uri def __load_remote(self) -> list[str]: response = requests.get(self.__remote_uri, timeout=5) if response.status_code != http.HTTPStatus.OK: msg = f"Failed to read {self.__remote_uri}." raise TLDError(msg) raw_lines = response.content.decode().split("\n") out_lines = [] for line in raw_lines: stripped_line = line.strip() if ( len(stripped_line) and not stripped_line.startswith("//") and not stripped_line.startswith("*") and unidecode.unidecode(stripped_line) == stripped_line ): if "." in stripped_line: for other in out_lines: if stripped_line.endswith(other): break else: out_lines += [stripped_line] else: out_lines += [stripped_line] return out_lines def load(self) -> None: self.tlds = self.__cache.get_or_set("tld_list", self.__load_remote) self.logger.info("Loaded %d TLD entries", len(self.tlds)) def __enter__(self) -> typing.Self: self.load() return self def __exit__(self, *_: object) -> None: pass