fix: use fixed tld list
This commit is contained in:
+9
-31
@@ -1,51 +1,29 @@
|
||||
import http
|
||||
import logging
|
||||
import pathlib
|
||||
import typing
|
||||
|
||||
import requests
|
||||
import unidecode
|
||||
|
||||
from cool_domain.cache import Cache
|
||||
from cool_domain.constants import BLACKLIST
|
||||
|
||||
|
||||
class TLDError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class TLDList:
|
||||
__slots__ = ["__cache", "__remote_uri", "logger", "tlds"]
|
||||
__slots__ = ["logger", "tlds"]
|
||||
|
||||
def __init__(self, cache: Cache, remote_uri: str) -> None:
|
||||
def __init__(self) -> None:
|
||||
self.logger: logging.Logger = logging.getLogger(self.__class__.__name__)
|
||||
self.tlds: list[str] | None = None
|
||||
self.__cache = cache
|
||||
self.__remote_uri = remote_uri
|
||||
|
||||
def __load_remote(self) -> list[str]:
|
||||
response = requests.get(self.__remote_uri, timeout=5)
|
||||
if response.status_code != http.HTTPStatus.OK:
|
||||
msg = f"Failed to read {self.__remote_uri}."
|
||||
raise TLDError(msg)
|
||||
raw_lines = response.content.decode().split("\n")
|
||||
def __load_file(self) -> list[str]:
|
||||
out_lines = []
|
||||
for line in raw_lines:
|
||||
stripped_line = line.strip().lower()
|
||||
if (
|
||||
len(stripped_line)
|
||||
and not stripped_line.startswith("//")
|
||||
and not stripped_line.startswith("*")
|
||||
and not stripped_line.startswith("#")
|
||||
and not stripped_line.startswith("xn--")
|
||||
and "." not in stripped_line
|
||||
and unidecode.unidecode(stripped_line) == stripped_line
|
||||
and stripped_line not in BLACKLIST
|
||||
):
|
||||
out_lines += [stripped_line]
|
||||
with (pathlib.Path(__file__).parent / "tld.txt").open() as file:
|
||||
for line in file:
|
||||
if len(line.strip()):
|
||||
out_lines += [line.strip()]
|
||||
return out_lines
|
||||
|
||||
def load(self) -> None:
|
||||
self.tlds = self.__cache.get_or_set("tld_list", self.__load_remote)
|
||||
self.tlds = self.__load_file()
|
||||
self.logger.info("Loaded %d TLD entries", len(self.tlds))
|
||||
|
||||
def __enter__(self) -> typing.Self:
|
||||
|
||||
Reference in New Issue
Block a user