fix: use fixed tld list

This commit is contained in:
2026-06-16 10:56:12 +02:00
parent 0a8775dfc9
commit 873303661a
9 changed files with 2651 additions and 1516 deletions
+9 -31
View File
@@ -1,51 +1,29 @@
import http
import logging
import pathlib
import typing
import requests
import unidecode
from cool_domain.cache import Cache
from cool_domain.constants import BLACKLIST
class TLDError(Exception):
pass
class TLDList:
__slots__ = ["__cache", "__remote_uri", "logger", "tlds"]
__slots__ = ["logger", "tlds"]
def __init__(self, cache: Cache, remote_uri: str) -> None:
def __init__(self) -> None:
self.logger: logging.Logger = logging.getLogger(self.__class__.__name__)
self.tlds: list[str] | None = None
self.__cache = cache
self.__remote_uri = remote_uri
def __load_remote(self) -> list[str]:
response = requests.get(self.__remote_uri, timeout=5)
if response.status_code != http.HTTPStatus.OK:
msg = f"Failed to read {self.__remote_uri}."
raise TLDError(msg)
raw_lines = response.content.decode().split("\n")
def __load_file(self) -> list[str]:
out_lines = []
for line in raw_lines:
stripped_line = line.strip().lower()
if (
len(stripped_line)
and not stripped_line.startswith("//")
and not stripped_line.startswith("*")
and not stripped_line.startswith("#")
and not stripped_line.startswith("xn--")
and "." not in stripped_line
and unidecode.unidecode(stripped_line) == stripped_line
and stripped_line not in BLACKLIST
):
out_lines += [stripped_line]
with (pathlib.Path(__file__).parent / "tld.txt").open() as file:
for line in file:
if len(line.strip()):
out_lines += [line.strip()]
return out_lines
def load(self) -> None:
self.tlds = self.__cache.get_or_set("tld_list", self.__load_remote)
self.tlds = self.__load_file()
self.logger.info("Loaded %d TLD entries", len(self.tlds))
def __enter__(self) -> typing.Self: