fix: use fixed tld list

This commit is contained in:
2026-06-16 10:56:12 +02:00
parent 0a8775dfc9
commit 873303661a
9 changed files with 2651 additions and 1516 deletions
+1 -3
View File
@@ -2,7 +2,6 @@ import logging
import sys
from cool_domain import PKG_NAME, PKG_VERSION
from cool_domain.cache import Cache
from cool_domain.constants import LANGS
from cool_domain.factory import Factory
from cool_domain.logs import setup_logs
@@ -18,8 +17,7 @@ def main() -> int:
langs = LANGS
langs["custom"] = WordList("custom", params.custom_words)
with (
Cache(filepath=params.cache_file, no_load=params.cache) as cache,
TLDList(cache=cache, remote_uri=params.tld_list_uri) as tld_list,
TLDList() as tld_list,
langs[params.lang] as word_list,
):
factory = Factory(tld_list=tld_list, word_list=word_list, words=params.words)
-48
View File
@@ -1,48 +0,0 @@
import json
import pathlib
import typing
T = typing.TypeVar("T")
class Cache:
__slots__ = ["__data", "__filepath", "__no_load"]
def __init__(self, filepath: str, no_load: bool) -> None: # noqa: FBT001
self.__filepath = pathlib.Path(filepath)
self.__no_load = no_load
self.__data: dict[str, typing.Any] | None = None
def load(self) -> None:
if not self.__no_load and self.__filepath.exists():
self.__data = json.loads(self.__filepath.read_text())
else:
self.__data = {}
def write(self) -> None:
self.__filepath.write_text(json.dumps(self.__data))
def __enter__(self) -> typing.Self:
self.load()
return self
def __exit__(self, *_: object) -> None:
self.write()
def has(self, key: str) -> bool:
return self.__data is not None and key in self.__data
def get(self, key: str) -> typing.Any:
if self.__data is not None and key in self.__data:
return self.__data[key]
return None
def set(self, key: str, value: typing.Any) -> None:
if self.__data is not None:
self.__data[key] = value
self.write()
def get_or_set(self, key: str, callback: typing.Callable[[], T]) -> T:
if not self.has(key):
self.set(key, callback())
return self.get(key)
File diff suppressed because it is too large Load Diff
-24
View File
@@ -1,8 +1,6 @@
import argparse
import dataclasses
import os
import pathlib
import tempfile
import typing
from cool_domain import PKG_NAME
@@ -18,8 +16,6 @@ class Parameters:
words: int = 1
dig: bool = True
lang: str = "english"
cache_file: str = str(pathlib.Path(tempfile.gettempdir()) / ".cool-domain-cache")
tld_list_uri: str = "https://data.iana.org/TLD/tlds-alpha-by-domain.txt"
custom_words: str = "words.txt"
@classmethod
@@ -131,25 +127,5 @@ def parse_parameters(args: typing.Sequence[str]) -> Parameters:
parser.add_argument(
"--quiet", action=argparse.BooleanOptionalAction, default=default_values.quiet
)
__add_arg_bool(
parser,
"--cache",
default=default_values.cache,
help_txt="use cache",
)
__add_arg_str(
parser,
"--cache-file",
env_var="CACHE_FILE",
default=default_values.cache_file,
help_txt="cache file location",
)
__add_arg_str(
parser,
"--tld-list-uri",
env_var="TLD_LIST_URI",
default=default_values.tld_list_uri,
help_txt="Top Level Domains list",
)
parsed_args = parser.parse_args(args)
return Parameters.from_namespace(parsed_args)
+2598
View File
File diff suppressed because it is too large Load Diff
+9 -31
View File
@@ -1,51 +1,29 @@
import http
import logging
import pathlib
import typing
import requests
import unidecode
from cool_domain.cache import Cache
from cool_domain.constants import BLACKLIST
class TLDError(Exception):
pass
class TLDList:
__slots__ = ["__cache", "__remote_uri", "logger", "tlds"]
__slots__ = ["logger", "tlds"]
def __init__(self, cache: Cache, remote_uri: str) -> None:
def __init__(self) -> None:
self.logger: logging.Logger = logging.getLogger(self.__class__.__name__)
self.tlds: list[str] | None = None
self.__cache = cache
self.__remote_uri = remote_uri
def __load_remote(self) -> list[str]:
response = requests.get(self.__remote_uri, timeout=5)
if response.status_code != http.HTTPStatus.OK:
msg = f"Failed to read {self.__remote_uri}."
raise TLDError(msg)
raw_lines = response.content.decode().split("\n")
def __load_file(self) -> list[str]:
out_lines = []
for line in raw_lines:
stripped_line = line.strip().lower()
if (
len(stripped_line)
and not stripped_line.startswith("//")
and not stripped_line.startswith("*")
and not stripped_line.startswith("#")
and not stripped_line.startswith("xn--")
and "." not in stripped_line
and unidecode.unidecode(stripped_line) == stripped_line
and stripped_line not in BLACKLIST
):
out_lines += [stripped_line]
with (pathlib.Path(__file__).parent / "tld.txt").open() as file:
for line in file:
if len(line.strip()):
out_lines += [line.strip()]
return out_lines
def load(self) -> None:
self.tlds = self.__cache.get_or_set("tld_list", self.__load_remote)
self.tlds = self.__load_file()
self.logger.info("Loaded %d TLD entries", len(self.tlds))
def __enter__(self) -> typing.Self: