fix: use fixed tld list
This commit is contained in:
@@ -2,7 +2,6 @@ import logging
|
||||
import sys
|
||||
|
||||
from cool_domain import PKG_NAME, PKG_VERSION
|
||||
from cool_domain.cache import Cache
|
||||
from cool_domain.constants import LANGS
|
||||
from cool_domain.factory import Factory
|
||||
from cool_domain.logs import setup_logs
|
||||
@@ -18,8 +17,7 @@ def main() -> int:
|
||||
langs = LANGS
|
||||
langs["custom"] = WordList("custom", params.custom_words)
|
||||
with (
|
||||
Cache(filepath=params.cache_file, no_load=params.cache) as cache,
|
||||
TLDList(cache=cache, remote_uri=params.tld_list_uri) as tld_list,
|
||||
TLDList() as tld_list,
|
||||
langs[params.lang] as word_list,
|
||||
):
|
||||
factory = Factory(tld_list=tld_list, word_list=word_list, words=params.words)
|
||||
|
||||
@@ -1,48 +0,0 @@
|
||||
import json
|
||||
import pathlib
|
||||
import typing
|
||||
|
||||
T = typing.TypeVar("T")
|
||||
|
||||
|
||||
class Cache:
|
||||
__slots__ = ["__data", "__filepath", "__no_load"]
|
||||
|
||||
def __init__(self, filepath: str, no_load: bool) -> None: # noqa: FBT001
|
||||
self.__filepath = pathlib.Path(filepath)
|
||||
self.__no_load = no_load
|
||||
self.__data: dict[str, typing.Any] | None = None
|
||||
|
||||
def load(self) -> None:
|
||||
if not self.__no_load and self.__filepath.exists():
|
||||
self.__data = json.loads(self.__filepath.read_text())
|
||||
else:
|
||||
self.__data = {}
|
||||
|
||||
def write(self) -> None:
|
||||
self.__filepath.write_text(json.dumps(self.__data))
|
||||
|
||||
def __enter__(self) -> typing.Self:
|
||||
self.load()
|
||||
return self
|
||||
|
||||
def __exit__(self, *_: object) -> None:
|
||||
self.write()
|
||||
|
||||
def has(self, key: str) -> bool:
|
||||
return self.__data is not None and key in self.__data
|
||||
|
||||
def get(self, key: str) -> typing.Any:
|
||||
if self.__data is not None and key in self.__data:
|
||||
return self.__data[key]
|
||||
return None
|
||||
|
||||
def set(self, key: str, value: typing.Any) -> None:
|
||||
if self.__data is not None:
|
||||
self.__data[key] = value
|
||||
self.write()
|
||||
|
||||
def get_or_set(self, key: str, callback: typing.Callable[[], T]) -> T:
|
||||
if not self.has(key):
|
||||
self.set(key, callback())
|
||||
return self.get(key)
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,8 +1,6 @@
|
||||
import argparse
|
||||
import dataclasses
|
||||
import os
|
||||
import pathlib
|
||||
import tempfile
|
||||
import typing
|
||||
|
||||
from cool_domain import PKG_NAME
|
||||
@@ -18,8 +16,6 @@ class Parameters:
|
||||
words: int = 1
|
||||
dig: bool = True
|
||||
lang: str = "english"
|
||||
cache_file: str = str(pathlib.Path(tempfile.gettempdir()) / ".cool-domain-cache")
|
||||
tld_list_uri: str = "https://data.iana.org/TLD/tlds-alpha-by-domain.txt"
|
||||
custom_words: str = "words.txt"
|
||||
|
||||
@classmethod
|
||||
@@ -131,25 +127,5 @@ def parse_parameters(args: typing.Sequence[str]) -> Parameters:
|
||||
parser.add_argument(
|
||||
"--quiet", action=argparse.BooleanOptionalAction, default=default_values.quiet
|
||||
)
|
||||
__add_arg_bool(
|
||||
parser,
|
||||
"--cache",
|
||||
default=default_values.cache,
|
||||
help_txt="use cache",
|
||||
)
|
||||
__add_arg_str(
|
||||
parser,
|
||||
"--cache-file",
|
||||
env_var="CACHE_FILE",
|
||||
default=default_values.cache_file,
|
||||
help_txt="cache file location",
|
||||
)
|
||||
__add_arg_str(
|
||||
parser,
|
||||
"--tld-list-uri",
|
||||
env_var="TLD_LIST_URI",
|
||||
default=default_values.tld_list_uri,
|
||||
help_txt="Top Level Domains list",
|
||||
)
|
||||
parsed_args = parser.parse_args(args)
|
||||
return Parameters.from_namespace(parsed_args)
|
||||
|
||||
+2598
File diff suppressed because it is too large
Load Diff
+9
-31
@@ -1,51 +1,29 @@
|
||||
import http
|
||||
import logging
|
||||
import pathlib
|
||||
import typing
|
||||
|
||||
import requests
|
||||
import unidecode
|
||||
|
||||
from cool_domain.cache import Cache
|
||||
from cool_domain.constants import BLACKLIST
|
||||
|
||||
|
||||
class TLDError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class TLDList:
|
||||
__slots__ = ["__cache", "__remote_uri", "logger", "tlds"]
|
||||
__slots__ = ["logger", "tlds"]
|
||||
|
||||
def __init__(self, cache: Cache, remote_uri: str) -> None:
|
||||
def __init__(self) -> None:
|
||||
self.logger: logging.Logger = logging.getLogger(self.__class__.__name__)
|
||||
self.tlds: list[str] | None = None
|
||||
self.__cache = cache
|
||||
self.__remote_uri = remote_uri
|
||||
|
||||
def __load_remote(self) -> list[str]:
|
||||
response = requests.get(self.__remote_uri, timeout=5)
|
||||
if response.status_code != http.HTTPStatus.OK:
|
||||
msg = f"Failed to read {self.__remote_uri}."
|
||||
raise TLDError(msg)
|
||||
raw_lines = response.content.decode().split("\n")
|
||||
def __load_file(self) -> list[str]:
|
||||
out_lines = []
|
||||
for line in raw_lines:
|
||||
stripped_line = line.strip().lower()
|
||||
if (
|
||||
len(stripped_line)
|
||||
and not stripped_line.startswith("//")
|
||||
and not stripped_line.startswith("*")
|
||||
and not stripped_line.startswith("#")
|
||||
and not stripped_line.startswith("xn--")
|
||||
and "." not in stripped_line
|
||||
and unidecode.unidecode(stripped_line) == stripped_line
|
||||
and stripped_line not in BLACKLIST
|
||||
):
|
||||
out_lines += [stripped_line]
|
||||
with (pathlib.Path(__file__).parent / "tld.txt").open() as file:
|
||||
for line in file:
|
||||
if len(line.strip()):
|
||||
out_lines += [line.strip()]
|
||||
return out_lines
|
||||
|
||||
def load(self) -> None:
|
||||
self.tlds = self.__cache.get_or_set("tld_list", self.__load_remote)
|
||||
self.tlds = self.__load_file()
|
||||
self.logger.info("Loaded %d TLD entries", len(self.tlds))
|
||||
|
||||
def __enter__(self) -> typing.Self:
|
||||
|
||||
Reference in New Issue
Block a user