From 3100e6fa2021d3720196450d7d5d692b55eab098 Mon Sep 17 00:00:00 2001 From: Klemek Date: Wed, 21 Apr 2021 11:26:37 +0200 Subject: [PATCH 01/12] mobile/mention to fix @invalid-user bug --- README.md | 3 +++ src/main.py | 2 +- src/scanners/scanner.py | 15 ++++++++++++--- src/utils/utils.py | 1 + 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index ad239ac..ebde127 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,7 @@ * all/everyone - include bots messages * fast: only read cache * fresh: does not read cache + * mobile/mention: mentions users (fix @invalid-user bug) (Sample dates: 2020 / 2021-11 / 2021-06-28 / 2020-06-28T23:00 / today / week / 8days / 1y) ``` @@ -109,6 +110,8 @@ python3 src/main.py ## Changelog +* **v1.14** + * `mobile/mention` arg to fix mobile bug * **v1.13** * improved scan `%words` * remove old and unused logs at start and guild leaving diff --git a/src/main.py b/src/main.py index d2278e5..53d4472 100644 --- a/src/main.py +++ b/src/main.py @@ -33,7 +33,7 @@ emojis.load_emojis() bot = Bot( "Discord Analyst", - "1.13", + "1.14", alias="%", ) diff --git a/src/scanners/scanner.py b/src/scanners/scanner.py index b4a96f1..eb4d30c 100644 --- a/src/scanners/scanner.py +++ b/src/scanners/scanner.py @@ -79,7 +79,9 @@ class Scanner(ABC): ) return if ( - arg not in self.valid_args + ["me", "here", "fast", "fresh"] + arg + not in self.valid_args + + ["me", "here", "fast", "fresh", "mobile", "mention"] and (not arg.isdigit() or not self.has_digit_args) and arg not in str_channel_mentions and arg not in str_mentions @@ -119,6 +121,8 @@ class Scanner(ABC): self.members += [message.author] self.raw_members += [message.author.id] + self.mention_users = "mention" in args or "mobile" in args + if not await self.init(message, *args): return @@ -214,13 +218,18 @@ class Scanner(ABC): logging.info(f"scan {guild.id} > results in {delta(t0):,}ms") response = "" first = True + allowed_mentions = ( + discord.AllowedMentions.all() + if self.mention_users + else discord.AllowedMentions.none() + ) for r in results: if r: if len(response + "\n" + r) > 2000: await message.channel.send( response, reference=message if first else None, - allowed_mentions=discord.AllowedMentions.none(), + allowed_mentions=allowed_mentions, ) first = False response = "" @@ -229,7 +238,7 @@ class Scanner(ABC): await message.channel.send( response, reference=message if first else None, - allowed_mentions=discord.AllowedMentions.none(), + allowed_mentions=allowed_mentions, ) # Delete custom progress message await progress.delete() diff --git a/src/utils/utils.py b/src/utils/utils.py index a439ffd..52d0bd1 100644 --- a/src/utils/utils.py +++ b/src/utils/utils.py @@ -18,6 +18,7 @@ COMMON_HELP_ARGS = [ " - filter before ", "fast - only read cache", "fresh - does not read cache (long)", + "mobile/mention - mentions users (fix @invalid-user bug)", ] From 7fad35a4b33a06e3502aa237b7149922b40a5f38 Mon Sep 17 00:00:00 2001 From: Klemek Date: Wed, 21 Apr 2021 20:14:06 +0200 Subject: [PATCH 02/12] command_cache for %repeat and %mobile --- README.md | 3 +++ src/main.py | 14 +++++++++++- src/scanners/__init__.py | 1 + src/scanners/scanner.py | 9 +++++++- src/utils/command_cache.py | 45 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 70 insertions(+), 2 deletions(-) create mode 100644 src/utils/command_cache.py diff --git a/README.md b/README.md index ebde127..d699661 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,8 @@ * %first - read first message * %rand - read a random message * %last - read last message +* %repeat - repeat last analysis (adding supplied arguments) +* %mobile - fix @invalid-user for last command but mentions users * %gdpr - displays GDPR information * %emojis - rank emotes by their usage * arguments: @@ -112,6 +114,7 @@ python3 src/main.py * **v1.14** * `mobile/mention` arg to fix mobile bug + * `%repeat`, `%mobile` to repeat commands * **v1.13** * improved scan `%words` * remove old and unused logs at start and guild leaving diff --git a/src/main.py b/src/main.py index 53d4472..fb3c194 100644 --- a/src/main.py +++ b/src/main.py @@ -6,7 +6,7 @@ if sys.version_info < (3, 7): print("Please upgrade your Python version to 3.7.0 or higher") sys.exit(1) -from utils import emojis, gdpr +from utils import emojis, gdpr, command_cache from scanners import ( EmotesScanner, FullScanner, @@ -71,6 +71,18 @@ bot.register_command( "words: (BETA) rank words by their usage", WordsScanner.help(), ) +bot.register_command( + "repeat", + command_cache.repeat, + "repeat: repeat last analysis (adding supplied arguments)", + "```\n%repeat: repeat last analysis (adding supplied arguments)\n```", +) +bot.register_command( + "mobile", + lambda *args: command_cache.repeat(*args, add_args=["mobile"]), + "mobile: fix @invalid-user for last command but mentions users", + "```\n%mobile: fix @invalid-user for last command but mentions users\n```", +) bot.register_command( "last", lambda *args: LastScanner().compute(*args), diff --git a/src/scanners/__init__.py b/src/scanners/__init__.py index ed9141d..8ac7b97 100644 --- a/src/scanners/__init__.py +++ b/src/scanners/__init__.py @@ -1,3 +1,4 @@ +from .scanner import Scanner from .emotes_scanner import EmotesScanner from .frequency_scanner import FrequencyScanner from .composition_scanner import CompositionScanner diff --git a/src/scanners/scanner.py b/src/scanners/scanner.py index eb4d30c..934c334 100644 --- a/src/scanners/scanner.py +++ b/src/scanners/scanner.py @@ -14,6 +14,7 @@ from utils import ( ISO8601_REGEX, RELATIVE_REGEX, parse_time, + command_cache, ) from logs import ( GuildLogs, @@ -48,7 +49,11 @@ class Scanner(ABC): self.chan_count = 0 async def compute( - self, client: discord.client, message: discord.Message, *args: str + self, + client: discord.client, + message: discord.Message, + *args: str, + other_mentions: List[str] = [], ): args = list(args) guild = message.guild @@ -85,6 +90,7 @@ class Scanner(ABC): and (not arg.isdigit() or not self.has_digit_args) and arg not in str_channel_mentions and arg not in str_mentions + and arg not in other_mentions and not skip_check ): await message.channel.send( @@ -240,6 +246,7 @@ class Scanner(ABC): reference=message if first else None, allowed_mentions=allowed_mentions, ) + command_cache.cache(self, message, args) # Delete custom progress message await progress.delete() diff --git a/src/utils/command_cache.py b/src/utils/command_cache.py new file mode 100644 index 0000000..4cdbf7b --- /dev/null +++ b/src/utils/command_cache.py @@ -0,0 +1,45 @@ +from typing import List +import logging +import discord + +from scanners import Scanner + +command_cache = {} + + +def cache(scanner: Scanner, message: discord.Message, args: List[str]): + id = message.channel.id + command_cache[id] = ( + type(scanner), + list(args), + [str(channel.id) for channel in message.channel_mentions] + + [str(member.id) for member in message.mentions], + ) + + +async def repeat( + client: discord.client, + message: discord.Message, + *args: str, + add_args: List[str] = [], +): + if "help" in args: + await client.bot.help(client, message, "help", args[0]) + return + id = message.channel.id + if id not in command_cache: + await message.channel.send( + "No command to repeat on this channel (type %help for more info)", + reference=message, + ) + return + ( + scannerType, + original_args, + original_mentions, + ) = command_cache[id] + args = original_args + add_args + list(args[1:]) + ["fast"] + logging.info(f"repeating {args}") + await scannerType().compute( + client, message, *args, other_mentions=original_mentions + ) From 634f34fb542840c24088ac09c20336f40ec90bb6 Mon Sep 17 00:00:00 2001 From: Klemek Date: Wed, 21 Apr 2021 20:14:15 +0200 Subject: [PATCH 03/12] better help for %gdpr --- src/utils/gdpr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/gdpr.py b/src/utils/gdpr.py index 5ae85c6..d913f5c 100644 --- a/src/utils/gdpr.py +++ b/src/utils/gdpr.py @@ -52,7 +52,7 @@ async def process(client: discord.client, message: discord.Message, *args: str): elif len(args) > 2: await message.channel.send(f"Too many arguments", reference=message) elif args[1] == "help": - await message.channel.send(HELP, reference=message) + await client.bot.help(client, message, "help", args[0]) elif args[1] in ["agree", "accept"]: GuildLogs.init_log(message.channel.guild) await message.channel.send(AGREE_TEXT, reference=message) From 6afb05148d27d0a201ff616007cb7ffa38d72cda Mon Sep 17 00:00:00 2001 From: Klemek Date: Wed, 21 Apr 2021 20:22:36 +0200 Subject: [PATCH 04/12] scanner exception handling --- README.md | 1 + src/scanners/scanner.py | 341 +++++++++++++++++++++------------------- 2 files changed, 178 insertions(+), 164 deletions(-) diff --git a/README.md b/README.md index d699661..eac16f3 100644 --- a/README.md +++ b/README.md @@ -115,6 +115,7 @@ python3 src/main.py * **v1.14** * `mobile/mention` arg to fix mobile bug * `%repeat`, `%mobile` to repeat commands + * bug fix * **v1.13** * improved scan `%words` * remove old and unused logs at start and guild leaving diff --git a/src/scanners/scanner.py b/src/scanners/scanner.py index 934c334..7ea4cda 100644 --- a/src/scanners/scanner.py +++ b/src/scanners/scanner.py @@ -57,198 +57,211 @@ class Scanner(ABC): ): args = list(args) guild = message.guild - with GuildLogs(guild) as logs: - # If "%cmd help" redirect to "%help cmd" - if "help" in args: - await client.bot.help(client, message, "help", args[0]) - return + progress = None + try: + with GuildLogs(guild) as logs: + # If "%cmd help" redirect to "%help cmd" + if "help" in args: + await client.bot.help(client, message, "help", args[0]) + return - # check args validity - str_channel_mentions = [ - str(channel.id) for channel in message.channel_mentions - ] - str_mentions = [str(member.id) for member in message.mentions] - dates = [] - for i, arg in enumerate(args[1:]): - skip_check = False - if re.match(r"^<@!?\d+>$", arg): - arg = arg[3:-1] if "!" in arg else arg[2:-1] - elif re.match(r"^<#!?\d+>$", arg): - arg = arg[3:-1] if "!" in arg else arg[2:-1] - elif re.match(ISO8601_REGEX, arg) or re.match(RELATIVE_REGEX, arg): - dates += [parse_time(arg)] - skip_check = True - if len(dates) > 2: + # check args validity + str_channel_mentions = [ + str(channel.id) for channel in message.channel_mentions + ] + str_mentions = [str(member.id) for member in message.mentions] + dates = [] + for i, arg in enumerate(args[1:]): + skip_check = False + if re.match(r"^<@!?\d+>$", arg): + arg = arg[3:-1] if "!" in arg else arg[2:-1] + elif re.match(r"^<#!?\d+>$", arg): + arg = arg[3:-1] if "!" in arg else arg[2:-1] + elif re.match(ISO8601_REGEX, arg) or re.match(RELATIVE_REGEX, arg): + dates += [parse_time(arg)] + skip_check = True + if len(dates) > 2: + await message.channel.send( + f"Too many date arguments: `{arg}`", reference=message + ) + return + if ( + arg + not in self.valid_args + + ["me", "here", "fast", "fresh", "mobile", "mention"] + and (not arg.isdigit() or not self.has_digit_args) + and arg not in str_channel_mentions + and arg not in str_mentions + and arg not in other_mentions + and not skip_check + ): await message.channel.send( - f"Too many date arguments: `{arg}`", reference=message + f"Unrecognized argument: `{arg}`", reference=message ) return - if ( - arg - not in self.valid_args - + ["me", "here", "fast", "fresh", "mobile", "mention"] - and (not arg.isdigit() or not self.has_digit_args) - and arg not in str_channel_mentions - and arg not in str_mentions - and arg not in other_mentions - and not skip_check - ): + + self.start_date = None if len(dates) < 1 else min(dates) + self.stop_date = None if len(dates) < 2 else max(dates) + + if self.start_date is not None and self.start_date > datetime.now(): await message.channel.send( - f"Unrecognized argument: `{arg}`", reference=message + f"Start date is after today", reference=message ) return - self.start_date = None if len(dates) < 1 else min(dates) - self.stop_date = None if len(dates) < 2 else max(dates) + # Get selected channels or all of them if no channel arguments + self.channels = no_duplicate(message.channel_mentions) - if self.start_date is not None and self.start_date > datetime.now(): - await message.channel.send( - f"Start date is after today", reference=message - ) - return + # transform the "here" arg + if "here" in args: + self.channels += [message.channel] - # Get selected channels or all of them if no channel arguments - self.channels = no_duplicate(message.channel_mentions) + self.full = len(self.channels) == 0 + if self.full: + self.channels = guild.text_channels - # transform the "here" arg - if "here" in args: - self.channels += [message.channel] + # Get selected members + self.members = no_duplicate(message.mentions) + self.raw_members = no_duplicate(message.raw_mentions) - self.full = len(self.channels) == 0 - if self.full: - self.channels = guild.text_channels + # transform the "me" arg + if "me" in args: + self.members += [message.author] + self.raw_members += [message.author.id] - # Get selected members - self.members = no_duplicate(message.mentions) - self.raw_members = no_duplicate(message.raw_mentions) + self.mention_users = "mention" in args or "mobile" in args - # transform the "me" arg - if "me" in args: - self.members += [message.author] - self.raw_members += [message.author.id] + if not await self.init(message, *args): + return - self.mention_users = "mention" in args or "mobile" in args - - if not await self.init(message, *args): - return - - # Start computing data - async with message.channel.typing(): - progress = await message.channel.send( - "```Starting analysis...```", - reference=message, - allowed_mentions=discord.AllowedMentions.none(), - ) - total_msg, total_chan = await logs.load( - progress, - self.channels, - self.start_date, - self.stop_date, - fast="fast" in args, - fresh="fresh" in args, - ) - if total_msg == CANCELLED: - await message.channel.send( - "Operation cancelled by user", + # Start computing data + async with message.channel.typing(): + progress = await message.channel.send( + "```Starting analysis...```", reference=message, + allowed_mentions=discord.AllowedMentions.none(), ) - elif total_msg == ALREADY_RUNNING: - await message.channel.send( - "An analysis is already running on this server, please be patient.", - reference=message, + total_msg, total_chan = await logs.load( + progress, + self.channels, + self.start_date, + self.stop_date, + fast="fast" in args, + fresh="fresh" in args, ) - elif total_msg == NO_FILE: - await message.channel.send(gdpr.TEXT) - else: - if self.start_date is not None and len(logs.channels) > 0: - self.start_date = max( - self.start_date, - min( - [ - logs.channels[channel.id].start_date - for channel in self.channels - if channel.id in logs.channels - and logs.channels[channel.id].start_date is not None - ] - ), - ) - if self.stop_date is None: - self.stop_date = datetime.utcnow() - - self.msg_count = 0 - self.total_msg = 0 - self.chan_count = 0 - t0 = datetime.now() - for channel in self.channels: - if channel.id in logs.channels: - channel_logs = logs.channels[channel.id] - count = sum( - [ - self.compute_message(channel_logs, message_log) - for message_log in channel_logs.messages - if ( - self.start_date is None - or message_log.created_at >= self.start_date - ) - and ( - self.stop_date is None - or message_log.created_at <= self.stop_date - ) - ] - ) - self.total_msg += len(channel_logs.messages) - self.msg_count += count - self.chan_count += 1 if count > 0 else 0 - logging.info(f"scan {guild.id} > scanned in {delta(t0):,}ms") - if self.msg_count == 0: + if total_msg == CANCELLED: await message.channel.send( - "There are no messages found matching the filters", + "Operation cancelled by user", reference=message, ) + elif total_msg == ALREADY_RUNNING: + await message.channel.send( + "An analysis is already running on this server, please be patient.", + reference=message, + ) + elif total_msg == NO_FILE: + await message.channel.send(gdpr.TEXT) else: - await progress.edit(content="```Computing results...```") - # Display results - t0 = datetime.now() - results = self.get_results( - get_intro( - self.intro_context, - self.full, - self.channels, - self.members, - self.msg_count, - self.chan_count, + if self.start_date is not None and len(logs.channels) > 0: + self.start_date = max( self.start_date, - self.stop_date, + min( + [ + logs.channels[channel.id].start_date + for channel in self.channels + if channel.id in logs.channels + and logs.channels[channel.id].start_date + is not None + ] + ), ) - ) - logging.info(f"scan {guild.id} > results in {delta(t0):,}ms") - response = "" - first = True - allowed_mentions = ( - discord.AllowedMentions.all() - if self.mention_users - else discord.AllowedMentions.none() - ) - for r in results: - if r: - if len(response + "\n" + r) > 2000: - await message.channel.send( - response, - reference=message if first else None, - allowed_mentions=allowed_mentions, - ) - first = False - response = "" - response += "\n" + r - if len(response) > 0: + if self.stop_date is None: + self.stop_date = datetime.utcnow() + + self.msg_count = 0 + self.total_msg = 0 + self.chan_count = 0 + t0 = datetime.now() + for channel in self.channels: + if channel.id in logs.channels: + channel_logs = logs.channels[channel.id] + count = sum( + [ + self.compute_message(channel_logs, message_log) + for message_log in channel_logs.messages + if ( + self.start_date is None + or message_log.created_at >= self.start_date + ) + and ( + self.stop_date is None + or message_log.created_at <= self.stop_date + ) + ] + ) + self.total_msg += len(channel_logs.messages) + self.msg_count += count + self.chan_count += 1 if count > 0 else 0 + logging.info(f"scan {guild.id} > scanned in {delta(t0):,}ms") + if self.msg_count == 0: await message.channel.send( - response, - reference=message if first else None, - allowed_mentions=allowed_mentions, + "There are no messages found matching the filters", + reference=message, ) - command_cache.cache(self, message, args) + else: + await progress.edit(content="```Computing results...```") + # Display results + t0 = datetime.now() + results = self.get_results( + get_intro( + self.intro_context, + self.full, + self.channels, + self.members, + self.msg_count, + self.chan_count, + self.start_date, + self.stop_date, + ) + ) + logging.info( + f"scan {guild.id} > results in {delta(t0):,}ms" + ) + response = "" + first = True + allowed_mentions = ( + discord.AllowedMentions.all() + if self.mention_users + else discord.AllowedMentions.none() + ) + for r in results: + if r: + if len(response + "\n" + r) > 2000: + await message.channel.send( + response, + reference=message if first else None, + allowed_mentions=allowed_mentions, + ) + first = False + response = "" + response += "\n" + r + if len(response) > 0: + await message.channel.send( + response, + reference=message if first else None, + allowed_mentions=allowed_mentions, + ) + command_cache.cache(self, message, args) # Delete custom progress message await progress.delete() + except Exception as error: + logging.exception(error) + await message.channel.send( + "An unexpected error happened while computing your command, we're sorry for the inconvenience.", + reference=message, + ) + if progress is not None: + await progress.delete() @abstractmethod async def init(self, message: discord.Message, *args: str) -> bool: From f8e294f64728572d9b1e95889fd266cd6bdaa6ce Mon Sep 17 00:00:00 2001 From: Klemek Date: Thu, 22 Apr 2021 13:09:07 +0200 Subject: [PATCH 05/12] emotes => emojis --- README.md | 6 +- src/data_types/__init__.py | 2 +- src/data_types/composition.py | 22 +++---- src/data_types/counter.py | 2 +- src/data_types/{emote.py => emoji.py} | 14 ++--- src/main.py | 6 +- src/scanners/__init__.py | 2 +- src/scanners/channels_scanner.py | 1 - src/scanners/composition_scanner.py | 22 +++---- .../{emotes_scanner.py => emojis_scanner.py} | 60 ++++++++++--------- src/scanners/mentioned_scanner.py | 2 - src/scanners/mentions_scanner.py | 2 - src/scanners/messages_scanner.py | 1 - src/scanners/reactions_scanner.py | 1 - src/scanners/words_scanner.py | 1 - 15 files changed, 69 insertions(+), 75 deletions(-) rename src/data_types/{emote.py => emoji.py} (92%) rename src/scanners/{emotes_scanner.py => emojis_scanner.py} (72%) diff --git a/README.md b/README.md index eac16f3..fae04f0 100644 --- a/README.md +++ b/README.md @@ -24,11 +24,11 @@ * %repeat - repeat last analysis (adding supplied arguments) * %mobile - fix @invalid-user for last command but mentions users * %gdpr - displays GDPR information -* %emojis - rank emotes by their usage +* %emojis - rank emojis by their usage * arguments: * - top emojis, default is 20 * all - list all common emojis in addition to this guild's - * members - show top member for each emote + * members - show top member for each emoji * sort:usage/reaction - other sorting methods * %mentions - rank mentions by their usage * arguments: @@ -150,7 +150,7 @@ python3 src/main.py * more scans: `%scan`, `%freq`, `%compo`, `%pres` * huge bug fix * **v1.5**: - * top emotes + * top emojis * bug fix * **v1.4**: * integrate miniscord diff --git a/src/data_types/__init__.py b/src/data_types/__init__.py index e4be59f..531bb61 100644 --- a/src/data_types/__init__.py +++ b/src/data_types/__init__.py @@ -1,4 +1,4 @@ -from .emote import Emote, get_emote_dict +from .emoji import Emoji, get_emoji_dict from .frequency import Frequency from .composition import Composition from .presence import Presence diff --git a/src/data_types/composition.py b/src/data_types/composition.py index 69364a1..b16797c 100644 --- a/src/data_types/composition.py +++ b/src/data_types/composition.py @@ -8,9 +8,9 @@ class Composition: def __init__(self): self.total_characters = 0 self.plain_text = 0 - self.emote_msg = 0 - self.emote_only = 0 - self.emotes = defaultdict(int) + self.emoji_msg = 0 + self.emoji_only = 0 + self.emojis = defaultdict(int) self.edited = 0 self.everyone = 0 self.answers = 0 @@ -23,8 +23,8 @@ class Composition: self.spoilers = 0 def to_string(self, msg_count: int) -> List[str]: - total_emotes = val_sum(self.emotes) - top_emote = top_key(self.emotes) + total_emojis = val_sum(self.emojis) + top_emoji = top_key(self.emojis) ret = [ f"- **avg. characters / message**: {self.total_characters/msg_count:.2f}", f"- **plain text messages**: {self.plain_text:,} ({percent(self.plain_text/msg_count)})" @@ -42,14 +42,14 @@ class Composition: f"- **answers**: {self.answers:,} ({percent(self.answers/msg_count)})" if self.answers > 0 else "", - f"- **emojis**: {total_emotes:,} (in {percent(self.emote_msg/msg_count)} of msg, avg. {precise(total_emotes/msg_count)}/msg)" - if total_emotes > 0 + f"- **emojis**: {total_emojis:,} (in {percent(self.emoji_msg/msg_count)} of msg, avg. {precise(total_emojis/msg_count)}/msg)" + if total_emojis > 0 else "", - f"- **most used emoji**: {top_emote} ({plural(self.emotes[top_emote], 'time')}, {percent(self.emotes[top_emote]/total_emotes)})" - if total_emotes > 0 + f"- **most used emoji**: {top_emoji} ({plural(self.emojis[top_emoji], 'time')}, {percent(self.emojis[top_emoji]/total_emojis)})" + if total_emojis > 0 else "", - f"- **emoji-only messages**: {self.emote_only:,} ({percent(self.emote_only/msg_count)})" - if self.emote_only > 0 + f"- **emoji-only messages**: {self.emoji_only:,} ({percent(self.emoji_only/msg_count)})" + if self.emoji_only > 0 else "", f"- **images**: {self.images:,} ({percent(self.images/msg_count)})" if self.images > 0 diff --git a/src/data_types/counter.py b/src/data_types/counter.py index c59ad43..6996808 100644 --- a/src/data_types/counter.py +++ b/src/data_types/counter.py @@ -19,7 +19,7 @@ class Counter: def score(self) -> float: # Score is compose of usages + reactions - # When 2 emotes have the same score, + # When 2 emojis have the same score, # the days since last use is stored in the digits # (more recent first) return self.all_usages() + 1 / ( diff --git a/src/data_types/emote.py b/src/data_types/emoji.py similarity index 92% rename from src/data_types/emote.py rename to src/data_types/emoji.py index 168263d..cb052e9 100644 --- a/src/data_types/emote.py +++ b/src/data_types/emoji.py @@ -8,9 +8,9 @@ import discord from utils import mention, plural, from_now, top_key, percent -class Emote: +class Emoji: """ - Custom class to store emotes data + Custom class to store emojis data """ def __init__(self, emoji: Optional[discord.Emoji] = None): @@ -34,7 +34,7 @@ class Emote: def score(self, *, usage_weight: int = 1, react_weight: int = 1) -> float: # Score is compose of usages + reactions - # When 2 emotes have the same score, + # When 2 emojis have the same score, # the days since last use is stored in the digits # (more recent first) return ( @@ -99,8 +99,8 @@ class Emote: return output -def get_emote_dict(guild: discord.Guild) -> Dict[str, Emote]: - emotes = defaultdict(Emote) +def get_emoji_dict(guild: discord.Guild) -> Dict[str, Emoji]: + emojis = defaultdict(Emoji) for emoji in guild.emojis: - emotes[str(emoji)] = Emote(emoji) - return emotes + emojis[str(emoji)] = Emoji(emoji) + return emojis diff --git a/src/main.py b/src/main.py index fb3c194..89b9809 100644 --- a/src/main.py +++ b/src/main.py @@ -8,7 +8,7 @@ if sys.version_info < (3, 7): from utils import emojis, gdpr, command_cache from scanners import ( - EmotesScanner, + EmojisScanner, FullScanner, FrequencyScanner, CompositionScanner, @@ -115,9 +115,9 @@ bot.register_command( ) bot.register_command( "(emojis?|emotes?)", - lambda *args: EmotesScanner().compute(*args), + lambda *args: EmojisScanner().compute(*args), "emojis: rank emojis by their usage", - EmotesScanner.help(), + EmojisScanner.help(), ) bot.register_command( "(react(ions?)?)", diff --git a/src/scanners/__init__.py b/src/scanners/__init__.py index 8ac7b97..eac71a5 100644 --- a/src/scanners/__init__.py +++ b/src/scanners/__init__.py @@ -1,5 +1,5 @@ from .scanner import Scanner -from .emotes_scanner import EmotesScanner +from .emojis_scanner import EmojisScanner from .frequency_scanner import FrequencyScanner from .composition_scanner import CompositionScanner from .presence_scanner import PresenceScanner diff --git a/src/scanners/channels_scanner.py b/src/scanners/channels_scanner.py index c766fb4..8359ef2 100644 --- a/src/scanners/channels_scanner.py +++ b/src/scanners/channels_scanner.py @@ -30,7 +30,6 @@ class ChannelsScanner(Scanner): ) async def init(self, message: discord.Message, *args: str) -> bool: - # get max emotes to view self.top = 10 for arg in args: if arg.isdigit(): diff --git a/src/scanners/composition_scanner.py b/src/scanners/composition_scanner.py index a2f3822..be64c22 100644 --- a/src/scanners/composition_scanner.py +++ b/src/scanners/composition_scanner.py @@ -57,19 +57,19 @@ class CompositionScanner(Scanner): impacted = True compo.total_characters += len(message.content) - emotes_found = emojis.regex.findall(message.content) - without_emote = message.content - for name in emotes_found: + emojis_found = emojis.regex.findall(message.content) + without_emoji = message.content + for name in emojis_found: if name in emojis.unicode_list or re.match( r"(|:[\w\\-\~]+:)", name ): - compo.emotes[name] += 1 - i = without_emote.index(name) - without_emote = without_emote[:i] + without_emote[i + len(name) :] - if len(message.content.strip()) > 0 and len(without_emote.strip()) == 0: - compo.emote_only += 1 - if len(emotes_found) > 0: - compo.emote_msg += 1 + compo.emojis[name] += 1 + i = without_emoji.index(name) + without_emoji = without_emoji[:i] + without_emoji[i + len(name) :] + if len(message.content.strip()) > 0 and len(without_emoji.strip()) == 0: + compo.emoji_only += 1 + if len(emojis_found) > 0: + compo.emoji_msg += 1 links_found = re.findall(r"https?:\/\/", message.content) compo.links += len(links_found) @@ -102,7 +102,7 @@ class CompositionScanner(Scanner): compo.tts += 1 if ( - len(emotes_found) == 0 + len(emojis_found) == 0 and message.reference is None and not message.image and len(message.mentions) == 0 diff --git a/src/scanners/emotes_scanner.py b/src/scanners/emojis_scanner.py similarity index 72% rename from src/scanners/emotes_scanner.py rename to src/scanners/emojis_scanner.py index b126812..c1c0de2 100644 --- a/src/scanners/emotes_scanner.py +++ b/src/scanners/emojis_scanner.py @@ -6,12 +6,12 @@ import discord # Custom libs from logs import ChannelLogs, MessageLog -from data_types import Emote, get_emote_dict +from data_types import Emoji, get_emoji_dict from .scanner import Scanner from utils import emojis, generate_help, plural, precise -class EmotesScanner(Scanner): +class EmojisScanner(Scanner): @staticmethod def help() -> str: return generate_help( @@ -31,13 +31,13 @@ class EmotesScanner(Scanner): super().__init__( has_digit_args=True, valid_args=["all", "members", "sort:usage", "sort:reaction", "everyone"], - help=EmotesScanner.help(), + help=EmojisScanner.help(), intro_context="Emoji usage", ) async def init(self, message: discord.Message, *args: str) -> bool: guild = message.channel.guild - # get max emotes to view + # get max emojis to view self.top = 20 for arg in args: if arg.isdigit(): @@ -47,8 +47,8 @@ class EmotesScanner(Scanner): self.show_members = "members" in args and ( len(self.members) == 0 or len(self.members) > 1 ) - # Create emotes dict from custom emojis of the guild - self.emotes = get_emote_dict(guild) + # Create emojis dict from custom emojis of the guild + self.emojis = get_emoji_dict(guild) self.sort = None if "sort:usage" in args: self.sort = "usage" @@ -58,36 +58,36 @@ class EmotesScanner(Scanner): return True def compute_message(self, channel: ChannelLogs, message: MessageLog): - return EmotesScanner.analyse_message( + return EmojisScanner.analyse_message( message, - self.emotes, + self.emojis, self.raw_members, all_emojis=self.all_emojis, all_messages=self.all_messages, ) def get_results(self, intro: str) -> List[str]: - names = [name for name in self.emotes] + names = [name for name in self.emojis] names.sort( - key=lambda name: self.emotes[name].score( + key=lambda name: self.emojis[name].score( usage_weight=(0 if self.sort == "reaction" else 1), react_weight=(0 if self.sort == "usage" else 1), ), reverse=True, ) names = names[: self.top] - # Get the total of all emotes used + # Get the total of all emojis used usage_count = 0 reaction_count = 0 - for name in self.emotes: - usage_count += self.emotes[name].usages - reaction_count += self.emotes[name].reactions + for name in self.emojis: + usage_count += self.emojis[name].usages + reaction_count += self.emojis[name].reactions res = [intro] allow_unused = self.full and len(self.members) == 0 if self.sort is not None: res += [f"(Sorted by {self.sort})"] res += [ - self.emotes[name].to_string( + self.emojis[name].to_string( names.index(name), name, total_usage=usage_count, @@ -96,7 +96,7 @@ class EmotesScanner(Scanner): show_members=self.show_members or len(self.raw_members) == 0, ) for name in names - if allow_unused or self.emotes[name].used() + if allow_unused or self.emojis[name].used() ] res += [ f"Total: {plural(usage_count,'time')} ({precise(usage_count/self.msg_count)}/msg)" @@ -108,7 +108,7 @@ class EmotesScanner(Scanner): @staticmethod def analyse_message( message: MessageLog, - emotes: Dict[str, Emote], + emojis_dict: Dict[str, Emoji], raw_members: List[int], *, all_emojis: bool, @@ -122,27 +122,29 @@ class EmotesScanner(Scanner): or message.author in raw_members ): impacted = True - # Find all emotes un the current message in the form "<:emoji:123456789>" - # Filter for known emotes + # Find all emojis un the current message in the form "<:emoji:123456789>" + # Filter for known emojis found = emojis.regex.findall(message.content) - # For each emote, update its usage + # For each emoji, update its usage for name in found: - if name not in emotes: + if name not in emojis_dict: if not all_emojis or name not in emojis.unicode_list: continue - emotes[name].usages += 1 - emotes[name].update_use(message.created_at, [message.author]) - # For each reaction of this message, test if known emote and update when it's the case + emojis_dict[name].usages += 1 + emojis_dict[name].update_use(message.created_at, [message.author]) + # For each reaction of this message, test if known emoji and update when it's the case for name in message.reactions: - if name not in emotes: + if name not in emojis_dict: if not all_emojis or name not in emojis.unicode_list: continue if len(raw_members) == 0: - emotes[name].reactions += len(message.reactions[name]) - emotes[name].update_use(message.created_at, message.reactions[name]) + emojis_dict[name].reactions += len(message.reactions[name]) + emojis_dict[name].update_use( + message.created_at, message.reactions[name] + ) else: for member in raw_members: if member in message.reactions[name]: - emotes[name].reactions += 1 - emotes[name].update_use(message.created_at, [member]) + emojis_dict[name].reactions += 1 + emojis_dict[name].update_use(message.created_at, [member]) return impacted diff --git a/src/scanners/mentioned_scanner.py b/src/scanners/mentioned_scanner.py index fa6c09e..cdb9b5d 100644 --- a/src/scanners/mentioned_scanner.py +++ b/src/scanners/mentioned_scanner.py @@ -31,7 +31,6 @@ class MentionedScanner(Scanner): ) async def init(self, message: discord.Message, *args: str) -> bool: - # get max emotes to view self.top = 10 for arg in args: if arg.isdigit(): @@ -55,7 +54,6 @@ class MentionedScanner(Scanner): names = [name for name in self.mentions] names.sort(key=lambda name: self.mentions[name].score(), reverse=True) names = names[: self.top] - # Get the total of all emotes used usage_count = Counter.total(self.mentions) res = [intro] res += [ diff --git a/src/scanners/mentions_scanner.py b/src/scanners/mentions_scanner.py index 50a0f5c..21e7246 100644 --- a/src/scanners/mentions_scanner.py +++ b/src/scanners/mentions_scanner.py @@ -42,7 +42,6 @@ class MentionsScanner(Scanner): ) async def init(self, message: discord.Message, *args: str) -> bool: - # get max emotes to view self.top = 10 for arg in args: if arg.isdigit(): @@ -67,7 +66,6 @@ class MentionsScanner(Scanner): names = [name for name in self.mentions] names.sort(key=lambda name: self.mentions[name].score(), reverse=True) names = names[: self.top] - # Get the total of all emotes used usage_count = Counter.total(self.mentions) res = [intro] res += [ diff --git a/src/scanners/messages_scanner.py b/src/scanners/messages_scanner.py index a79735e..4c775e6 100644 --- a/src/scanners/messages_scanner.py +++ b/src/scanners/messages_scanner.py @@ -30,7 +30,6 @@ class MessagesScanner(Scanner): ) async def init(self, message: discord.Message, *args: str) -> bool: - # get max emotes to view self.top = 10 for arg in args: if arg.isdigit(): diff --git a/src/scanners/reactions_scanner.py b/src/scanners/reactions_scanner.py index 3603a06..25a1a88 100644 --- a/src/scanners/reactions_scanner.py +++ b/src/scanners/reactions_scanner.py @@ -29,7 +29,6 @@ class ReactionsScanner(Scanner): ) async def init(self, message: discord.Message, *args: str) -> bool: - # get max emotes to view self.top = 10 for arg in args: if arg.isdigit(): diff --git a/src/scanners/words_scanner.py b/src/scanners/words_scanner.py index f7f6dd7..4460938 100644 --- a/src/scanners/words_scanner.py +++ b/src/scanners/words_scanner.py @@ -67,7 +67,6 @@ class WordsScanner(Scanner): words = [word for word in self.words] words.sort(key=lambda word: self.words[word].score(), reverse=True) words = words[: self.top] - # Get the total of all emotes used usage_count = Counter.total(self.words) print(len(self.words)) res = [intro.format(self.letters)] From 1871ff1d1355ffebc8d8c4c9f2c00f78542ff755 Mon Sep 17 00:00:00 2001 From: Klemek Date: Thu, 22 Apr 2021 13:25:07 +0200 Subject: [PATCH 06/12] fix relative time at start of day --- src/utils/utils.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/utils/utils.py b/src/utils/utils.py index 52d0bd1..a5d934e 100644 --- a/src/utils/utils.py +++ b/src/utils/utils.py @@ -185,11 +185,12 @@ RELATIVE_REGEX = r"(yesterday|today|\d*h(ours?)?|\d*d(ays?)?|\d*w(eeks?)?|\d*m(o def parse_relative_time(src: str) -> datetime: - timezone_delta = datetime.utcnow() - datetime.now() + today = datetime.utcnow().date() + today = datetime(today.year, today.month, today.day) if src == "today": - return datetime.today() + timezone_delta + return today elif src == "yesterday": - return datetime.today() - relativedelta(days=1) + timezone_delta + return today - relativedelta(days=1) else: m = re.match("(\d*)(\w+)", src) delta = None From 4ce3d6023ecece657590394984659a4741041cf7 Mon Sep 17 00:00:00 2001 From: Klemek Date: Thu, 22 Apr 2021 14:50:48 +0200 Subject: [PATCH 07/12] more info when available --- src/data_types/counter.py | 26 +++++++++++++++++--------- src/scanners/channels_scanner.py | 1 + src/scanners/mentioned_scanner.py | 4 +++- src/scanners/mentions_scanner.py | 16 +++++++++++----- src/scanners/messages_scanner.py | 1 + src/scanners/reactions_scanner.py | 1 + src/scanners/scanner.py | 21 ++++++++++++++------- src/scanners/words_scanner.py | 11 ++++------- 8 files changed, 52 insertions(+), 29 deletions(-) diff --git a/src/data_types/counter.py b/src/data_types/counter.py index 6996808..5f6d569 100644 --- a/src/data_types/counter.py +++ b/src/data_types/counter.py @@ -37,21 +37,29 @@ class Counter: total_usage: int, counted: str = "time", transform: Optional[Callable[[int], str]] = None, + ranking: bool = True, + top: bool = True, ) -> str: # place output = "" - if i == 0: - output += ":first_place:" - elif i == 1: - output += ":second_place:" - elif i == 2: - output += ":third_place:" + if ranking: + if i == 0: + output += ":first_place: " + elif i == 1: + output += ":second_place: " + elif i == 2: + output += ":third_place: " + else: + output += f"**#{i + 1}** " else: - output += f"**#{i + 1}**" + output += f"- " sum = val_sum(self.usages) - output += f" {name} - {plural(sum, counted)} ({percent(sum/total_usage)}, last {from_now(self.last_used)})" + if sum > 0: + output += f"{name} - {plural(sum, counted)} ({percent(sum/total_usage)}, last {from_now(self.last_used)})" + else: + output += f"{name} - unused" top_item = top_key(self.usages) - if top_item != 0 and transform is not None: + if top and top_item != 0 and transform is not None: if self.usages[top_item] == sum: output += f" (all{transform(top_item)})" else: diff --git a/src/scanners/channels_scanner.py b/src/scanners/channels_scanner.py index 8359ef2..d4dcc7f 100644 --- a/src/scanners/channels_scanner.py +++ b/src/scanners/channels_scanner.py @@ -61,6 +61,7 @@ class ChannelsScanner(Scanner): total_usage=usage_count, counted="message", transform=lambda id: f" by {mention(id)}", + top=len(self.members) != 1, ) for name in names ] diff --git a/src/scanners/mentioned_scanner.py b/src/scanners/mentioned_scanner.py index cdb9b5d..b2f3fe4 100644 --- a/src/scanners/mentioned_scanner.py +++ b/src/scanners/mentioned_scanner.py @@ -61,6 +61,8 @@ class MentionedScanner(Scanner): names.index(name), name, total_usage=usage_count, + transform=lambda id: f" for {mention(id)}", + top=len(self.members) != 1, ) for name in names ] @@ -85,6 +87,6 @@ class MentionedScanner(Scanner): mention(member_id) ) + message.content.count(alt_mention(member_id)) mentions[mention(message.author)].update_use( - count, message.created_at + count, message.created_at, member_id ) return impacted diff --git a/src/scanners/mentions_scanner.py b/src/scanners/mentions_scanner.py index 21e7246..10cecf2 100644 --- a/src/scanners/mentions_scanner.py +++ b/src/scanners/mentions_scanner.py @@ -73,6 +73,8 @@ class MentionsScanner(Scanner): names.index(name), name, total_usage=usage_count, + transform=lambda id: f" by {mention(id)}", + top=len(self.members) != 1, ) for name in names ] @@ -103,24 +105,28 @@ class MentionsScanner(Scanner): count = message.content.count(name) + message.content.count( alt_mention(member_id) ) - mentions[name].update_use(count, message.created_at) + mentions[name].update_use(count, message.created_at, message.author) if all_mentions: for role_id in message.role_mentions: name = role_mention(role_id) mentions[name].update_use( - message.content.count(name), message.created_at + message.content.count(name), message.created_at, message.author ) for channel_id in message.channel_mentions: name = channel_mention(channel_id) mentions[name].update_use( - message.content.count(name), message.created_at + message.content.count(name), message.created_at, message.author ) if "@everyone" in message.content: mentions["@\u200beveryone"].update_use( - message.content.count("@everyone"), message.created_at + message.content.count("@everyone"), + message.created_at, + message.author, ) if "@here" in message.content: mentions["@\u200bhere"].update_use( - message.content.count("@here"), message.created_at + message.content.count("@here"), + message.created_at, + message.author, ) return impacted diff --git a/src/scanners/messages_scanner.py b/src/scanners/messages_scanner.py index 4c775e6..086a123 100644 --- a/src/scanners/messages_scanner.py +++ b/src/scanners/messages_scanner.py @@ -61,6 +61,7 @@ class MessagesScanner(Scanner): total_usage=usage_count, counted="message", transform=lambda id: f" in {channel_mention(id)}", + top=self.channels != 1, ) for name in names ] diff --git a/src/scanners/reactions_scanner.py b/src/scanners/reactions_scanner.py index 25a1a88..8b68c36 100644 --- a/src/scanners/reactions_scanner.py +++ b/src/scanners/reactions_scanner.py @@ -58,6 +58,7 @@ class ReactionsScanner(Scanner): total_usage=usage_count, counted="reaction", transform=lambda id: f" in {channel_mention(id)}", + top=self.channels != 1, ) for name in names ] diff --git a/src/scanners/scanner.py b/src/scanners/scanner.py index 7ea4cda..b642594 100644 --- a/src/scanners/scanner.py +++ b/src/scanners/scanner.py @@ -27,6 +27,8 @@ from logs import ( class Scanner(ABC): + VALID_ARGS = ["me", "here", "fast", "fresh", "mobile", "mention"] + def __init__( self, *, @@ -34,12 +36,16 @@ class Scanner(ABC): valid_args: List[str] = [], help: str, intro_context: str, + all_args: bool = False, ): self.has_digit_args = has_digit_args self.valid_args = valid_args + self.all_args = all_args self.help = help self.intro_context = intro_context + self.other_args = [] + self.members = [] self.raw_members = [] self.full = False @@ -86,19 +92,20 @@ class Scanner(ABC): ) return if ( - arg - not in self.valid_args - + ["me", "here", "fast", "fresh", "mobile", "mention"] + arg not in self.valid_args + Scanner.VALID_ARGS and (not arg.isdigit() or not self.has_digit_args) and arg not in str_channel_mentions and arg not in str_mentions and arg not in other_mentions and not skip_check ): - await message.channel.send( - f"Unrecognized argument: `{arg}`", reference=message - ) - return + if self.all_args: + self.other_args += [arg] + else: + await message.channel.send( + f"Unrecognized argument: `{arg}`", reference=message + ) + return self.start_date = None if len(dates) < 1 else min(dates) self.stop_date = None if len(dates) < 2 else max(dates) diff --git a/src/scanners/words_scanner.py b/src/scanners/words_scanner.py index 4460938..f5c2528 100644 --- a/src/scanners/words_scanner.py +++ b/src/scanners/words_scanner.py @@ -8,11 +8,7 @@ import re from logs import ChannelLogs, MessageLog from .scanner import Scanner from data_types import Counter -from utils import ( - generate_help, - plural, - precise, -) +from utils import generate_help, plural, precise, mention class WordsScanner(Scanner): @@ -68,13 +64,14 @@ class WordsScanner(Scanner): words.sort(key=lambda word: self.words[word].score(), reverse=True) words = words[: self.top] usage_count = Counter.total(self.words) - print(len(self.words)) res = [intro.format(self.letters)] res += [ self.words[word].to_string( words.index(word), f"`{word}`", total_usage=usage_count, + transform=lambda id: f" by {mention(id)}", + top=len(self.members) != 1, ) for word in words ] @@ -121,5 +118,5 @@ class WordsScanner(Scanner): words[word] = words[word + case] del words[word + case] break - words[word].update_use(1, message.created_at) + words[word].update_use(1, message.created_at, message.author) return impacted From 3721f1aef2d7639dee634a03299ffe4302b1371a Mon Sep 17 00:00:00 2001 From: Klemek Date: Thu, 22 Apr 2021 14:58:08 +0200 Subject: [PATCH 08/12] imports refactor --- src/data_types/__init__.py | 4 +- src/main.py | 73 ++++++++++++++---------------------- src/scanners/__init__.py | 19 +++++----- src/scanners/full_scanner.py | 4 +- 4 files changed, 44 insertions(+), 56 deletions(-) diff --git a/src/data_types/__init__.py b/src/data_types/__init__.py index 531bb61..2650009 100644 --- a/src/data_types/__init__.py +++ b/src/data_types/__init__.py @@ -1,6 +1,6 @@ from .emoji import Emoji, get_emoji_dict -from .frequency import Frequency from .composition import Composition -from .presence import Presence from .counter import Counter +from .frequency import Frequency from .history import History +from .presence import Presence diff --git a/src/main.py b/src/main.py index 89b9809..1cee0f6 100644 --- a/src/main.py +++ b/src/main.py @@ -7,22 +7,7 @@ if sys.version_info < (3, 7): sys.exit(1) from utils import emojis, gdpr, command_cache -from scanners import ( - EmojisScanner, - FullScanner, - FrequencyScanner, - CompositionScanner, - PresenceScanner, - MentionsScanner, - MentionedScanner, - MessagesScanner, - ChannelsScanner, - ReactionsScanner, - FirstScanner, - RandomScanner, - LastScanner, - WordsScanner, -) +import scanners from logs import GuildLogs logging.basicConfig( @@ -67,9 +52,9 @@ bot.register_command( ) bot.register_command( "words", - lambda *args: WordsScanner().compute(*args), + lambda *args: scanners.WordsScanner().compute(*args), "words: (BETA) rank words by their usage", - WordsScanner.help(), + scanners.WordsScanner.help(), ) bot.register_command( "repeat", @@ -85,81 +70,81 @@ bot.register_command( ) bot.register_command( "last", - lambda *args: LastScanner().compute(*args), + lambda *args: scanners.LastScanner().compute(*args), "last: read last message", - LastScanner.help(), + scanners.LastScanner.help(), ) bot.register_command( "rand(om)?", - lambda *args: RandomScanner().compute(*args), + lambda *args: scanners.RandomScanner().compute(*args), "rand: read a random message", - RandomScanner.help(), + scanners.RandomScanner.help(), ) bot.register_command( "first", - lambda *args: FirstScanner().compute(*args), + lambda *args: scanners.FirstScanner().compute(*args), "first: read first message", - FirstScanner.help(), + scanners.FirstScanner.help(), ) bot.register_command( "mentioned", - lambda *args: MentionedScanner().compute(*args), + lambda *args: scanners.MentionedScanner().compute(*args), "mentioned: rank specific user mentions by their usage", - MentionedScanner.help(), + scanners.MentionedScanner.help(), ) bot.register_command( "(mentions?)", - lambda *args: MentionsScanner().compute(*args), + lambda *args: scanners.MentionsScanner().compute(*args), "mentions: rank mentions by their usage", - MentionsScanner.help(), + scanners.MentionsScanner.help(), ) bot.register_command( "(emojis?|emotes?)", - lambda *args: EmojisScanner().compute(*args), + lambda *args: scanners.EmojisScanner().compute(*args), "emojis: rank emojis by their usage", - EmojisScanner.help(), + scanners.EmojisScanner.help(), ) bot.register_command( "(react(ions?)?)", - lambda *args: ReactionsScanner().compute(*args), + lambda *args: scanners.ReactionsScanner().compute(*args), "react: rank users by their reactions", - ReactionsScanner.help(), + scanners.ReactionsScanner.help(), ) bot.register_command( "(channels?|chan)", - lambda *args: ChannelsScanner().compute(*args), + lambda *args: scanners.ChannelsScanner().compute(*args), "chan: rank channels by their messages", - ChannelsScanner.help(), + scanners.ChannelsScanner.help(), ) bot.register_command( "(messages?|msg)", - lambda *args: MessagesScanner().compute(*args), + lambda *args: scanners.MessagesScanner().compute(*args), "msg: rank users by their messages", - MessagesScanner.help(), + scanners.MessagesScanner.help(), ) bot.register_command( "pres(ence)?", - lambda *args: PresenceScanner().compute(*args), + lambda *args: scanners.PresenceScanner().compute(*args), "pres: presence analysis", - PresenceScanner.help(), + scanners.PresenceScanner.help(), ) bot.register_command( "compo(sition)?", - lambda *args: CompositionScanner().compute(*args), + lambda *args: scanners.CompositionScanner().compute(*args), "compo: composition analysis", - CompositionScanner.help(), + scanners.CompositionScanner.help(), ) bot.register_command( "freq(ency)?", - lambda *args: FrequencyScanner().compute(*args), + lambda *args: scanners.FrequencyScanner().compute(*args), "freq: frequency analysis", - FrequencyScanner.help(), + scanners.FrequencyScanner.help(), ) bot.register_command( "(full|scan)", - lambda *args: FullScanner().compute(*args), + lambda *args: scanners.FullScanner().compute(*args), "scan: full analysis", - FullScanner.help(), + scanners.FullScanner.help(), ) bot.start() diff --git a/src/scanners/__init__.py b/src/scanners/__init__.py index eac71a5..aca349b 100644 --- a/src/scanners/__init__.py +++ b/src/scanners/__init__.py @@ -1,15 +1,16 @@ from .scanner import Scanner -from .emojis_scanner import EmojisScanner -from .frequency_scanner import FrequencyScanner -from .composition_scanner import CompositionScanner -from .presence_scanner import PresenceScanner -from .full_scanner import FullScanner -from .mentions_scanner import MentionsScanner -from .mentioned_scanner import MentionedScanner -from .messages_scanner import MessagesScanner + from .channels_scanner import ChannelsScanner -from .reactions_scanner import ReactionsScanner +from .composition_scanner import CompositionScanner +from .emojis_scanner import EmojisScanner from .first_scanner import FirstScanner +from .frequency_scanner import FrequencyScanner +from .full_scanner import FullScanner from .last_scanner import LastScanner +from .mentioned_scanner import MentionedScanner +from .mentions_scanner import MentionsScanner +from .messages_scanner import MessagesScanner +from .presence_scanner import PresenceScanner from .random_scanner import RandomScanner +from .reactions_scanner import ReactionsScanner from .words_scanner import WordsScanner diff --git a/src/scanners/full_scanner.py b/src/scanners/full_scanner.py index ac5cb0e..e07cf9d 100644 --- a/src/scanners/full_scanner.py +++ b/src/scanners/full_scanner.py @@ -5,7 +5,9 @@ import discord # Custom libs from .scanner import Scanner -from . import FrequencyScanner, CompositionScanner, PresenceScanner +from .composition_scanner import CompositionScanner +from .frequency_scanner import FrequencyScanner +from .presence_scanner import PresenceScanner from data_types import Frequency, Composition, Presence from logs import ChannelLogs, MessageLog from utils import generate_help From fc5d9b82c1d78b50342f3669f6925a9e6e42fc91 Mon Sep 17 00:00:00 2001 From: Klemek Date: Thu, 22 Apr 2021 15:13:47 +0200 Subject: [PATCH 09/12] fix relative date regex --- src/utils/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/utils/utils.py b/src/utils/utils.py index a5d934e..7e5e9b6 100644 --- a/src/utils/utils.py +++ b/src/utils/utils.py @@ -181,7 +181,7 @@ def parse_iso_datetime(str_date: str) -> datetime: return dateutil.parser.parse(str_date) -RELATIVE_REGEX = r"(yesterday|today|\d*h(ours?)?|\d*d(ays?)?|\d*w(eeks?)?|\d*m(onths?)?|\d*y(ears?)?)" +RELATIVE_REGEX = r"(yesterday|today|\d*hours?|\d+h(ours?)?|\d*days?|\d+d(ays?)?|\d*weeks?|\d+w(eeks?)?|\d*months?|\d+m(onths?)?|\d*years?|\d+y(ears?)?)" def parse_relative_time(src: str) -> datetime: From 5f8dfce640eaa3dc592e14775c032ea8478a1be3 Mon Sep 17 00:00:00 2001 From: Klemek Date: Thu, 22 Apr 2021 15:15:32 +0200 Subject: [PATCH 10/12] %find command --- README.md | 2 + src/data_types/counter.py | 6 ++- src/main.py | 6 +++ src/scanners/__init__.py | 1 + src/scanners/find_scanner.py | 99 ++++++++++++++++++++++++++++++++++++ 5 files changed, 112 insertions(+), 2 deletions(-) create mode 100644 src/scanners/find_scanner.py diff --git a/README.md b/README.md index fae04f0..32b94a1 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ * %first - read first message * %rand - read a random message * %last - read last message +* %find - find specific words or phrases * %repeat - repeat last analysis (adding supplied arguments) * %mobile - fix @invalid-user for last command but mentions users * %gdpr - displays GDPR information @@ -115,6 +116,7 @@ python3 src/main.py * **v1.14** * `mobile/mention` arg to fix mobile bug * `%repeat`, `%mobile` to repeat commands + * more scan: `%find` * bug fix * **v1.13** * improved scan `%words` diff --git a/src/data_types/counter.py b/src/data_types/counter.py index 5f6d569..212cf89 100644 --- a/src/data_types/counter.py +++ b/src/data_types/counter.py @@ -14,7 +14,7 @@ class Counter: def update_use(self, count: int, date: datetime, item: int = 0): self.usages[item] += count - if self.last_used is None or date > self.last_used: + if count > 0 and (self.last_used is None or date > self.last_used): self.last_used = date def score(self) -> float: @@ -22,6 +22,8 @@ class Counter: # When 2 emojis have the same score, # the days since last use is stored in the digits # (more recent first) + if self.last_used is None: + return 0 return self.all_usages() + 1 / ( 100000 * ((datetime.today() - self.last_used).days + 1) ) @@ -59,7 +61,7 @@ class Counter: else: output += f"{name} - unused" top_item = top_key(self.usages) - if top and top_item != 0 and transform is not None: + if sum > 0 and top and top_item != 0 and transform is not None: if self.usages[top_item] == sum: output += f" (all{transform(top_item)})" else: diff --git a/src/main.py b/src/main.py index 1cee0f6..97dda5b 100644 --- a/src/main.py +++ b/src/main.py @@ -68,6 +68,12 @@ bot.register_command( "mobile: fix @invalid-user for last command but mentions users", "```\n%mobile: fix @invalid-user for last command but mentions users\n```", ) +bot.register_command( + "find", + lambda *args: scanners.FindScanner().compute(*args), + "find: find specific words or phrases", + scanners.FindScanner.help(), +) bot.register_command( "last", lambda *args: scanners.LastScanner().compute(*args), diff --git a/src/scanners/__init__.py b/src/scanners/__init__.py index aca349b..c174ed7 100644 --- a/src/scanners/__init__.py +++ b/src/scanners/__init__.py @@ -3,6 +3,7 @@ from .scanner import Scanner from .channels_scanner import ChannelsScanner from .composition_scanner import CompositionScanner from .emojis_scanner import EmojisScanner +from .find_scanner import FindScanner from .first_scanner import FirstScanner from .frequency_scanner import FrequencyScanner from .full_scanner import FullScanner diff --git a/src/scanners/find_scanner.py b/src/scanners/find_scanner.py new file mode 100644 index 0000000..334beb0 --- /dev/null +++ b/src/scanners/find_scanner.py @@ -0,0 +1,99 @@ +from typing import Dict, List +from collections import defaultdict +import discord +import re + +# Custom libs + +from logs import ChannelLogs, MessageLog +from .scanner import Scanner +from data_types import Counter +from utils import ( + generate_help, + plural, + precise, + mention, +) + + +class FindScanner(Scanner): + @staticmethod + def help() -> str: + return generate_help( + "find", + "Find specific words or phrases", + args=[ + "all/everyone - include bots", + ], + example='#mychannel1 #mychannel2 @user "I love you" "you too"', + ) + + def __init__(self): + super().__init__( + all_args=True, + valid_args=["all", "everyone"], + help=FindScanner.help(), + intro_context="Matches", + ) + + async def init(self, message: discord.Message, *args: str) -> bool: + self.matches = defaultdict(Counter) + self.all_messages = "all" in args or "everyone" in args + return True + + def compute_message(self, channel: ChannelLogs, message: MessageLog): + return FindScanner.analyse_message( + message, + self.matches, + self.other_args, + self.raw_members, + all_messages=self.all_messages, + ) + + def get_results(self, intro: str) -> List[str]: + matches = [match for match in self.matches] + matches.sort(key=lambda match: self.matches[match].score(), reverse=True) + usage_count = Counter.total(self.matches) + res = [intro] + res += [ + self.matches[match].to_string( + matches.index(match), + f"`{match}`", + total_usage=self.msg_count, + ranking=False, + transform=lambda id: f" by {mention(id)}", + top=len(self.members) != 1, + ) + for match in matches + ] + if len(matches) > 1: + res += [ + f"Total: {plural(usage_count,'time')} ({precise(usage_count/self.msg_count)}/msg)" + ] + return res + + special_cases = ["'s", "s"] + + @staticmethod + def analyse_message( + message: MessageLog, + matches: Dict[str, Counter], + queries: List[str], + raw_members: List[int], + *, + all_messages: bool, + ) -> bool: + impacted = False + # If author is included in the selection (empty list is all) + if ( + (not message.bot or all_messages) + and len(raw_members) == 0 + or message.author in raw_members + ): + impacted = True + content = message.content.lower() + for query in queries: + matches[query].update_use( + content.count(query.lower()), message.created_at, message.author + ) + return impacted From e77e46b361c4b86d97fc9c23be0a9a43fafb8b4c Mon Sep 17 00:00:00 2001 From: Klemek Date: Thu, 22 Apr 2021 15:20:49 +0200 Subject: [PATCH 11/12] fix help position in arguments --- src/scanners/scanner.py | 3 ++- src/utils/command_cache.py | 2 +- src/utils/gdpr.py | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/scanners/scanner.py b/src/scanners/scanner.py index b642594..e5b41b4 100644 --- a/src/scanners/scanner.py +++ b/src/scanners/scanner.py @@ -67,7 +67,7 @@ class Scanner(ABC): try: with GuildLogs(guild) as logs: # If "%cmd help" redirect to "%help cmd" - if "help" in args: + if len(args) > 1 and args[1] == "help": await client.bot.help(client, message, "help", args[0]) return @@ -98,6 +98,7 @@ class Scanner(ABC): and arg not in str_mentions and arg not in other_mentions and not skip_check + and len(arg) > 0 ): if self.all_args: self.other_args += [arg] diff --git a/src/utils/command_cache.py b/src/utils/command_cache.py index 4cdbf7b..252313e 100644 --- a/src/utils/command_cache.py +++ b/src/utils/command_cache.py @@ -23,7 +23,7 @@ async def repeat( *args: str, add_args: List[str] = [], ): - if "help" in args: + if len(args) > 1 and args[1] == "help": await client.bot.help(client, message, "help", args[0]) return id = message.channel.id diff --git a/src/utils/gdpr.py b/src/utils/gdpr.py index d913f5c..e14fdb4 100644 --- a/src/utils/gdpr.py +++ b/src/utils/gdpr.py @@ -49,10 +49,10 @@ async def process(client: discord.client, message: discord.Message, *args: str): args = list(args) if len(args) == 1: await message.channel.send(TEXT) - elif len(args) > 2: - await message.channel.send(f"Too many arguments", reference=message) elif args[1] == "help": await client.bot.help(client, message, "help", args[0]) + elif len(args) > 2: + await message.channel.send(f"Too many arguments", reference=message) elif args[1] in ["agree", "accept"]: GuildLogs.init_log(message.channel.guild) await message.channel.send(AGREE_TEXT, reference=message) From 3f7abd9a15dd4e76b6a0ef1cc64f6c913a618b51 Mon Sep 17 00:00:00 2001 From: Klemek Date: Thu, 22 Apr 2021 15:21:01 +0200 Subject: [PATCH 12/12] check for argument in %find --- src/scanners/find_scanner.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/scanners/find_scanner.py b/src/scanners/find_scanner.py index 334beb0..34a5397 100644 --- a/src/scanners/find_scanner.py +++ b/src/scanners/find_scanner.py @@ -21,7 +21,7 @@ class FindScanner(Scanner): def help() -> str: return generate_help( "find", - "Find specific words or phrases", + "Find specific words or phrases (you can use quotes to add spaces in queries)", args=[ "all/everyone - include bots", ], @@ -39,6 +39,12 @@ class FindScanner(Scanner): async def init(self, message: discord.Message, *args: str) -> bool: self.matches = defaultdict(Counter) self.all_messages = "all" in args or "everyone" in args + if len(self.other_args) == 0: + await message.channel.send( + "You need to add a query to find (you can use quotes to add spaces in queries)", + reference=message, + ) + return False return True def compute_message(self, channel: ChannelLogs, message: MessageLog):