diff --git a/README.md b/README.md index ad239ac..32b94a1 100644 --- a/README.md +++ b/README.md @@ -21,12 +21,15 @@ * %first - read first message * %rand - read a random message * %last - read last message +* %find - find specific words or phrases +* %repeat - repeat last analysis (adding supplied arguments) +* %mobile - fix @invalid-user for last command but mentions users * %gdpr - displays GDPR information -* %emojis - rank emotes by their usage +* %emojis - rank emojis by their usage * arguments: * - top emojis, default is 20 * all - list all common emojis in addition to this guild's - * members - show top member for each emote + * members - show top member for each emoji * sort:usage/reaction - other sorting methods * %mentions - rank mentions by their usage * arguments: @@ -58,6 +61,7 @@ * all/everyone - include bots messages * fast: only read cache * fresh: does not read cache + * mobile/mention: mentions users (fix @invalid-user bug) (Sample dates: 2020 / 2021-11 / 2021-06-28 / 2020-06-28T23:00 / today / week / 8days / 1y) ``` @@ -109,6 +113,11 @@ python3 src/main.py ## Changelog +* **v1.14** + * `mobile/mention` arg to fix mobile bug + * `%repeat`, `%mobile` to repeat commands + * more scan: `%find` + * bug fix * **v1.13** * improved scan `%words` * remove old and unused logs at start and guild leaving @@ -143,7 +152,7 @@ python3 src/main.py * more scans: `%scan`, `%freq`, `%compo`, `%pres` * huge bug fix * **v1.5**: - * top emotes + * top emojis * bug fix * **v1.4**: * integrate miniscord diff --git a/src/data_types/__init__.py b/src/data_types/__init__.py index e4be59f..2650009 100644 --- a/src/data_types/__init__.py +++ b/src/data_types/__init__.py @@ -1,6 +1,6 @@ -from .emote import Emote, get_emote_dict -from .frequency import Frequency +from .emoji import Emoji, get_emoji_dict from .composition import Composition -from .presence import Presence from .counter import Counter +from .frequency import Frequency from .history import History +from .presence import Presence diff --git a/src/data_types/composition.py b/src/data_types/composition.py index 69364a1..b16797c 100644 --- a/src/data_types/composition.py +++ b/src/data_types/composition.py @@ -8,9 +8,9 @@ class Composition: def __init__(self): self.total_characters = 0 self.plain_text = 0 - self.emote_msg = 0 - self.emote_only = 0 - self.emotes = defaultdict(int) + self.emoji_msg = 0 + self.emoji_only = 0 + self.emojis = defaultdict(int) self.edited = 0 self.everyone = 0 self.answers = 0 @@ -23,8 +23,8 @@ class Composition: self.spoilers = 0 def to_string(self, msg_count: int) -> List[str]: - total_emotes = val_sum(self.emotes) - top_emote = top_key(self.emotes) + total_emojis = val_sum(self.emojis) + top_emoji = top_key(self.emojis) ret = [ f"- **avg. characters / message**: {self.total_characters/msg_count:.2f}", f"- **plain text messages**: {self.plain_text:,} ({percent(self.plain_text/msg_count)})" @@ -42,14 +42,14 @@ class Composition: f"- **answers**: {self.answers:,} ({percent(self.answers/msg_count)})" if self.answers > 0 else "", - f"- **emojis**: {total_emotes:,} (in {percent(self.emote_msg/msg_count)} of msg, avg. {precise(total_emotes/msg_count)}/msg)" - if total_emotes > 0 + f"- **emojis**: {total_emojis:,} (in {percent(self.emoji_msg/msg_count)} of msg, avg. {precise(total_emojis/msg_count)}/msg)" + if total_emojis > 0 else "", - f"- **most used emoji**: {top_emote} ({plural(self.emotes[top_emote], 'time')}, {percent(self.emotes[top_emote]/total_emotes)})" - if total_emotes > 0 + f"- **most used emoji**: {top_emoji} ({plural(self.emojis[top_emoji], 'time')}, {percent(self.emojis[top_emoji]/total_emojis)})" + if total_emojis > 0 else "", - f"- **emoji-only messages**: {self.emote_only:,} ({percent(self.emote_only/msg_count)})" - if self.emote_only > 0 + f"- **emoji-only messages**: {self.emoji_only:,} ({percent(self.emoji_only/msg_count)})" + if self.emoji_only > 0 else "", f"- **images**: {self.images:,} ({percent(self.images/msg_count)})" if self.images > 0 diff --git a/src/data_types/counter.py b/src/data_types/counter.py index c59ad43..212cf89 100644 --- a/src/data_types/counter.py +++ b/src/data_types/counter.py @@ -14,14 +14,16 @@ class Counter: def update_use(self, count: int, date: datetime, item: int = 0): self.usages[item] += count - if self.last_used is None or date > self.last_used: + if count > 0 and (self.last_used is None or date > self.last_used): self.last_used = date def score(self) -> float: # Score is compose of usages + reactions - # When 2 emotes have the same score, + # When 2 emojis have the same score, # the days since last use is stored in the digits # (more recent first) + if self.last_used is None: + return 0 return self.all_usages() + 1 / ( 100000 * ((datetime.today() - self.last_used).days + 1) ) @@ -37,21 +39,29 @@ class Counter: total_usage: int, counted: str = "time", transform: Optional[Callable[[int], str]] = None, + ranking: bool = True, + top: bool = True, ) -> str: # place output = "" - if i == 0: - output += ":first_place:" - elif i == 1: - output += ":second_place:" - elif i == 2: - output += ":third_place:" + if ranking: + if i == 0: + output += ":first_place: " + elif i == 1: + output += ":second_place: " + elif i == 2: + output += ":third_place: " + else: + output += f"**#{i + 1}** " else: - output += f"**#{i + 1}**" + output += f"- " sum = val_sum(self.usages) - output += f" {name} - {plural(sum, counted)} ({percent(sum/total_usage)}, last {from_now(self.last_used)})" + if sum > 0: + output += f"{name} - {plural(sum, counted)} ({percent(sum/total_usage)}, last {from_now(self.last_used)})" + else: + output += f"{name} - unused" top_item = top_key(self.usages) - if top_item != 0 and transform is not None: + if sum > 0 and top and top_item != 0 and transform is not None: if self.usages[top_item] == sum: output += f" (all{transform(top_item)})" else: diff --git a/src/data_types/emote.py b/src/data_types/emoji.py similarity index 92% rename from src/data_types/emote.py rename to src/data_types/emoji.py index 168263d..cb052e9 100644 --- a/src/data_types/emote.py +++ b/src/data_types/emoji.py @@ -8,9 +8,9 @@ import discord from utils import mention, plural, from_now, top_key, percent -class Emote: +class Emoji: """ - Custom class to store emotes data + Custom class to store emojis data """ def __init__(self, emoji: Optional[discord.Emoji] = None): @@ -34,7 +34,7 @@ class Emote: def score(self, *, usage_weight: int = 1, react_weight: int = 1) -> float: # Score is compose of usages + reactions - # When 2 emotes have the same score, + # When 2 emojis have the same score, # the days since last use is stored in the digits # (more recent first) return ( @@ -99,8 +99,8 @@ class Emote: return output -def get_emote_dict(guild: discord.Guild) -> Dict[str, Emote]: - emotes = defaultdict(Emote) +def get_emoji_dict(guild: discord.Guild) -> Dict[str, Emoji]: + emojis = defaultdict(Emoji) for emoji in guild.emojis: - emotes[str(emoji)] = Emote(emoji) - return emotes + emojis[str(emoji)] = Emoji(emoji) + return emojis diff --git a/src/main.py b/src/main.py index d2278e5..97dda5b 100644 --- a/src/main.py +++ b/src/main.py @@ -6,23 +6,8 @@ if sys.version_info < (3, 7): print("Please upgrade your Python version to 3.7.0 or higher") sys.exit(1) -from utils import emojis, gdpr -from scanners import ( - EmotesScanner, - FullScanner, - FrequencyScanner, - CompositionScanner, - PresenceScanner, - MentionsScanner, - MentionedScanner, - MessagesScanner, - ChannelsScanner, - ReactionsScanner, - FirstScanner, - RandomScanner, - LastScanner, - WordsScanner, -) +from utils import emojis, gdpr, command_cache +import scanners from logs import GuildLogs logging.basicConfig( @@ -33,7 +18,7 @@ emojis.load_emojis() bot = Bot( "Discord Analyst", - "1.13", + "1.14", alias="%", ) @@ -67,87 +52,105 @@ bot.register_command( ) bot.register_command( "words", - lambda *args: WordsScanner().compute(*args), + lambda *args: scanners.WordsScanner().compute(*args), "words: (BETA) rank words by their usage", - WordsScanner.help(), + scanners.WordsScanner.help(), +) +bot.register_command( + "repeat", + command_cache.repeat, + "repeat: repeat last analysis (adding supplied arguments)", + "```\n%repeat: repeat last analysis (adding supplied arguments)\n```", +) +bot.register_command( + "mobile", + lambda *args: command_cache.repeat(*args, add_args=["mobile"]), + "mobile: fix @invalid-user for last command but mentions users", + "```\n%mobile: fix @invalid-user for last command but mentions users\n```", +) +bot.register_command( + "find", + lambda *args: scanners.FindScanner().compute(*args), + "find: find specific words or phrases", + scanners.FindScanner.help(), ) bot.register_command( "last", - lambda *args: LastScanner().compute(*args), + lambda *args: scanners.LastScanner().compute(*args), "last: read last message", - LastScanner.help(), + scanners.LastScanner.help(), ) bot.register_command( "rand(om)?", - lambda *args: RandomScanner().compute(*args), + lambda *args: scanners.RandomScanner().compute(*args), "rand: read a random message", - RandomScanner.help(), + scanners.RandomScanner.help(), ) bot.register_command( "first", - lambda *args: FirstScanner().compute(*args), + lambda *args: scanners.FirstScanner().compute(*args), "first: read first message", - FirstScanner.help(), + scanners.FirstScanner.help(), ) bot.register_command( "mentioned", - lambda *args: MentionedScanner().compute(*args), + lambda *args: scanners.MentionedScanner().compute(*args), "mentioned: rank specific user mentions by their usage", - MentionedScanner.help(), + scanners.MentionedScanner.help(), ) bot.register_command( "(mentions?)", - lambda *args: MentionsScanner().compute(*args), + lambda *args: scanners.MentionsScanner().compute(*args), "mentions: rank mentions by their usage", - MentionsScanner.help(), + scanners.MentionsScanner.help(), ) bot.register_command( "(emojis?|emotes?)", - lambda *args: EmotesScanner().compute(*args), + lambda *args: scanners.EmojisScanner().compute(*args), "emojis: rank emojis by their usage", - EmotesScanner.help(), + scanners.EmojisScanner.help(), ) bot.register_command( "(react(ions?)?)", - lambda *args: ReactionsScanner().compute(*args), + lambda *args: scanners.ReactionsScanner().compute(*args), "react: rank users by their reactions", - ReactionsScanner.help(), + scanners.ReactionsScanner.help(), ) bot.register_command( "(channels?|chan)", - lambda *args: ChannelsScanner().compute(*args), + lambda *args: scanners.ChannelsScanner().compute(*args), "chan: rank channels by their messages", - ChannelsScanner.help(), + scanners.ChannelsScanner.help(), ) bot.register_command( "(messages?|msg)", - lambda *args: MessagesScanner().compute(*args), + lambda *args: scanners.MessagesScanner().compute(*args), "msg: rank users by their messages", - MessagesScanner.help(), + scanners.MessagesScanner.help(), ) bot.register_command( "pres(ence)?", - lambda *args: PresenceScanner().compute(*args), + lambda *args: scanners.PresenceScanner().compute(*args), "pres: presence analysis", - PresenceScanner.help(), + scanners.PresenceScanner.help(), ) bot.register_command( "compo(sition)?", - lambda *args: CompositionScanner().compute(*args), + lambda *args: scanners.CompositionScanner().compute(*args), "compo: composition analysis", - CompositionScanner.help(), + scanners.CompositionScanner.help(), ) bot.register_command( "freq(ency)?", - lambda *args: FrequencyScanner().compute(*args), + lambda *args: scanners.FrequencyScanner().compute(*args), "freq: frequency analysis", - FrequencyScanner.help(), + scanners.FrequencyScanner.help(), ) bot.register_command( "(full|scan)", - lambda *args: FullScanner().compute(*args), + lambda *args: scanners.FullScanner().compute(*args), "scan: full analysis", - FullScanner.help(), + scanners.FullScanner.help(), ) bot.start() diff --git a/src/scanners/__init__.py b/src/scanners/__init__.py index ed9141d..c174ed7 100644 --- a/src/scanners/__init__.py +++ b/src/scanners/__init__.py @@ -1,14 +1,17 @@ -from .emotes_scanner import EmotesScanner -from .frequency_scanner import FrequencyScanner -from .composition_scanner import CompositionScanner -from .presence_scanner import PresenceScanner -from .full_scanner import FullScanner -from .mentions_scanner import MentionsScanner -from .mentioned_scanner import MentionedScanner -from .messages_scanner import MessagesScanner +from .scanner import Scanner + from .channels_scanner import ChannelsScanner -from .reactions_scanner import ReactionsScanner +from .composition_scanner import CompositionScanner +from .emojis_scanner import EmojisScanner +from .find_scanner import FindScanner from .first_scanner import FirstScanner +from .frequency_scanner import FrequencyScanner +from .full_scanner import FullScanner from .last_scanner import LastScanner +from .mentioned_scanner import MentionedScanner +from .mentions_scanner import MentionsScanner +from .messages_scanner import MessagesScanner +from .presence_scanner import PresenceScanner from .random_scanner import RandomScanner +from .reactions_scanner import ReactionsScanner from .words_scanner import WordsScanner diff --git a/src/scanners/channels_scanner.py b/src/scanners/channels_scanner.py index c766fb4..d4dcc7f 100644 --- a/src/scanners/channels_scanner.py +++ b/src/scanners/channels_scanner.py @@ -30,7 +30,6 @@ class ChannelsScanner(Scanner): ) async def init(self, message: discord.Message, *args: str) -> bool: - # get max emotes to view self.top = 10 for arg in args: if arg.isdigit(): @@ -62,6 +61,7 @@ class ChannelsScanner(Scanner): total_usage=usage_count, counted="message", transform=lambda id: f" by {mention(id)}", + top=len(self.members) != 1, ) for name in names ] diff --git a/src/scanners/composition_scanner.py b/src/scanners/composition_scanner.py index a2f3822..be64c22 100644 --- a/src/scanners/composition_scanner.py +++ b/src/scanners/composition_scanner.py @@ -57,19 +57,19 @@ class CompositionScanner(Scanner): impacted = True compo.total_characters += len(message.content) - emotes_found = emojis.regex.findall(message.content) - without_emote = message.content - for name in emotes_found: + emojis_found = emojis.regex.findall(message.content) + without_emoji = message.content + for name in emojis_found: if name in emojis.unicode_list or re.match( r"(|:[\w\\-\~]+:)", name ): - compo.emotes[name] += 1 - i = without_emote.index(name) - without_emote = without_emote[:i] + without_emote[i + len(name) :] - if len(message.content.strip()) > 0 and len(without_emote.strip()) == 0: - compo.emote_only += 1 - if len(emotes_found) > 0: - compo.emote_msg += 1 + compo.emojis[name] += 1 + i = without_emoji.index(name) + without_emoji = without_emoji[:i] + without_emoji[i + len(name) :] + if len(message.content.strip()) > 0 and len(without_emoji.strip()) == 0: + compo.emoji_only += 1 + if len(emojis_found) > 0: + compo.emoji_msg += 1 links_found = re.findall(r"https?:\/\/", message.content) compo.links += len(links_found) @@ -102,7 +102,7 @@ class CompositionScanner(Scanner): compo.tts += 1 if ( - len(emotes_found) == 0 + len(emojis_found) == 0 and message.reference is None and not message.image and len(message.mentions) == 0 diff --git a/src/scanners/emotes_scanner.py b/src/scanners/emojis_scanner.py similarity index 72% rename from src/scanners/emotes_scanner.py rename to src/scanners/emojis_scanner.py index b126812..c1c0de2 100644 --- a/src/scanners/emotes_scanner.py +++ b/src/scanners/emojis_scanner.py @@ -6,12 +6,12 @@ import discord # Custom libs from logs import ChannelLogs, MessageLog -from data_types import Emote, get_emote_dict +from data_types import Emoji, get_emoji_dict from .scanner import Scanner from utils import emojis, generate_help, plural, precise -class EmotesScanner(Scanner): +class EmojisScanner(Scanner): @staticmethod def help() -> str: return generate_help( @@ -31,13 +31,13 @@ class EmotesScanner(Scanner): super().__init__( has_digit_args=True, valid_args=["all", "members", "sort:usage", "sort:reaction", "everyone"], - help=EmotesScanner.help(), + help=EmojisScanner.help(), intro_context="Emoji usage", ) async def init(self, message: discord.Message, *args: str) -> bool: guild = message.channel.guild - # get max emotes to view + # get max emojis to view self.top = 20 for arg in args: if arg.isdigit(): @@ -47,8 +47,8 @@ class EmotesScanner(Scanner): self.show_members = "members" in args and ( len(self.members) == 0 or len(self.members) > 1 ) - # Create emotes dict from custom emojis of the guild - self.emotes = get_emote_dict(guild) + # Create emojis dict from custom emojis of the guild + self.emojis = get_emoji_dict(guild) self.sort = None if "sort:usage" in args: self.sort = "usage" @@ -58,36 +58,36 @@ class EmotesScanner(Scanner): return True def compute_message(self, channel: ChannelLogs, message: MessageLog): - return EmotesScanner.analyse_message( + return EmojisScanner.analyse_message( message, - self.emotes, + self.emojis, self.raw_members, all_emojis=self.all_emojis, all_messages=self.all_messages, ) def get_results(self, intro: str) -> List[str]: - names = [name for name in self.emotes] + names = [name for name in self.emojis] names.sort( - key=lambda name: self.emotes[name].score( + key=lambda name: self.emojis[name].score( usage_weight=(0 if self.sort == "reaction" else 1), react_weight=(0 if self.sort == "usage" else 1), ), reverse=True, ) names = names[: self.top] - # Get the total of all emotes used + # Get the total of all emojis used usage_count = 0 reaction_count = 0 - for name in self.emotes: - usage_count += self.emotes[name].usages - reaction_count += self.emotes[name].reactions + for name in self.emojis: + usage_count += self.emojis[name].usages + reaction_count += self.emojis[name].reactions res = [intro] allow_unused = self.full and len(self.members) == 0 if self.sort is not None: res += [f"(Sorted by {self.sort})"] res += [ - self.emotes[name].to_string( + self.emojis[name].to_string( names.index(name), name, total_usage=usage_count, @@ -96,7 +96,7 @@ class EmotesScanner(Scanner): show_members=self.show_members or len(self.raw_members) == 0, ) for name in names - if allow_unused or self.emotes[name].used() + if allow_unused or self.emojis[name].used() ] res += [ f"Total: {plural(usage_count,'time')} ({precise(usage_count/self.msg_count)}/msg)" @@ -108,7 +108,7 @@ class EmotesScanner(Scanner): @staticmethod def analyse_message( message: MessageLog, - emotes: Dict[str, Emote], + emojis_dict: Dict[str, Emoji], raw_members: List[int], *, all_emojis: bool, @@ -122,27 +122,29 @@ class EmotesScanner(Scanner): or message.author in raw_members ): impacted = True - # Find all emotes un the current message in the form "<:emoji:123456789>" - # Filter for known emotes + # Find all emojis un the current message in the form "<:emoji:123456789>" + # Filter for known emojis found = emojis.regex.findall(message.content) - # For each emote, update its usage + # For each emoji, update its usage for name in found: - if name not in emotes: + if name not in emojis_dict: if not all_emojis or name not in emojis.unicode_list: continue - emotes[name].usages += 1 - emotes[name].update_use(message.created_at, [message.author]) - # For each reaction of this message, test if known emote and update when it's the case + emojis_dict[name].usages += 1 + emojis_dict[name].update_use(message.created_at, [message.author]) + # For each reaction of this message, test if known emoji and update when it's the case for name in message.reactions: - if name not in emotes: + if name not in emojis_dict: if not all_emojis or name not in emojis.unicode_list: continue if len(raw_members) == 0: - emotes[name].reactions += len(message.reactions[name]) - emotes[name].update_use(message.created_at, message.reactions[name]) + emojis_dict[name].reactions += len(message.reactions[name]) + emojis_dict[name].update_use( + message.created_at, message.reactions[name] + ) else: for member in raw_members: if member in message.reactions[name]: - emotes[name].reactions += 1 - emotes[name].update_use(message.created_at, [member]) + emojis_dict[name].reactions += 1 + emojis_dict[name].update_use(message.created_at, [member]) return impacted diff --git a/src/scanners/find_scanner.py b/src/scanners/find_scanner.py new file mode 100644 index 0000000..34a5397 --- /dev/null +++ b/src/scanners/find_scanner.py @@ -0,0 +1,105 @@ +from typing import Dict, List +from collections import defaultdict +import discord +import re + +# Custom libs + +from logs import ChannelLogs, MessageLog +from .scanner import Scanner +from data_types import Counter +from utils import ( + generate_help, + plural, + precise, + mention, +) + + +class FindScanner(Scanner): + @staticmethod + def help() -> str: + return generate_help( + "find", + "Find specific words or phrases (you can use quotes to add spaces in queries)", + args=[ + "all/everyone - include bots", + ], + example='#mychannel1 #mychannel2 @user "I love you" "you too"', + ) + + def __init__(self): + super().__init__( + all_args=True, + valid_args=["all", "everyone"], + help=FindScanner.help(), + intro_context="Matches", + ) + + async def init(self, message: discord.Message, *args: str) -> bool: + self.matches = defaultdict(Counter) + self.all_messages = "all" in args or "everyone" in args + if len(self.other_args) == 0: + await message.channel.send( + "You need to add a query to find (you can use quotes to add spaces in queries)", + reference=message, + ) + return False + return True + + def compute_message(self, channel: ChannelLogs, message: MessageLog): + return FindScanner.analyse_message( + message, + self.matches, + self.other_args, + self.raw_members, + all_messages=self.all_messages, + ) + + def get_results(self, intro: str) -> List[str]: + matches = [match for match in self.matches] + matches.sort(key=lambda match: self.matches[match].score(), reverse=True) + usage_count = Counter.total(self.matches) + res = [intro] + res += [ + self.matches[match].to_string( + matches.index(match), + f"`{match}`", + total_usage=self.msg_count, + ranking=False, + transform=lambda id: f" by {mention(id)}", + top=len(self.members) != 1, + ) + for match in matches + ] + if len(matches) > 1: + res += [ + f"Total: {plural(usage_count,'time')} ({precise(usage_count/self.msg_count)}/msg)" + ] + return res + + special_cases = ["'s", "s"] + + @staticmethod + def analyse_message( + message: MessageLog, + matches: Dict[str, Counter], + queries: List[str], + raw_members: List[int], + *, + all_messages: bool, + ) -> bool: + impacted = False + # If author is included in the selection (empty list is all) + if ( + (not message.bot or all_messages) + and len(raw_members) == 0 + or message.author in raw_members + ): + impacted = True + content = message.content.lower() + for query in queries: + matches[query].update_use( + content.count(query.lower()), message.created_at, message.author + ) + return impacted diff --git a/src/scanners/full_scanner.py b/src/scanners/full_scanner.py index ac5cb0e..e07cf9d 100644 --- a/src/scanners/full_scanner.py +++ b/src/scanners/full_scanner.py @@ -5,7 +5,9 @@ import discord # Custom libs from .scanner import Scanner -from . import FrequencyScanner, CompositionScanner, PresenceScanner +from .composition_scanner import CompositionScanner +from .frequency_scanner import FrequencyScanner +from .presence_scanner import PresenceScanner from data_types import Frequency, Composition, Presence from logs import ChannelLogs, MessageLog from utils import generate_help diff --git a/src/scanners/mentioned_scanner.py b/src/scanners/mentioned_scanner.py index fa6c09e..b2f3fe4 100644 --- a/src/scanners/mentioned_scanner.py +++ b/src/scanners/mentioned_scanner.py @@ -31,7 +31,6 @@ class MentionedScanner(Scanner): ) async def init(self, message: discord.Message, *args: str) -> bool: - # get max emotes to view self.top = 10 for arg in args: if arg.isdigit(): @@ -55,7 +54,6 @@ class MentionedScanner(Scanner): names = [name for name in self.mentions] names.sort(key=lambda name: self.mentions[name].score(), reverse=True) names = names[: self.top] - # Get the total of all emotes used usage_count = Counter.total(self.mentions) res = [intro] res += [ @@ -63,6 +61,8 @@ class MentionedScanner(Scanner): names.index(name), name, total_usage=usage_count, + transform=lambda id: f" for {mention(id)}", + top=len(self.members) != 1, ) for name in names ] @@ -87,6 +87,6 @@ class MentionedScanner(Scanner): mention(member_id) ) + message.content.count(alt_mention(member_id)) mentions[mention(message.author)].update_use( - count, message.created_at + count, message.created_at, member_id ) return impacted diff --git a/src/scanners/mentions_scanner.py b/src/scanners/mentions_scanner.py index 50a0f5c..10cecf2 100644 --- a/src/scanners/mentions_scanner.py +++ b/src/scanners/mentions_scanner.py @@ -42,7 +42,6 @@ class MentionsScanner(Scanner): ) async def init(self, message: discord.Message, *args: str) -> bool: - # get max emotes to view self.top = 10 for arg in args: if arg.isdigit(): @@ -67,7 +66,6 @@ class MentionsScanner(Scanner): names = [name for name in self.mentions] names.sort(key=lambda name: self.mentions[name].score(), reverse=True) names = names[: self.top] - # Get the total of all emotes used usage_count = Counter.total(self.mentions) res = [intro] res += [ @@ -75,6 +73,8 @@ class MentionsScanner(Scanner): names.index(name), name, total_usage=usage_count, + transform=lambda id: f" by {mention(id)}", + top=len(self.members) != 1, ) for name in names ] @@ -105,24 +105,28 @@ class MentionsScanner(Scanner): count = message.content.count(name) + message.content.count( alt_mention(member_id) ) - mentions[name].update_use(count, message.created_at) + mentions[name].update_use(count, message.created_at, message.author) if all_mentions: for role_id in message.role_mentions: name = role_mention(role_id) mentions[name].update_use( - message.content.count(name), message.created_at + message.content.count(name), message.created_at, message.author ) for channel_id in message.channel_mentions: name = channel_mention(channel_id) mentions[name].update_use( - message.content.count(name), message.created_at + message.content.count(name), message.created_at, message.author ) if "@everyone" in message.content: mentions["@\u200beveryone"].update_use( - message.content.count("@everyone"), message.created_at + message.content.count("@everyone"), + message.created_at, + message.author, ) if "@here" in message.content: mentions["@\u200bhere"].update_use( - message.content.count("@here"), message.created_at + message.content.count("@here"), + message.created_at, + message.author, ) return impacted diff --git a/src/scanners/messages_scanner.py b/src/scanners/messages_scanner.py index a79735e..086a123 100644 --- a/src/scanners/messages_scanner.py +++ b/src/scanners/messages_scanner.py @@ -30,7 +30,6 @@ class MessagesScanner(Scanner): ) async def init(self, message: discord.Message, *args: str) -> bool: - # get max emotes to view self.top = 10 for arg in args: if arg.isdigit(): @@ -62,6 +61,7 @@ class MessagesScanner(Scanner): total_usage=usage_count, counted="message", transform=lambda id: f" in {channel_mention(id)}", + top=self.channels != 1, ) for name in names ] diff --git a/src/scanners/reactions_scanner.py b/src/scanners/reactions_scanner.py index 3603a06..8b68c36 100644 --- a/src/scanners/reactions_scanner.py +++ b/src/scanners/reactions_scanner.py @@ -29,7 +29,6 @@ class ReactionsScanner(Scanner): ) async def init(self, message: discord.Message, *args: str) -> bool: - # get max emotes to view self.top = 10 for arg in args: if arg.isdigit(): @@ -59,6 +58,7 @@ class ReactionsScanner(Scanner): total_usage=usage_count, counted="reaction", transform=lambda id: f" in {channel_mention(id)}", + top=self.channels != 1, ) for name in names ] diff --git a/src/scanners/scanner.py b/src/scanners/scanner.py index b4a96f1..e5b41b4 100644 --- a/src/scanners/scanner.py +++ b/src/scanners/scanner.py @@ -14,6 +14,7 @@ from utils import ( ISO8601_REGEX, RELATIVE_REGEX, parse_time, + command_cache, ) from logs import ( GuildLogs, @@ -26,6 +27,8 @@ from logs import ( class Scanner(ABC): + VALID_ARGS = ["me", "here", "fast", "fresh", "mobile", "mention"] + def __init__( self, *, @@ -33,12 +36,16 @@ class Scanner(ABC): valid_args: List[str] = [], help: str, intro_context: str, + all_args: bool = False, ): self.has_digit_args = has_digit_args self.valid_args = valid_args + self.all_args = all_args self.help = help self.intro_context = intro_context + self.other_args = [] + self.members = [] self.raw_members = [] self.full = False @@ -48,191 +55,221 @@ class Scanner(ABC): self.chan_count = 0 async def compute( - self, client: discord.client, message: discord.Message, *args: str + self, + client: discord.client, + message: discord.Message, + *args: str, + other_mentions: List[str] = [], ): args = list(args) guild = message.guild - with GuildLogs(guild) as logs: - # If "%cmd help" redirect to "%help cmd" - if "help" in args: - await client.bot.help(client, message, "help", args[0]) - return + progress = None + try: + with GuildLogs(guild) as logs: + # If "%cmd help" redirect to "%help cmd" + if len(args) > 1 and args[1] == "help": + await client.bot.help(client, message, "help", args[0]) + return - # check args validity - str_channel_mentions = [ - str(channel.id) for channel in message.channel_mentions - ] - str_mentions = [str(member.id) for member in message.mentions] - dates = [] - for i, arg in enumerate(args[1:]): - skip_check = False - if re.match(r"^<@!?\d+>$", arg): - arg = arg[3:-1] if "!" in arg else arg[2:-1] - elif re.match(r"^<#!?\d+>$", arg): - arg = arg[3:-1] if "!" in arg else arg[2:-1] - elif re.match(ISO8601_REGEX, arg) or re.match(RELATIVE_REGEX, arg): - dates += [parse_time(arg)] - skip_check = True - if len(dates) > 2: - await message.channel.send( - f"Too many date arguments: `{arg}`", reference=message - ) - return - if ( - arg not in self.valid_args + ["me", "here", "fast", "fresh"] - and (not arg.isdigit() or not self.has_digit_args) - and arg not in str_channel_mentions - and arg not in str_mentions - and not skip_check - ): + # check args validity + str_channel_mentions = [ + str(channel.id) for channel in message.channel_mentions + ] + str_mentions = [str(member.id) for member in message.mentions] + dates = [] + for i, arg in enumerate(args[1:]): + skip_check = False + if re.match(r"^<@!?\d+>$", arg): + arg = arg[3:-1] if "!" in arg else arg[2:-1] + elif re.match(r"^<#!?\d+>$", arg): + arg = arg[3:-1] if "!" in arg else arg[2:-1] + elif re.match(ISO8601_REGEX, arg) or re.match(RELATIVE_REGEX, arg): + dates += [parse_time(arg)] + skip_check = True + if len(dates) > 2: + await message.channel.send( + f"Too many date arguments: `{arg}`", reference=message + ) + return + if ( + arg not in self.valid_args + Scanner.VALID_ARGS + and (not arg.isdigit() or not self.has_digit_args) + and arg not in str_channel_mentions + and arg not in str_mentions + and arg not in other_mentions + and not skip_check + and len(arg) > 0 + ): + if self.all_args: + self.other_args += [arg] + else: + await message.channel.send( + f"Unrecognized argument: `{arg}`", reference=message + ) + return + + self.start_date = None if len(dates) < 1 else min(dates) + self.stop_date = None if len(dates) < 2 else max(dates) + + if self.start_date is not None and self.start_date > datetime.now(): await message.channel.send( - f"Unrecognized argument: `{arg}`", reference=message + f"Start date is after today", reference=message ) return - self.start_date = None if len(dates) < 1 else min(dates) - self.stop_date = None if len(dates) < 2 else max(dates) + # Get selected channels or all of them if no channel arguments + self.channels = no_duplicate(message.channel_mentions) - if self.start_date is not None and self.start_date > datetime.now(): - await message.channel.send( - f"Start date is after today", reference=message - ) - return + # transform the "here" arg + if "here" in args: + self.channels += [message.channel] - # Get selected channels or all of them if no channel arguments - self.channels = no_duplicate(message.channel_mentions) + self.full = len(self.channels) == 0 + if self.full: + self.channels = guild.text_channels - # transform the "here" arg - if "here" in args: - self.channels += [message.channel] + # Get selected members + self.members = no_duplicate(message.mentions) + self.raw_members = no_duplicate(message.raw_mentions) - self.full = len(self.channels) == 0 - if self.full: - self.channels = guild.text_channels + # transform the "me" arg + if "me" in args: + self.members += [message.author] + self.raw_members += [message.author.id] - # Get selected members - self.members = no_duplicate(message.mentions) - self.raw_members = no_duplicate(message.raw_mentions) + self.mention_users = "mention" in args or "mobile" in args - # transform the "me" arg - if "me" in args: - self.members += [message.author] - self.raw_members += [message.author.id] + if not await self.init(message, *args): + return - if not await self.init(message, *args): - return - - # Start computing data - async with message.channel.typing(): - progress = await message.channel.send( - "```Starting analysis...```", - reference=message, - allowed_mentions=discord.AllowedMentions.none(), - ) - total_msg, total_chan = await logs.load( - progress, - self.channels, - self.start_date, - self.stop_date, - fast="fast" in args, - fresh="fresh" in args, - ) - if total_msg == CANCELLED: - await message.channel.send( - "Operation cancelled by user", + # Start computing data + async with message.channel.typing(): + progress = await message.channel.send( + "```Starting analysis...```", reference=message, + allowed_mentions=discord.AllowedMentions.none(), ) - elif total_msg == ALREADY_RUNNING: - await message.channel.send( - "An analysis is already running on this server, please be patient.", - reference=message, + total_msg, total_chan = await logs.load( + progress, + self.channels, + self.start_date, + self.stop_date, + fast="fast" in args, + fresh="fresh" in args, ) - elif total_msg == NO_FILE: - await message.channel.send(gdpr.TEXT) - else: - if self.start_date is not None and len(logs.channels) > 0: - self.start_date = max( - self.start_date, - min( - [ - logs.channels[channel.id].start_date - for channel in self.channels - if channel.id in logs.channels - and logs.channels[channel.id].start_date is not None - ] - ), - ) - if self.stop_date is None: - self.stop_date = datetime.utcnow() - - self.msg_count = 0 - self.total_msg = 0 - self.chan_count = 0 - t0 = datetime.now() - for channel in self.channels: - if channel.id in logs.channels: - channel_logs = logs.channels[channel.id] - count = sum( - [ - self.compute_message(channel_logs, message_log) - for message_log in channel_logs.messages - if ( - self.start_date is None - or message_log.created_at >= self.start_date - ) - and ( - self.stop_date is None - or message_log.created_at <= self.stop_date - ) - ] - ) - self.total_msg += len(channel_logs.messages) - self.msg_count += count - self.chan_count += 1 if count > 0 else 0 - logging.info(f"scan {guild.id} > scanned in {delta(t0):,}ms") - if self.msg_count == 0: + if total_msg == CANCELLED: await message.channel.send( - "There are no messages found matching the filters", + "Operation cancelled by user", reference=message, ) - else: - await progress.edit(content="```Computing results...```") - # Display results - t0 = datetime.now() - results = self.get_results( - get_intro( - self.intro_context, - self.full, - self.channels, - self.members, - self.msg_count, - self.chan_count, - self.start_date, - self.stop_date, - ) + elif total_msg == ALREADY_RUNNING: + await message.channel.send( + "An analysis is already running on this server, please be patient.", + reference=message, ) - logging.info(f"scan {guild.id} > results in {delta(t0):,}ms") - response = "" - first = True - for r in results: - if r: - if len(response + "\n" + r) > 2000: - await message.channel.send( - response, - reference=message if first else None, - allowed_mentions=discord.AllowedMentions.none(), - ) - first = False - response = "" - response += "\n" + r - if len(response) > 0: - await message.channel.send( - response, - reference=message if first else None, - allowed_mentions=discord.AllowedMentions.none(), + elif total_msg == NO_FILE: + await message.channel.send(gdpr.TEXT) + else: + if self.start_date is not None and len(logs.channels) > 0: + self.start_date = max( + self.start_date, + min( + [ + logs.channels[channel.id].start_date + for channel in self.channels + if channel.id in logs.channels + and logs.channels[channel.id].start_date + is not None + ] + ), ) + if self.stop_date is None: + self.stop_date = datetime.utcnow() + + self.msg_count = 0 + self.total_msg = 0 + self.chan_count = 0 + t0 = datetime.now() + for channel in self.channels: + if channel.id in logs.channels: + channel_logs = logs.channels[channel.id] + count = sum( + [ + self.compute_message(channel_logs, message_log) + for message_log in channel_logs.messages + if ( + self.start_date is None + or message_log.created_at >= self.start_date + ) + and ( + self.stop_date is None + or message_log.created_at <= self.stop_date + ) + ] + ) + self.total_msg += len(channel_logs.messages) + self.msg_count += count + self.chan_count += 1 if count > 0 else 0 + logging.info(f"scan {guild.id} > scanned in {delta(t0):,}ms") + if self.msg_count == 0: + await message.channel.send( + "There are no messages found matching the filters", + reference=message, + ) + else: + await progress.edit(content="```Computing results...```") + # Display results + t0 = datetime.now() + results = self.get_results( + get_intro( + self.intro_context, + self.full, + self.channels, + self.members, + self.msg_count, + self.chan_count, + self.start_date, + self.stop_date, + ) + ) + logging.info( + f"scan {guild.id} > results in {delta(t0):,}ms" + ) + response = "" + first = True + allowed_mentions = ( + discord.AllowedMentions.all() + if self.mention_users + else discord.AllowedMentions.none() + ) + for r in results: + if r: + if len(response + "\n" + r) > 2000: + await message.channel.send( + response, + reference=message if first else None, + allowed_mentions=allowed_mentions, + ) + first = False + response = "" + response += "\n" + r + if len(response) > 0: + await message.channel.send( + response, + reference=message if first else None, + allowed_mentions=allowed_mentions, + ) + command_cache.cache(self, message, args) # Delete custom progress message await progress.delete() + except Exception as error: + logging.exception(error) + await message.channel.send( + "An unexpected error happened while computing your command, we're sorry for the inconvenience.", + reference=message, + ) + if progress is not None: + await progress.delete() @abstractmethod async def init(self, message: discord.Message, *args: str) -> bool: diff --git a/src/scanners/words_scanner.py b/src/scanners/words_scanner.py index f7f6dd7..f5c2528 100644 --- a/src/scanners/words_scanner.py +++ b/src/scanners/words_scanner.py @@ -8,11 +8,7 @@ import re from logs import ChannelLogs, MessageLog from .scanner import Scanner from data_types import Counter -from utils import ( - generate_help, - plural, - precise, -) +from utils import generate_help, plural, precise, mention class WordsScanner(Scanner): @@ -67,15 +63,15 @@ class WordsScanner(Scanner): words = [word for word in self.words] words.sort(key=lambda word: self.words[word].score(), reverse=True) words = words[: self.top] - # Get the total of all emotes used usage_count = Counter.total(self.words) - print(len(self.words)) res = [intro.format(self.letters)] res += [ self.words[word].to_string( words.index(word), f"`{word}`", total_usage=usage_count, + transform=lambda id: f" by {mention(id)}", + top=len(self.members) != 1, ) for word in words ] @@ -122,5 +118,5 @@ class WordsScanner(Scanner): words[word] = words[word + case] del words[word + case] break - words[word].update_use(1, message.created_at) + words[word].update_use(1, message.created_at, message.author) return impacted diff --git a/src/utils/command_cache.py b/src/utils/command_cache.py new file mode 100644 index 0000000..252313e --- /dev/null +++ b/src/utils/command_cache.py @@ -0,0 +1,45 @@ +from typing import List +import logging +import discord + +from scanners import Scanner + +command_cache = {} + + +def cache(scanner: Scanner, message: discord.Message, args: List[str]): + id = message.channel.id + command_cache[id] = ( + type(scanner), + list(args), + [str(channel.id) for channel in message.channel_mentions] + + [str(member.id) for member in message.mentions], + ) + + +async def repeat( + client: discord.client, + message: discord.Message, + *args: str, + add_args: List[str] = [], +): + if len(args) > 1 and args[1] == "help": + await client.bot.help(client, message, "help", args[0]) + return + id = message.channel.id + if id not in command_cache: + await message.channel.send( + "No command to repeat on this channel (type %help for more info)", + reference=message, + ) + return + ( + scannerType, + original_args, + original_mentions, + ) = command_cache[id] + args = original_args + add_args + list(args[1:]) + ["fast"] + logging.info(f"repeating {args}") + await scannerType().compute( + client, message, *args, other_mentions=original_mentions + ) diff --git a/src/utils/gdpr.py b/src/utils/gdpr.py index 5ae85c6..e14fdb4 100644 --- a/src/utils/gdpr.py +++ b/src/utils/gdpr.py @@ -49,10 +49,10 @@ async def process(client: discord.client, message: discord.Message, *args: str): args = list(args) if len(args) == 1: await message.channel.send(TEXT) + elif args[1] == "help": + await client.bot.help(client, message, "help", args[0]) elif len(args) > 2: await message.channel.send(f"Too many arguments", reference=message) - elif args[1] == "help": - await message.channel.send(HELP, reference=message) elif args[1] in ["agree", "accept"]: GuildLogs.init_log(message.channel.guild) await message.channel.send(AGREE_TEXT, reference=message) diff --git a/src/utils/utils.py b/src/utils/utils.py index a439ffd..7e5e9b6 100644 --- a/src/utils/utils.py +++ b/src/utils/utils.py @@ -18,6 +18,7 @@ COMMON_HELP_ARGS = [ " - filter before ", "fast - only read cache", "fresh - does not read cache (long)", + "mobile/mention - mentions users (fix @invalid-user bug)", ] @@ -180,15 +181,16 @@ def parse_iso_datetime(str_date: str) -> datetime: return dateutil.parser.parse(str_date) -RELATIVE_REGEX = r"(yesterday|today|\d*h(ours?)?|\d*d(ays?)?|\d*w(eeks?)?|\d*m(onths?)?|\d*y(ears?)?)" +RELATIVE_REGEX = r"(yesterday|today|\d*hours?|\d+h(ours?)?|\d*days?|\d+d(ays?)?|\d*weeks?|\d+w(eeks?)?|\d*months?|\d+m(onths?)?|\d*years?|\d+y(ears?)?)" def parse_relative_time(src: str) -> datetime: - timezone_delta = datetime.utcnow() - datetime.now() + today = datetime.utcnow().date() + today = datetime(today.year, today.month, today.day) if src == "today": - return datetime.today() + timezone_delta + return today elif src == "yesterday": - return datetime.today() - relativedelta(days=1) + timezone_delta + return today - relativedelta(days=1) else: m = re.match("(\d*)(\w+)", src) delta = None