From 98481e7b1df46314caa0da15c074e6db4904591c Mon Sep 17 00:00:00 2001 From: klemek Date: Thu, 14 Nov 2019 20:40:19 +0100 Subject: [PATCH 1/6] better command routing --- bot.py | 37 ++++++++++++++++++++++++++++--------- help.py | 1 - utils.py | 1 + 3 files changed, 29 insertions(+), 10 deletions(-) diff --git a/bot.py b/bot.py index bf00586..3c1424b 100644 --- a/bot.py +++ b/bot.py @@ -8,7 +8,7 @@ import emotes import help from utils import debug -VERSION = "1.0" +VERSION = "1.1-dev" t0 = datetime.now() # Loading token @@ -18,6 +18,25 @@ token = os.getenv('DISCORD_TOKEN') client = discord.Client() +async def info(message, args): + """ + Computes the %info command + + :param message: message sent + :type message: :class:`discord.Message` + :param args: arguments of the command + :type args: list[:class:`str`] + """ + await message.channel.send(f"```Discord Analyst v{VERSION} started at {t0:%Y-%m-%d %H:%M}```") + + +COMMANDS = { + "%help": help.compute, + "%emotes": emotes.compute, + "%info": info +} + + @client.event async def on_ready(): """ @@ -47,6 +66,13 @@ async def on_message(message): if message.author == client.user: return + args = message.content.split(" ") + + if len(args) < 1 or args[0] not in COMMANDS: + return + + debug(message, f"command '{message.content}'") + # Check if bot can respond on current channel or DM user permissions = message.channel.permissions_for(message.guild.me) if not permissions.send_messages: @@ -58,14 +84,7 @@ async def on_message(message): return # Redirect to the correct command - args = message.content.split(" ") - if args[0] == "%info": - debug(message, f"command '{message.content}'") - await message.channel.send(f"Discord Analyst v{VERSION} started at {t0.isoformat()}") - if args[0] == "%help": - await help.compute(message, args) - if args[0] == "%emotes": - await emotes.compute(message, args) + await COMMANDS[args[0]](message, args) # Launch client diff --git a/help.py b/help.py index 4b25418..cf6c984 100644 --- a/help.py +++ b/help.py @@ -10,7 +10,6 @@ async def compute(message, args): :param args: arguments of the command :type args: list[str] """ - debug(message, f"command '{message.content}'") # Select correct response to send diff --git a/utils.py b/utils.py index fdfa46a..66e8a49 100644 --- a/utils.py +++ b/utils.py @@ -1,5 +1,6 @@ # DISCORD API + def debug(message, txt): """ Print a log with the context of the current event From ea74778e1574faa1134d57ed9d6ab8cdf8e196b8 Mon Sep 17 00:00:00 2001 From: klemek Date: Thu, 14 Nov 2019 20:46:45 +0100 Subject: [PATCH 2/6] loading big history by 10k chunks --- emotes.py | 54 ++++++++++++++++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 22 deletions(-) diff --git a/emotes.py b/emotes.py index 5da6ccc..d52b426 100644 --- a/emotes.py +++ b/emotes.py @@ -6,6 +6,10 @@ import re import help from utils import debug, aggregate, no_duplicate +# CONSTANTS + +CHUNK_SIZE = 10000 + # MAIN @@ -18,7 +22,6 @@ async def compute(message, args): :param args: arguments of the command :type args: list[:class:`str`] """ - debug(message, f"command '{message.content}'") guild = message.guild @@ -81,6 +84,7 @@ class Emote: :ivar last_used: date of last use :vartype last_used: datetime """ + def __init__(self, emoji): self.emoji = emoji self.usages = 0 @@ -157,32 +161,38 @@ async def analyse_channel(channel, emotes, members, progress, nm0, nc): nm = 0 nmm = 0 try: - # Read ALL messages from the channel (pretty long : 300 msg/s) - async for m in channel.history(limit=None): - # If author is not bot or included in the selection (empty list is all) - if not m.author.bot and (len(members) == 0 or m.author in members): - # Find all emotes un the current message in the form "<:emoji:123456789>" - # Filter for known emotes - found = [name for name in re.findall(r"(<:\w+:\d+>)", m.content) if name in emotes] - # For each emote, update its usage - for name in found: - emotes[name].usages += 1 - emotes[name].update_use(m.created_at) - # Count this message as impacted - nmm += 1 - # If we include all members, get reactions - if len(members) == 0: + messages = [None] + while len(messages) >= CHUNK_SIZE or messages[-1] is None: + messages = await channel.history(limit=CHUNK_SIZE, before=messages[-1]).flatten() + for m in messages: + tm0 = datetime.now() + # If author is not bot or included in the selection (empty list is all) + if not m.author.bot and (len(members) == 0 or m.author in members): + # Find all emotes un the current message in the form "<:emoji:123456789>" + # Filter for known emotes + found = [name for name in re.findall(r"(<:\w+:\d+>)", m.content) if name in emotes] + # For each emote, update its usage + for name in found: + emotes[name].usages += 1 + emotes[name].update_use(m.created_at) + # Count this message as impacted + nmm += 1 # For each reaction of this message, test if known emote and update when it's the case for reaction in m.reactions: name = str(reaction.emoji) # reaction.emoji can be only str, we don't want that if not (isinstance(reaction.emoji, str)) and name in emotes: - emotes[name].reactions += reaction.count - emotes[name].update_use(m.created_at) - # Count this message as treated and show progress every 1k messages - nm += 1 - if (nm0 + nm) % 1000 == 0: - await progress.edit(content=f"```{(nm0 + nm) // 1000}k messages and {nc} channels analysed```") + if len(members) == 0: + emotes[name].reactions += reaction.count + emotes[name].update_use(m.created_at) + """ else: + users = await reaction.users().flatten() + for member in members: + if member in users: + emotes[name].reactions += 1 + emotes[name].update_use(m.created_at)""" + nm += len(messages) + await progress.edit(content=f"```{nm0 + nm:,} messages and {nc} channels analysed```") return nm, nmm except discord.errors.HTTPException: # When an exception occurs (like Forbidden) sent -1 From 355ce448245b2c5a14e0bb114aff012ce1aba5c9 Mon Sep 17 00:00:00 2001 From: klemek Date: Thu, 14 Nov 2019 20:47:03 +0100 Subject: [PATCH 3/6] comma separator for big numbers --- emotes.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/emotes.py b/emotes.py index d52b426..f3f5ed8 100644 --- a/emotes.py +++ b/emotes.py @@ -262,29 +262,29 @@ def get_intro(emotes, full, channels, members, nmm, nc): if len(members) == 0: # Full scan of the server if full: - return f"{len(emotes)} emotes in this server ({nc} channels, {nmm} messages):" + return f"{len(emotes)} emotes in this server ({nc} channels, {nmm:,} messages):" elif len(channels) < 5: - return f"{aggregate([c.mention for c in channels])} emotes usage in {nmm} messages:" + return f"{aggregate([c.mention for c in channels])} emotes usage in {nmm:,} messages:" else: - return f"These {len(channels)} channels emotes usage in {nmm} messages:" + return f"These {len(channels)} channels emotes usage in {nmm:,} messages:" elif len(members) < 5: if full: - return f"{aggregate([m.mention for m in members])} emotes usage in {nmm} messages:" + return f"{aggregate([m.mention for m in members])} emotes usage in {nmm:,} messages:" elif len(channels) < 5: return f"{aggregate([m.mention for m in members])} on {aggregate([c.mention for c in channels])} " \ - f"emotes usage in {nmm} messages:" + f"emotes usage in {nmm:,} messages:" else: return f"{aggregate([m.mention for m in members])} on these {len(channels)} channels " \ - f"emotes usage in {nmm} messages:" + f"emotes usage in {nmm:,} messages:" else: if full: - return f"These {len(members)} members emotes usage in {nmm} messages:" + return f"These {len(members)} members emotes usage in {nmm:,} messages:" elif len(channels) < 5: return f"These {len(members)} members on {aggregate([c.mention for c in channels])} " \ - f"emotes usage in {nmm} messages:" + f"emotes usage in {nmm:,} messages:" else: return f"These {len(members)} members on these {len(channels)} channels " \ - f"emotes usage in {nmm} messages:" + f"emotes usage in {nmm:,} messages:" def get_place(i): @@ -318,7 +318,7 @@ def get_usage(emote): elif emote.usages == 1: return "1 time " else: - return f"{emote.usages} times " + return f"{emote.usages:,} times " def get_reactions(emote): @@ -333,7 +333,7 @@ def get_reactions(emote): elif emote.reactions == 1: return "and 1 reaction " else: - return f"and {emote.reactions} reactions " + return f"and {emote.reactions:,} reactions " def get_life(emote, show_life): @@ -387,6 +387,6 @@ def get_total(emotes, nmm): nu += emotes[name].usages nr += emotes[name].reactions if nr > 0: - return f"Total: {nu} times ({round(nu / nmm, 4)} / message) and {nr} reactions" + return f"Total: {nu:,} times ({nu / nmm:.4f} / message) and {nr:,} reactions" else: - return f"Total: {nu} times ({round(nu / nmm, 4)} / message)" + return f"Total: {nu:,} times ({nu / nmm:.4f} / message)" From 52e46b558026d48c28a29568335125c28c50fc87 Mon Sep 17 00:00:00 2001 From: klemek Date: Thu, 14 Nov 2019 20:47:34 +0100 Subject: [PATCH 4/6] updated README.md --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index ac19553..dc0ccf8 100644 --- a/README.md +++ b/README.md @@ -43,3 +43,11 @@ You will need: ``` python3 bot.py ``` + +## Changelog + +* **v1.1**: + * coma separator for big numbers + * history loading by chunks for big channels (performance increase) + * bug fix +* **v1.0**: stable release From 49b3aded7e41d314513ba78c110852d4e67fe18c Mon Sep 17 00:00:00 2001 From: klemek Date: Thu, 14 Nov 2019 20:47:44 +0100 Subject: [PATCH 5/6] release 1.1 --- bot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot.py b/bot.py index 3c1424b..e78b9f6 100644 --- a/bot.py +++ b/bot.py @@ -8,7 +8,7 @@ import emotes import help from utils import debug -VERSION = "1.1-dev" +VERSION = "1.1" t0 = datetime.now() # Loading token From 443ab035dfe193bfb6bc71c3ea1c8c84bf9b5279 Mon Sep 17 00:00:00 2001 From: klemek Date: Thu, 14 Nov 2019 20:59:08 +0100 Subject: [PATCH 6/6] removed dead code --- emotes.py | 1 - help.py | 1 - 2 files changed, 2 deletions(-) diff --git a/emotes.py b/emotes.py index f3f5ed8..18b7979 100644 --- a/emotes.py +++ b/emotes.py @@ -165,7 +165,6 @@ async def analyse_channel(channel, emotes, members, progress, nm0, nc): while len(messages) >= CHUNK_SIZE or messages[-1] is None: messages = await channel.history(limit=CHUNK_SIZE, before=messages[-1]).flatten() for m in messages: - tm0 = datetime.now() # If author is not bot or included in the selection (empty list is all) if not m.author.bot and (len(members) == 0 or m.author in members): # Find all emotes un the current message in the form "<:emoji:123456789>" diff --git a/help.py b/help.py index cf6c984..2e371b2 100644 --- a/help.py +++ b/help.py @@ -1,4 +1,3 @@ -from utils import debug async def compute(message, args):