loading big history by 10k chunks

emotes.py

@@ -6,6 +6,10 @@ import re
 import help
 from utils import debug, aggregate, no_duplicate
 
+# CONSTANTS
+
+CHUNK_SIZE = 10000
+
 
 # MAIN
 
@@ -18,7 +22,6 @@ async def compute(message, args):
     :param args: arguments of the command
     :type args: list[:class:`str`]
     """
-    debug(message, f"command '{message.content}'")
 
     guild = message.guild
 
@@ -81,6 +84,7 @@ class Emote:
     :ivar last_used: date of last use
     :vartype last_used: datetime
     """
+
     def __init__(self, emoji):
         self.emoji = emoji
         self.usages = 0
@@ -157,32 +161,38 @@ async def analyse_channel(channel, emotes, members, progress, nm0, nc):
     nm = 0
     nmm = 0
     try:
-        # Read ALL messages from the channel (pretty long : 300 msg/s)
-        async for m in channel.history(limit=None):
-            # If author is not bot or included in the selection (empty list is all)
-            if not m.author.bot and (len(members) == 0 or m.author in members):
-                # Find all emotes un the current message in the form "<:emoji:123456789>"
-                # Filter for known emotes
-                found = [name for name in re.findall(r"(<:\w+:\d+>)", m.content) if name in emotes]
-                # For each emote, update its usage
-                for name in found:
-                    emotes[name].usages += 1
-                    emotes[name].update_use(m.created_at)
-                # Count this message as impacted
-                nmm += 1
-            # If we include all members, get reactions
-            if len(members) == 0:
+        messages = [None]
+        while len(messages) >= CHUNK_SIZE or messages[-1] is None:
+            messages = await channel.history(limit=CHUNK_SIZE, before=messages[-1]).flatten()
+            for m in messages:
+                tm0 = datetime.now()
+                # If author is not bot or included in the selection (empty list is all)
+                if not m.author.bot and (len(members) == 0 or m.author in members):
+                    # Find all emotes un the current message in the form "<:emoji:123456789>"
+                    # Filter for known emotes
+                    found = [name for name in re.findall(r"(<:\w+:\d+>)", m.content) if name in emotes]
+                    # For each emote, update its usage
+                    for name in found:
+                        emotes[name].usages += 1
+                        emotes[name].update_use(m.created_at)
+                    # Count this message as impacted
+                    nmm += 1
                 # For each reaction of this message, test if known emote and update when it's the case
                 for reaction in m.reactions:
                     name = str(reaction.emoji)
                     # reaction.emoji can be only str, we don't want that
                     if not (isinstance(reaction.emoji, str)) and name in emotes:
-                        emotes[name].reactions += reaction.count
-                        emotes[name].update_use(m.created_at)
-            # Count this message as treated and show progress every 1k messages
-            nm += 1
-            if (nm0 + nm) % 1000 == 0:
-                await progress.edit(content=f"```{(nm0 + nm) // 1000}k messages and {nc} channels analysed```")
+                        if len(members) == 0:
+                            emotes[name].reactions += reaction.count
+                            emotes[name].update_use(m.created_at)
+                        """ else:
+                            users = await reaction.users().flatten()
+                            for member in members:
+                                if member in users:
+                                    emotes[name].reactions += 1
+                                    emotes[name].update_use(m.created_at)"""
+            nm += len(messages)
+            await progress.edit(content=f"```{nm0 + nm:,} messages and {nc} channels analysed```")
         return nm, nmm
     except discord.errors.HTTPException:
         # When an exception occurs (like Forbidden) sent -1