loading big history by 10k chunks

emotes.py

@@ -6,6 +6,10 @@ import re
 import help
 from utils import debug, aggregate, no_duplicate
 
+# CONSTANTS
+
+CHUNK_SIZE = 10000
+
 
 # MAIN
 
@@ -18,7 +22,6 @@ async def compute(message, args):
     :param args: arguments of the command
     :type args: list[:class:`str`]
     """
-    debug(message, f"command '{message.content}'")
 
     guild = message.guild
 
@@ -81,6 +84,7 @@ class Emote:
     :ivar last_used: date of last use
     :vartype last_used: datetime
     """
+
     def __init__(self, emoji):
         self.emoji = emoji
         self.usages = 0
@@ -157,8 +161,11 @@ async def analyse_channel(channel, emotes, members, progress, nm0, nc):
     nm = 0
     nmm = 0
     try:
-        # Read ALL messages from the channel (pretty long : 300 msg/s)
+        messages = [None]
-        async for m in channel.history(limit=None):
+        while len(messages) >= CHUNK_SIZE or messages[-1] is None:
+            messages = await channel.history(limit=CHUNK_SIZE, before=messages[-1]).flatten()
+            for m in messages:
+                tm0 = datetime.now()
                 # If author is not bot or included in the selection (empty list is all)
                 if not m.author.bot and (len(members) == 0 or m.author in members):
                     # Find all emotes un the current message in the form "<:emoji:123456789>"
@@ -170,19 +177,22 @@ async def analyse_channel(channel, emotes, members, progress, nm0, nc):
                         emotes[name].update_use(m.created_at)
                     # Count this message as impacted
                     nmm += 1
-            # If we include all members, get reactions
-            if len(members) == 0:
                 # For each reaction of this message, test if known emote and update when it's the case
                 for reaction in m.reactions:
                     name = str(reaction.emoji)
                     # reaction.emoji can be only str, we don't want that
                     if not (isinstance(reaction.emoji, str)) and name in emotes:
+                        if len(members) == 0:
                             emotes[name].reactions += reaction.count
                             emotes[name].update_use(m.created_at)
-            # Count this message as treated and show progress every 1k messages
+                        """ else:
-            nm += 1
+                            users = await reaction.users().flatten()
-            if (nm0 + nm) % 1000 == 0:
+                            for member in members:
-                await progress.edit(content=f"```{(nm0 + nm) // 1000}k messages and {nc} channels analysed```")
+                                if member in users:
+                                    emotes[name].reactions += 1
+                                    emotes[name].update_use(m.created_at)"""
+            nm += len(messages)
+            await progress.edit(content=f"```{nm0 + nm:,} messages and {nc} channels analysed```")
         return nm, nmm
     except discord.errors.HTTPException:
         # When an exception occurs (like Forbidden) sent -1