v1.10

Dockerfile

@@ -3,10 +3,14 @@ FROM python
 # Create app directory
 WORKDIR /usr/src/app
 
+VOLUME ["/usr/src/app/logs"]
+
 COPY requirements.txt ./
 
 RUN pip install -r requirements.txt
 
+RUN touch logs/guilds.log && ln -s logs/guilds.log guilds.log
+
 # Bundle app source
 COPY . .
 

README.md

@@ -39,10 +39,11 @@ When you need statistics about your discord server
 * %cancel - cancel current analysis
 
 * Common arguments:
-    * @member/me : filter for one or more member
-    * #channel/here : filter for one or more channel
+    * @member/me: filter for one or more member
+    * #channel/here: filter for one or more channel
     * all/everyone - include bots messages
-    * fast : only read cache
+    * fast: only read cache
+    * fresh: does not read cache
 ```
 
 ## Running this bot
@@ -78,11 +79,11 @@ You will need:
 python3 src/main.py
 ```
 
-## Recommanded permissions
+## Recommended permissions
 
-[x] View Channels
-[x] Read Message History
-[x] Send Messages 
+- [x] View Channels
+- [x] Read Message History
+- [x] Send Messages
 
 > On large servers, you should disable "Send Messages" and enable it on an read-only channel where only administrators can launch commands. The bot can't be triggered elsewhere if it can't answer.
 
@@ -92,6 +93,9 @@ python3 src/main.py
 
 ## Changelog
 
+* **v1.10**
+  * multithreading for queries
+  * bug fix
 * **v1.9**:
   * `all/everyone` to include bots in scans
   * `fresh` to not use previously cached data

src/logs/channel_logs.py

@@ -4,7 +4,7 @@ import discord
 from . import MessageLog
 from utils import FakeMessage
 
-CHUNK_SIZE = 1000
+CHUNK_SIZE = 2000
 FORMAT = 3
 
 
@@ -70,6 +70,7 @@ class ChannelLogs:
                         yield len(self.messages), False
                     self.last_message_id = channel.last_message_id
         except discord.errors.HTTPException:
+            yield -1, True
             return  # When an exception occurs (like Forbidden)
         yield len(self.messages), True
 

src/logs/guild_logs.py

@@ -5,6 +5,7 @@ import json
 import gzip
 from datetime import datetime
 import logging
+import asyncio
 
 
 from . import ChannelLogs
@@ -20,6 +21,30 @@ ALREADY_RUNNING = -100
 CANCELLED = -200
 
 
+class Worker:
+    def __init__(self, channel_log: ChannelLogs, channel: discord.TextChannel):
+        self.channel_log = channel_log
+        self.channel = channel
+        self.start_msg = len(channel_log.messages)
+        self.total_msg = self.start_msg
+        self.queried_msg = 0
+        self.done = False
+        self.cancelled = False
+        self.loop = asyncio.get_event_loop()
+
+    def start(self):
+        asyncio.run_coroutine_threadsafe(self.process(), self.loop)
+
+    async def process(self):
+        async for count, done in self.channel_log.load(self.channel):
+            if count > 0:
+                self.queried_msg = count - self.start_msg
+                self.total_msg = count
+            self.done = done
+            if self.cancelled:
+                return
+
+
 class GuildLogs:
     def __init__(self, guild: discord.Guild):
         self.guild = guild
@@ -112,44 +137,60 @@ class GuildLogs:
             # load channels
             t0 = datetime.now()
             if len(target_channels) == 0:
-                target_channels = self.guild.text_channels
+                target_channels = (
+                    self.guild.text_channels if not fast else self.channels.keys()
+                )
             loading_new = 0
             queried_msg = 0
             total_chan = 0
             max_chan = len(target_channels)
             if self.check_cancelled():
                 return CANCELLED, 0
-            await code_message(
-                progress,
-                f"Reading new history...\n0 messages in 0/{max_chan:,} channels\n(this might take a while)",
-            )
+            workers = []
             for channel in target_channels:
                 if channel.id not in self.channels or fresh:
                     loading_new += 1
                     self.channels[channel.id] = ChannelLogs(channel)
-                start_msg = len(self.channels[channel.id].messages)
-                count = 0
-                async for count, done in self.channels[channel.id].load(channel):
-                    if count > 0:
-                        tmp_queried_msg = queried_msg + count - start_msg
-                        tmp_msg = total_msg + count
-                        warning_msg = "(this might take a while)"
-                        if len(target_channels) > 5 and loading_new > 5:
-                            warning_msg = "(most channels are new, this might take a looong while)"
-                        elif loading_new > 0:
-                            warning_msg = (
-                                "(some channels are new, this might take a long while)"
-                            )
-                        if self.check_cancelled():
-                            return CANCELLED, 0
-                        await code_message(
-                            progress,
-                            f"Reading new history...\n{tmp_msg:,} messages in {total_chan + 1:,}/{max_chan:,} channels ({round(tmp_queried_msg/deltas(t0)):,}m/s)\n{warning_msg}",
-                        )
-                        if done:
-                            total_chan += 1
-                total_msg += len(self.channels[channel.id].messages)
-                queried_msg += count - start_msg
+                workers += [Worker(self.channels[channel.id], channel)]
+            warning_msg = "(this might take a while)"
+            if len(target_channels) > 5 and loading_new > 5:
+                warning_msg = "(most channels are new, this will take a long while)"
+            elif loading_new > 0:
+                warning_msg = "(some channels are new, this might take a long while)"
+            await code_message(
+                progress,
+                f"Reading new history...\n0 messages in 0/{max_chan:,} channels\n{warning_msg}",
+            )
+            for worker in workers:
+                worker.start()
+            done = False
+            while not done:
+                if self.check_cancelled():
+                    for worker in workers:
+                        worker.cancelled = True
+                    return CANCELLED, 0
+
+                await asyncio.sleep(0.5)
+
+                remaining = [
+                    worker.channel.name for worker in workers if not worker.done
+                ]
+                total_chan = max_chan - len(remaining)
+                queried_msg = sum([worker.queried_msg for worker in workers])
+                total_msg = sum([worker.total_msg for worker in workers])
+
+                if total_chan == max_chan:
+                    done = True
+
+                remaining_msg = ""
+
+                if len(remaining) <= 5:
+                    remaining_msg = "\nRemaining: " + ", ".join(remaining)
+
+                await code_message(
+                    progress,
+                    f"Reading new history...\n{total_msg:,} messages in {total_chan:,}/{max_chan:,} channels ({round(queried_msg/deltas(t0)):,}m/s)\n{warning_msg}{remaining_msg}",
+                )
             logging.info(
                 f"log {self.guild.id} > queried in {delta(t0):,}ms -> {queried_msg / deltas(t0):,.3f} m/s"
             )

src/main.py

@@ -24,7 +24,7 @@ emojis.load_emojis()
 
 bot = Bot(
     "Discord Analyst",
-    "1.9",
+    "1.10",
     alias="%",
 )
 

src/scanners/scanner.py

@@ -111,49 +111,56 @@ class Scanner(ABC):
                 self.chan_count = 0
                 t0 = datetime.now()
                 for channel in self.channels:
-                    channel_logs = logs.channels[channel.id]
-                    count = sum(
-                        [
-                            self.compute_message(channel_logs, message_log)
-                            for message_log in channel_logs.messages
-                        ]
-                    )
-                    self.total_msg += len(channel_logs.messages)
-                    self.msg_count += count
-                    self.chan_count += 1 if count > 0 else 0
+                    if channel.id in logs.channels:
+                        channel_logs = logs.channels[channel.id]
+                        count = sum(
+                            [
+                                self.compute_message(channel_logs, message_log)
+                                for message_log in channel_logs.messages
+                            ]
+                        )
+                        self.total_msg += len(channel_logs.messages)
+                        self.msg_count += count
+                        self.chan_count += 1 if count > 0 else 0
                 logging.info(f"scan {guild.id} > scanned in {delta(t0):,}ms")
-                await progress.edit(content="```Computing results...```")
-                # Display results
-                t0 = datetime.now()
-                results = self.get_results(
-                    get_intro(
-                        self.intro_context,
-                        self.full,
-                        self.channels,
-                        self.members,
-                        self.msg_count,
-                        self.chan_count,
+                if self.total_msg == 0:
+                    await message.channel.send(
+                        "There are no messages found matching the filters",
+                        reference=message,
                     )
-                )
-                logging.info(f"scan {guild.id} > results in {delta(t0):,}ms")
-                response = ""
-                first = True
-                for r in results:
-                    if len(response + "\n" + r) > 2000:
+                else:
+                    await progress.edit(content="```Computing results...```")
+                    # Display results
+                    t0 = datetime.now()
+                    results = self.get_results(
+                        get_intro(
+                            self.intro_context,
+                            self.full,
+                            self.channels,
+                            self.members,
+                            self.msg_count,
+                            self.chan_count,
+                        )
+                    )
+                    logging.info(f"scan {guild.id} > results in {delta(t0):,}ms")
+                    response = ""
+                    first = True
+                    for r in results:
+                        if len(response + "\n" + r) > 2000:
+                            await message.channel.send(
+                                response,
+                                reference=message if first else None,
+                                allowed_mentions=discord.AllowedMentions.none(),
+                            )
+                            first = False
+                            response = ""
+                        response += "\n" + r
+                    if len(response) > 0:
                         await message.channel.send(
                             response,
                             reference=message if first else None,
                             allowed_mentions=discord.AllowedMentions.none(),
                         )
-                        first = False
-                        response = ""
-                    response += "\n" + r
-                if len(response) > 0:
-                    await message.channel.send(
-                        response,
-                        reference=message if first else None,
-                        allowed_mentions=discord.AllowedMentions.none(),
-                    )
             # Delete custom progress message
             await progress.delete()