better memory handling

.gitignore

@@ -5,3 +5,4 @@ __pycache__
 error_*
 *.log
 /logs/
+.vscode

src/logs/guild_logs.py

@@ -58,6 +58,15 @@ class GuildLogs:
         self.channels = {}
         self.locked = False
 
+    def __enter__(self):
+        return self
+
+    def __exit__(self, type, value, tb):
+        del self.channels
+        del self.guild
+        if self.locked:
+            self.unlock()
+
     def dict(self) -> dict:
         return {id: self.channels[id].dict() for id in self.channels}
 
@@ -77,7 +86,8 @@ class GuildLogs:
     def unlock(self):
         self.locked = False
         current_analysis_lock.acquire()
-        current_analysis.remove(self.log_file)
+        if self.log_file in current_analysis:
+            current_analysis.remove(self.log_file)
         current_analysis_lock.release()
 
     async def load(
@@ -111,6 +121,7 @@ class GuildLogs:
                 await code_message(progress, "Reading saved history (2/4)...")
                 t0 = datetime.now()
                 json_data = gzip.decompress(gziped_data)
+                del gziped_data
                 logging.info(
                     f"log {self.guild.id} > gzip decompress in {delta(t0):,}ms"
                 )
@@ -119,6 +130,7 @@ class GuildLogs:
                 await code_message(progress, "Reading saved history (3/4)...")
                 t0 = datetime.now()
                 channels = json.loads(json_data)
+                del json_data
                 logging.info(f"log {self.guild.id} > json parse in {delta(t0):,}ms")
                 if self.check_cancelled():
                     return CANCELLED, 0
@@ -267,6 +279,7 @@ class GuildLogs:
             )
             t0 = datetime.now()
             gziped_data = gzip.compress(json_data)
+            del json_data
             logging.info(
                 f"log {self.guild.id} > gzip in {delta(t0):,}ms -> {real_total_msg / deltas(t0):,.3f} m/s"
             )
@@ -279,6 +292,7 @@ class GuildLogs:
             t0 = datetime.now()
             with open(self.log_file, mode="wb") as f:
                 f.write(gziped_data)
+            del gziped_data
             logging.info(
                 f"log {self.guild.id} > saved in {delta(t0):,}ms -> {real_total_msg / deltas(t0):,.3f} m/s"
             )

src/scanners/scanner.py

@@ -36,133 +36,134 @@ class Scanner(ABC):
     ):
         args = list(args)
         guild = message.guild
-        logs = GuildLogs(guild)
+        with GuildLogs(guild) as logs:
-
+            # If "%cmd help" redirect to "%help cmd"
-        # If "%cmd help" redirect to "%help cmd"
+            if "help" in args:
-        if "help" in args:
+                await client.bot.help(client, message, "help", args[0])
-            await client.bot.help(client, message, "help", args[0])
-            return
-
-        # check args validity
-        str_channel_mentions = [str(channel.id) for channel in message.channel_mentions]
-        str_mentions = [str(member.id) for member in message.mentions]
-        for i, arg in enumerate(args[1:]):
-            if re.match(r"^<@!?\d+>$", arg):
-                arg = arg[3:-1] if "!" in arg else arg[2:-1]
-            elif re.match(r"^<#!?\d+>$", arg):
-                arg = arg[3:-1] if "!" in arg else arg[2:-1]
-            if (
-                arg not in self.valid_args + ["me", "here", "fast", "fresh"]
-                and (not arg.isdigit() or not self.has_digit_args)
-                and arg not in str_channel_mentions
-                and arg not in str_mentions
-            ):
-                await message.channel.send(
-                    f"Unrecognized argument: `{arg}`", reference=message
-                )
                 return
 
-        # Get selected channels or all of them if no channel arguments
+            # check args validity
-        self.channels = no_duplicate(message.channel_mentions)
+            str_channel_mentions = [
-
+                str(channel.id) for channel in message.channel_mentions
-        # transform the "here" arg
+            ]
-        if "here" in args:
+            str_mentions = [str(member.id) for member in message.mentions]
-            self.channels += [message.channel]
+            for i, arg in enumerate(args[1:]):
-
+                if re.match(r"^<@!?\d+>$", arg):
-        self.full = len(self.channels) == 0
+                    arg = arg[3:-1] if "!" in arg else arg[2:-1]
-        if self.full:
+                elif re.match(r"^<#!?\d+>$", arg):
-            self.channels = guild.text_channels
+                    arg = arg[3:-1] if "!" in arg else arg[2:-1]
-
+                if (
-        # Get selected members
+                    arg not in self.valid_args + ["me", "here", "fast", "fresh"]
-        self.members = no_duplicate(message.mentions)
+                    and (not arg.isdigit() or not self.has_digit_args)
-        self.raw_members = no_duplicate(message.raw_mentions)
+                    and arg not in str_channel_mentions
-
+                    and arg not in str_mentions
-        # transform the "me" arg
+                ):
-        if "me" in args:
-            self.members += [message.author]
-            self.raw_members += [message.author.id]
-
-        if not await self.init(message, *args):
-            return
-
-        # Start computing data
-        async with message.channel.typing():
-            progress = await message.channel.send(
-                "```Starting analysis...```",
-                reference=message,
-                allowed_mentions=discord.AllowedMentions.none(),
-            )
-            total_msg, total_chan = await logs.load(
-                progress, self.channels, fast="fast" in args, fresh="fresh" in args
-            )
-            if total_msg == CANCELLED:
-                await message.channel.send(
-                    "Operation cancelled by user",
-                    reference=message,
-                )
-            elif total_msg == ALREADY_RUNNING:
-                await message.channel.send(
-                    "An analysis is already running on this server, please be patient.",
-                    reference=message,
-                )
-            else:
-                self.msg_count = 0
-                self.total_msg = 0
-                self.chan_count = 0
-                t0 = datetime.now()
-                for channel in self.channels:
-                    if channel.id in logs.channels:
-                        channel_logs = logs.channels[channel.id]
-                        count = sum(
-                            [
-                                self.compute_message(channel_logs, message_log)
-                                for message_log in channel_logs.messages
-                            ]
-                        )
-                        self.total_msg += len(channel_logs.messages)
-                        self.msg_count += count
-                        self.chan_count += 1 if count > 0 else 0
-                logging.info(f"scan {guild.id} > scanned in {delta(t0):,}ms")
-                if self.total_msg == 0:
                     await message.channel.send(
-                        "There are no messages found matching the filters",
+                        f"Unrecognized argument: `{arg}`", reference=message
+                    )
+                    return
+
+            # Get selected channels or all of them if no channel arguments
+            self.channels = no_duplicate(message.channel_mentions)
+
+            # transform the "here" arg
+            if "here" in args:
+                self.channels += [message.channel]
+
+            self.full = len(self.channels) == 0
+            if self.full:
+                self.channels = guild.text_channels
+
+            # Get selected members
+            self.members = no_duplicate(message.mentions)
+            self.raw_members = no_duplicate(message.raw_mentions)
+
+            # transform the "me" arg
+            if "me" in args:
+                self.members += [message.author]
+                self.raw_members += [message.author.id]
+
+            if not await self.init(message, *args):
+                return
+
+            # Start computing data
+            async with message.channel.typing():
+                progress = await message.channel.send(
+                    "```Starting analysis...```",
+                    reference=message,
+                    allowed_mentions=discord.AllowedMentions.none(),
+                )
+                total_msg, total_chan = await logs.load(
+                    progress, self.channels, fast="fast" in args, fresh="fresh" in args
+                )
+                if total_msg == CANCELLED:
+                    await message.channel.send(
+                        "Operation cancelled by user",
+                        reference=message,
+                    )
+                elif total_msg == ALREADY_RUNNING:
+                    await message.channel.send(
+                        "An analysis is already running on this server, please be patient.",
                         reference=message,
                     )
                 else:
-                    await progress.edit(content="```Computing results...```")
+                    self.msg_count = 0
-                    # Display results
+                    self.total_msg = 0
+                    self.chan_count = 0
                     t0 = datetime.now()
-                    results = self.get_results(
+                    for channel in self.channels:
-                        get_intro(
+                        if channel.id in logs.channels:
-                            self.intro_context,
+                            channel_logs = logs.channels[channel.id]
-                            self.full,
+                            count = sum(
-                            self.channels,
+                                [
-                            self.members,
+                                    self.compute_message(channel_logs, message_log)
-                            self.msg_count,
+                                    for message_log in channel_logs.messages
-                            self.chan_count,
+                                ]
+                            )
+                            self.total_msg += len(channel_logs.messages)
+                            self.msg_count += count
+                            self.chan_count += 1 if count > 0 else 0
+                    logging.info(f"scan {guild.id} > scanned in {delta(t0):,}ms")
+                    if self.total_msg == 0:
+                        await message.channel.send(
+                            "There are no messages found matching the filters",
+                            reference=message,
                         )
-                    )
+                    else:
-                    logging.info(f"scan {guild.id} > results in {delta(t0):,}ms")
+                        await progress.edit(content="```Computing results...```")
-                    response = ""
+                        # Display results
-                    first = True
+                        t0 = datetime.now()
-                    for r in results:
+                        results = self.get_results(
-                        if len(response + "\n" + r) > 2000:
+                            get_intro(
+                                self.intro_context,
+                                self.full,
+                                self.channels,
+                                self.members,
+                                self.msg_count,
+                                self.chan_count,
+                            )
+                        )
+                        logging.info(f"scan {guild.id} > results in {delta(t0):,}ms")
+                        response = ""
+                        first = True
+                        for r in results:
+                            if len(response + "\n" + r) > 2000:
+                                await message.channel.send(
+                                    response,
+                                    reference=message if first else None,
+                                    allowed_mentions=discord.AllowedMentions.none(),
+                                )
+                                first = False
+                                response = ""
+                            response += "\n" + r
+                        if len(response) > 0:
                             await message.channel.send(
                                 response,
                                 reference=message if first else None,
                                 allowed_mentions=discord.AllowedMentions.none(),
                             )
-                            first = False
+                # Delete custom progress message
-                            response = ""
+                await progress.delete()
-                        response += "\n" + r
-                    if len(response) > 0:
-                        await message.channel.send(
-                            response,
-                            reference=message if first else None,
-                            allowed_mentions=discord.AllowedMentions.none(),
-                        )
-            # Delete custom progress message
-            await progress.delete()
 
     @abstractmethod
     async def init(self, message: discord.Message, *args: str) -> bool: