diff --git a/.gitignore b/.gitignore index c9e2809..4a7fec5 100755 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ __pycache__ error_* *.log /logs/ +.vscode \ No newline at end of file diff --git a/src/logs/guild_logs.py b/src/logs/guild_logs.py index afd8de0..88d8823 100644 --- a/src/logs/guild_logs.py +++ b/src/logs/guild_logs.py @@ -58,6 +58,15 @@ class GuildLogs: self.channels = {} self.locked = False + def __enter__(self): + return self + + def __exit__(self, type, value, tb): + del self.channels + del self.guild + if self.locked: + self.unlock() + def dict(self) -> dict: return {id: self.channels[id].dict() for id in self.channels} @@ -77,7 +86,8 @@ class GuildLogs: def unlock(self): self.locked = False current_analysis_lock.acquire() - current_analysis.remove(self.log_file) + if self.log_file in current_analysis: + current_analysis.remove(self.log_file) current_analysis_lock.release() async def load( @@ -111,6 +121,7 @@ class GuildLogs: await code_message(progress, "Reading saved history (2/4)...") t0 = datetime.now() json_data = gzip.decompress(gziped_data) + del gziped_data logging.info( f"log {self.guild.id} > gzip decompress in {delta(t0):,}ms" ) @@ -119,6 +130,7 @@ class GuildLogs: await code_message(progress, "Reading saved history (3/4)...") t0 = datetime.now() channels = json.loads(json_data) + del json_data logging.info(f"log {self.guild.id} > json parse in {delta(t0):,}ms") if self.check_cancelled(): return CANCELLED, 0 @@ -267,6 +279,7 @@ class GuildLogs: ) t0 = datetime.now() gziped_data = gzip.compress(json_data) + del json_data logging.info( f"log {self.guild.id} > gzip in {delta(t0):,}ms -> {real_total_msg / deltas(t0):,.3f} m/s" ) @@ -279,6 +292,7 @@ class GuildLogs: t0 = datetime.now() with open(self.log_file, mode="wb") as f: f.write(gziped_data) + del gziped_data logging.info( f"log {self.guild.id} > saved in {delta(t0):,}ms -> {real_total_msg / deltas(t0):,.3f} m/s" ) diff --git a/src/scanners/scanner.py b/src/scanners/scanner.py index 4a3b749..9a7e712 100644 --- a/src/scanners/scanner.py +++ b/src/scanners/scanner.py @@ -36,133 +36,134 @@ class Scanner(ABC): ): args = list(args) guild = message.guild - logs = GuildLogs(guild) - - # If "%cmd help" redirect to "%help cmd" - if "help" in args: - await client.bot.help(client, message, "help", args[0]) - return - - # check args validity - str_channel_mentions = [str(channel.id) for channel in message.channel_mentions] - str_mentions = [str(member.id) for member in message.mentions] - for i, arg in enumerate(args[1:]): - if re.match(r"^<@!?\d+>$", arg): - arg = arg[3:-1] if "!" in arg else arg[2:-1] - elif re.match(r"^<#!?\d+>$", arg): - arg = arg[3:-1] if "!" in arg else arg[2:-1] - if ( - arg not in self.valid_args + ["me", "here", "fast", "fresh"] - and (not arg.isdigit() or not self.has_digit_args) - and arg not in str_channel_mentions - and arg not in str_mentions - ): - await message.channel.send( - f"Unrecognized argument: `{arg}`", reference=message - ) + with GuildLogs(guild) as logs: + # If "%cmd help" redirect to "%help cmd" + if "help" in args: + await client.bot.help(client, message, "help", args[0]) return - # Get selected channels or all of them if no channel arguments - self.channels = no_duplicate(message.channel_mentions) - - # transform the "here" arg - if "here" in args: - self.channels += [message.channel] - - self.full = len(self.channels) == 0 - if self.full: - self.channels = guild.text_channels - - # Get selected members - self.members = no_duplicate(message.mentions) - self.raw_members = no_duplicate(message.raw_mentions) - - # transform the "me" arg - if "me" in args: - self.members += [message.author] - self.raw_members += [message.author.id] - - if not await self.init(message, *args): - return - - # Start computing data - async with message.channel.typing(): - progress = await message.channel.send( - "```Starting analysis...```", - reference=message, - allowed_mentions=discord.AllowedMentions.none(), - ) - total_msg, total_chan = await logs.load( - progress, self.channels, fast="fast" in args, fresh="fresh" in args - ) - if total_msg == CANCELLED: - await message.channel.send( - "Operation cancelled by user", - reference=message, - ) - elif total_msg == ALREADY_RUNNING: - await message.channel.send( - "An analysis is already running on this server, please be patient.", - reference=message, - ) - else: - self.msg_count = 0 - self.total_msg = 0 - self.chan_count = 0 - t0 = datetime.now() - for channel in self.channels: - if channel.id in logs.channels: - channel_logs = logs.channels[channel.id] - count = sum( - [ - self.compute_message(channel_logs, message_log) - for message_log in channel_logs.messages - ] - ) - self.total_msg += len(channel_logs.messages) - self.msg_count += count - self.chan_count += 1 if count > 0 else 0 - logging.info(f"scan {guild.id} > scanned in {delta(t0):,}ms") - if self.total_msg == 0: + # check args validity + str_channel_mentions = [ + str(channel.id) for channel in message.channel_mentions + ] + str_mentions = [str(member.id) for member in message.mentions] + for i, arg in enumerate(args[1:]): + if re.match(r"^<@!?\d+>$", arg): + arg = arg[3:-1] if "!" in arg else arg[2:-1] + elif re.match(r"^<#!?\d+>$", arg): + arg = arg[3:-1] if "!" in arg else arg[2:-1] + if ( + arg not in self.valid_args + ["me", "here", "fast", "fresh"] + and (not arg.isdigit() or not self.has_digit_args) + and arg not in str_channel_mentions + and arg not in str_mentions + ): await message.channel.send( - "There are no messages found matching the filters", + f"Unrecognized argument: `{arg}`", reference=message + ) + return + + # Get selected channels or all of them if no channel arguments + self.channels = no_duplicate(message.channel_mentions) + + # transform the "here" arg + if "here" in args: + self.channels += [message.channel] + + self.full = len(self.channels) == 0 + if self.full: + self.channels = guild.text_channels + + # Get selected members + self.members = no_duplicate(message.mentions) + self.raw_members = no_duplicate(message.raw_mentions) + + # transform the "me" arg + if "me" in args: + self.members += [message.author] + self.raw_members += [message.author.id] + + if not await self.init(message, *args): + return + + # Start computing data + async with message.channel.typing(): + progress = await message.channel.send( + "```Starting analysis...```", + reference=message, + allowed_mentions=discord.AllowedMentions.none(), + ) + total_msg, total_chan = await logs.load( + progress, self.channels, fast="fast" in args, fresh="fresh" in args + ) + if total_msg == CANCELLED: + await message.channel.send( + "Operation cancelled by user", + reference=message, + ) + elif total_msg == ALREADY_RUNNING: + await message.channel.send( + "An analysis is already running on this server, please be patient.", reference=message, ) else: - await progress.edit(content="```Computing results...```") - # Display results + self.msg_count = 0 + self.total_msg = 0 + self.chan_count = 0 t0 = datetime.now() - results = self.get_results( - get_intro( - self.intro_context, - self.full, - self.channels, - self.members, - self.msg_count, - self.chan_count, + for channel in self.channels: + if channel.id in logs.channels: + channel_logs = logs.channels[channel.id] + count = sum( + [ + self.compute_message(channel_logs, message_log) + for message_log in channel_logs.messages + ] + ) + self.total_msg += len(channel_logs.messages) + self.msg_count += count + self.chan_count += 1 if count > 0 else 0 + logging.info(f"scan {guild.id} > scanned in {delta(t0):,}ms") + if self.total_msg == 0: + await message.channel.send( + "There are no messages found matching the filters", + reference=message, ) - ) - logging.info(f"scan {guild.id} > results in {delta(t0):,}ms") - response = "" - first = True - for r in results: - if len(response + "\n" + r) > 2000: + else: + await progress.edit(content="```Computing results...```") + # Display results + t0 = datetime.now() + results = self.get_results( + get_intro( + self.intro_context, + self.full, + self.channels, + self.members, + self.msg_count, + self.chan_count, + ) + ) + logging.info(f"scan {guild.id} > results in {delta(t0):,}ms") + response = "" + first = True + for r in results: + if len(response + "\n" + r) > 2000: + await message.channel.send( + response, + reference=message if first else None, + allowed_mentions=discord.AllowedMentions.none(), + ) + first = False + response = "" + response += "\n" + r + if len(response) > 0: await message.channel.send( response, reference=message if first else None, allowed_mentions=discord.AllowedMentions.none(), ) - first = False - response = "" - response += "\n" + r - if len(response) > 0: - await message.channel.send( - response, - reference=message if first else None, - allowed_mentions=discord.AllowedMentions.none(), - ) - # Delete custom progress message - await progress.delete() + # Delete custom progress message + await progress.delete() @abstractmethod async def init(self, message: discord.Message, *args: str) -> bool: