From b671ee527e7f3ac16e20d46d4b00ffdae14b0642 Mon Sep 17 00:00:00 2001 From: klemek Date: Mon, 11 Jan 2021 15:04:02 +0100 Subject: [PATCH] improved loading logs --- src/logs/guild_logs.py | 58 ++++++++++++++++++++++++++++-------------- src/utils/utils.py | 11 ++++++++ 2 files changed, 50 insertions(+), 19 deletions(-) diff --git a/src/logs/guild_logs.py b/src/logs/guild_logs.py index b11c811..2ef4974 100644 --- a/src/logs/guild_logs.py +++ b/src/logs/guild_logs.py @@ -8,7 +8,7 @@ import logging from . import ChannelLogs -from utils import code_message +from utils import code_message, delta, deltas LOG_DIR = "logs" @@ -32,8 +32,8 @@ class GuildLogs: if self.log_file in current_analysis: return -1, -1 current_analysis += [self.log_file] + t00 = datetime.now() # read logs - t0 = datetime.now() if not os.path.exists(LOG_DIR): os.mkdir(LOG_DIR) if os.path.exists(self.log_file): @@ -41,13 +41,22 @@ class GuildLogs: try: gziped_data = None await code_message(progress, "Reading saved history (1/4)...") + t0 = datetime.now() with open(self.log_file, mode="rb") as f: gziped_data = f.read() + logging.info(f"log {self.guild.id} > read in {delta(t0):,}ms") await code_message(progress, "Reading saved history (2/4)...") + t0 = datetime.now() json_data = gzip.decompress(gziped_data) + logging.info( + f"log {self.guild.id} > gzip decompress in {delta(t0):,}ms" + ) await code_message(progress, "Reading saved history (3/4)...") + t0 = datetime.now() channels = json.loads(json_data) + logging.info(f"log {self.guild.id} > json parse in {delta(t0):,}ms") await code_message(progress, "Reading saved history (4/4)...") + t0 = datetime.now() self.channels = {int(id): ChannelLogs(channels[id]) for id in channels} # remove invalid format self.channels = { @@ -55,8 +64,7 @@ class GuildLogs: for id in self.channels if self.channels[id].is_format() } - dt = (datetime.now() - t0).total_seconds() - logging.info(f"log {self.guild.id} > loaded in {dt} s") + logging.info(f"log {self.guild.id} > loaded in {delta(t0):,}ms") except json.decoder.JSONDecodeError: logging.error(f"log {self.guild.id} > invalid JSON") except IOError: @@ -72,7 +80,7 @@ class GuildLogs: max_chan = len(target_channels) await code_message( progress, - f"Reading new history...\n0 messages in 0/{max_chan} channels\n(this might take a while)", + f"Reading new history...\n0 messages in 0/{max_chan:,} channels\n(this might take a while)", ) for channel in target_channels: if channel.id not in self.channels: @@ -92,41 +100,53 @@ class GuildLogs: warning_msg = ( "(some channels are new, this might take a long while)" ) - dt = (datetime.now() - t0).total_seconds() await code_message( progress, - f"Reading new history...\n{tmp_msg:,} messages in {total_chan + 1}/{max_chan} channels ({round(tmp_queried_msg/dt)}m/s)\n{warning_msg}", + f"Reading new history...\n{tmp_msg:,} messages in {total_chan + 1:,}/{max_chan:,} channels ({round(tmp_queried_msg/deltas(t0)):,}m/s)\n{warning_msg}", ) if done: total_chan += 1 total_msg += len(self.channels[channel.id].messages) queried_msg += count - start_msg - dt = (datetime.now() - t0).total_seconds() logging.info( - f"log {self.guild.id} > queried in {dt} s -> {queried_msg / dt} m/s" + f"log {self.guild.id} > queried in {delta(t0):,}ms -> {queried_msg / deltas(t0):,.3f} m/s" ) # write logs + real_total_msg = sum( + [len(channel.messages) for channel in self.channels.values()] + ) + real_total_chan = len(self.channels) + await code_message( + progress, + f"Saving history (1/3)...\n{real_total_msg:,} messages in {real_total_chan:,} channels", + ) t0 = datetime.now() - await code_message( - progress, - f"Saving history (1/3)...\n{total_msg:,} messages in {total_chan} channels", - ) json_data = bytes(json.dumps(self.dict()), "utf-8") + logging.info( + f"log {self.guild.id} > json dump in {delta(t0):,}ms -> {real_total_msg / deltas(t0):,.3f} m/s" + ) await code_message( progress, - f"Saving history (2/3)...\n{total_msg:,} messages in {total_chan} channels", + f"Saving history (2/3)...\n{real_total_msg:,} messages in {real_total_chan:,} channels", ) + t0 = datetime.now() gziped_data = gzip.compress(json_data) + logging.info( + f"log {self.guild.id} > gzip in {delta(t0):,}ms -> {real_total_msg / deltas(t0):,.3f} m/s" + ) await code_message( progress, - f"Saving history (3/3)...\n{total_msg:,} messages in {total_chan} channels", + f"Saving history (3/3)...\n{real_total_msg:,} messages in {real_total_chan:,} channels", ) + t0 = datetime.now() with open(self.log_file, mode="wb") as f: f.write(gziped_data) - dt = (datetime.now() - t0).total_seconds() - logging.info(f"log {self.guild.id} > written in {dt} s") - await code_message( - progress, f"Analysing...\n{total_msg:,} messages in {total_chan} channels" + logging.info( + f"log {self.guild.id} > saved in {delta(t0):,}ms -> {real_total_msg / deltas(t0):,.3f} m/s" ) + await code_message( + progress, f"Analysing...\n{total_msg:,} messages in {total_chan:,} channels" + ) + logging.info(f"log {self.guild.id} > TOTAL TIME: {delta(t00):,}ms") current_analysis.remove(self.log_file) return total_msg, total_chan diff --git a/src/utils/utils.py b/src/utils/utils.py index c46a020..11d3f3b 100644 --- a/src/utils/utils.py +++ b/src/utils/utils.py @@ -4,6 +4,17 @@ import logging import discord from datetime import datetime +# OTHER + + +def delta(t0: datetime): + return round((datetime.now() - t0).total_seconds() * 1000) + + +def deltas(t0: datetime): + return (datetime.now() - t0).total_seconds() + + # DISCORD API