klemek/discord-analyst
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fast argument to only load cache

Browse Source
This commit is contained in:
klemek
2021-01-11 16:43:50 +01:00
parent ae9cc69ed2
commit a1d7354280
2 changed files with 99 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/logs/guild_logs.py
+95 -73
View File
@@ -26,7 +26,11 @@ class GuildLogs:
return {id: self.channels[id].dict() for id in self.channels} return {id: self.channels[id].dict() for id in self.channels}
async def load( async def load(
self, progress: discord.Message, target_channels: List[discord.TextChannel] = [] self,
progress: discord.Message,
target_channels: List[discord.TextChannel] = [],
*,
fast: bool,
) -> Tuple[int, int]: ) -> Tuple[int, int]:
global current_analysis global current_analysis
if self.log_file in current_analysis: if self.log_file in current_analysis:
@@ -69,83 +73,101 @@ class GuildLogs:
logging.error(f"log {self.guild.id} > invalid JSON") logging.error(f"log {self.guild.id} > invalid JSON")
except IOError: except IOError:
logging.error(f"log {self.guild.id} > cannot read") logging.error(f"log {self.guild.id} > cannot read")
# load channels
t0 = datetime.now()
if len(target_channels) == 0:
target_channels = self.guild.text_channels
loading_new = 0
total_msg = 0 total_msg = 0
queried_msg = 0
total_chan = 0 total_chan = 0
max_chan = len(target_channels) if fast:
await code_message( if len(target_channels) == 0:
progress, total_msg = sum(
f"Reading new history...\n0 messages in 0/{max_chan:,} channels\n(this might take a while)", [len(channel.messages) for channel in self.channels.values()]
) )
for channel in target_channels: total_chan = len(self.channels)
if channel.id not in self.channels: else:
loading_new += 1 target_channels_id = [channel.id for channel in target_channels]
self.channels[channel.id] = ChannelLogs(channel) total_msg = sum(
start_msg = len(self.channels[channel.id].messages) [
async for count, done in self.channels[channel.id].load(channel): len(channel.messages)
if count > 0: for channel in self.channels.values()
tmp_queried_msg = queried_msg + count - start_msg if channel.id in target_channels_id
tmp_msg = total_msg + count ]
warning_msg = "(this might take a while)" )
if len(target_channels) > 5 and loading_new > 5: total_chan = len(target_channels)
warning_msg = ( else:
"(most channels are new, this might take a looong while)" # load channels
t0 = datetime.now()
if len(target_channels) == 0:
target_channels = self.guild.text_channels
loading_new = 0
queried_msg = 0
total_chan = 0
max_chan = len(target_channels)
await code_message(
progress,
f"Reading new history...\n0 messages in 0/{max_chan:,} channels\n(this might take a while)",
)
for channel in target_channels:
if channel.id not in self.channels:
loading_new += 1
self.channels[channel.id] = ChannelLogs(channel)
start_msg = len(self.channels[channel.id].messages)
async for count, done in self.channels[channel.id].load(channel):
if count > 0:
tmp_queried_msg = queried_msg + count - start_msg
tmp_msg = total_msg + count
warning_msg = "(this might take a while)"
if len(target_channels) > 5 and loading_new > 5:
warning_msg = "(most channels are new, this might take a looong while)"
elif loading_new > 0:
warning_msg = (
"(some channels are new, this might take a long while)"
)
await code_message(
progress,
f"Reading new history...\n{tmp_msg:,} messages in {total_chan + 1:,}/{max_chan:,} channels ({round(tmp_queried_msg/deltas(t0)):,}m/s)\n{warning_msg}",
) )
elif loading_new > 0: if done:
warning_msg = ( total_chan += 1
"(some channels are new, this might take a long while)" total_msg += len(self.channels[channel.id].messages)
) queried_msg += count - start_msg
await code_message( logging.info(
progress, f"log {self.guild.id} > queried in {delta(t0):,}ms -> {queried_msg / deltas(t0):,.3f} m/s"
f"Reading new history...\n{tmp_msg:,} messages in {total_chan + 1:,}/{max_chan:,} channels ({round(tmp_queried_msg/deltas(t0)):,}m/s)\n{warning_msg}", )
) # write logs
if done: real_total_msg = sum(
total_chan += 1 [len(channel.messages) for channel in self.channels.values()]
total_msg += len(self.channels[channel.id].messages) )
queried_msg += count - start_msg real_total_chan = len(self.channels)
logging.info( await code_message(
f"log {self.guild.id} > queried in {delta(t0):,}ms -> {queried_msg / deltas(t0):,.3f} m/s" progress,
) f"Saving history (1/3)...\n{real_total_msg:,} messages in {real_total_chan:,} channels",
# write logs )
real_total_msg = sum( t0 = datetime.now()
[len(channel.messages) for channel in self.channels.values()] json_data = bytes(json.dumps(self.dict()), "utf-8")
) logging.info(
real_total_chan = len(self.channels) f"log {self.guild.id} > json dump in {delta(t0):,}ms -> {real_total_msg / deltas(t0):,.3f} m/s"
)
await code_message(
progress,
f"Saving history (2/3)...\n{real_total_msg:,} messages in {real_total_chan:,} channels",
)
t0 = datetime.now()
gziped_data = gzip.compress(json_data)
logging.info(
f"log {self.guild.id} > gzip in {delta(t0):,}ms -> {real_total_msg / deltas(t0):,.3f} m/s"
)
await code_message(
progress,
f"Saving history (3/3)...\n{real_total_msg:,} messages in {real_total_chan:,} channels",
)
t0 = datetime.now()
with open(self.log_file, mode="wb") as f:
f.write(gziped_data)
logging.info(
f"log {self.guild.id} > saved in {delta(t0):,}ms -> {real_total_msg / deltas(t0):,.3f} m/s"
)
await code_message( await code_message(
progress, progress,
f"Saving history (1/3)...\n{real_total_msg:,} messages in {real_total_chan:,} channels", f"Analysing...\n{total_msg:,} messages in {total_chan:,} channels",
)
t0 = datetime.now()
json_data = bytes(json.dumps(self.dict()), "utf-8")
logging.info(
f"log {self.guild.id} > json dump in {delta(t0):,}ms -> {real_total_msg / deltas(t0):,.3f} m/s"
)
await code_message(
progress,
f"Saving history (2/3)...\n{real_total_msg:,} messages in {real_total_chan:,} channels",
)
t0 = datetime.now()
gziped_data = gzip.compress(json_data)
logging.info(
f"log {self.guild.id} > gzip in {delta(t0):,}ms -> {real_total_msg / deltas(t0):,.3f} m/s"
)
await code_message(
progress,
f"Saving history (3/3)...\n{real_total_msg:,} messages in {real_total_chan:,} channels",
)
t0 = datetime.now()
with open(self.log_file, mode="wb") as f:
f.write(gziped_data)
logging.info(
f"log {self.guild.id} > saved in {delta(t0):,}ms -> {real_total_msg / deltas(t0):,.3f} m/s"
)
await code_message(
progress, f"Analysing...\n{total_msg:,} messages in {total_chan:,} channels"
) )
logging.info(f"log {self.guild.id} > TOTAL TIME: {delta(t00):,}ms") logging.info(f"log {self.guild.id} > TOTAL TIME: {delta(t00):,}ms")
current_analysis.remove(self.log_file) current_analysis.remove(self.log_file)
src/scanners/scanner.py
+4 -2
View File
@@ -44,7 +44,7 @@ class Scanner(ABC):
str_mentions = [member.mention for member in message.mentions] str_mentions = [member.mention for member in message.mentions]
for arg in args[1:]: for arg in args[1:]:
if ( if (
arg not in self.valid_args + ["me", "here"] arg not in self.valid_args + ["me", "here", "fast"]
and (not arg.isdigit() or not self.has_digit_args) and (not arg.isdigit() or not self.has_digit_args)
and arg not in str_channel_mentions and arg not in str_channel_mentions
and arg not in str_mentions and arg not in str_mentions
@@ -80,7 +80,9 @@ class Scanner(ABC):
# Start computing data # Start computing data
async with message.channel.typing(): async with message.channel.typing():
progress = await message.channel.send("```Starting analysis...```") progress = await message.channel.send("```Starting analysis...```")
total_msg, total_chan = await logs.load(progress, self.channels) total_msg, total_chan = await logs.load(
progress, self.channels, fast="fast" in args
)
if total_msg == -1: if total_msg == -1:
await message.channel.send( await message.channel.send(
f"{message.author.mention} An analysis is already running on this server, please be patient." f"{message.author.mention} An analysis is already running on this server, please be patient."