better memory handling
This commit is contained in:
Klemek
@@ -5,3 +5,4 @@ __pycache__
|
||||
error_*
|
||||
*.log
|
||||
/logs/
|
||||
.vscode
|
||||
+15
-1
@@ -58,6 +58,15 @@ class GuildLogs:
|
||||
self.channels = {}
|
||||
self.locked = False
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, type, value, tb):
|
||||
del self.channels
|
||||
del self.guild
|
||||
if self.locked:
|
||||
self.unlock()
|
||||
|
||||
def dict(self) -> dict:
|
||||
return {id: self.channels[id].dict() for id in self.channels}
|
||||
|
||||
@@ -77,7 +86,8 @@ class GuildLogs:
|
||||
def unlock(self):
|
||||
self.locked = False
|
||||
current_analysis_lock.acquire()
|
||||
current_analysis.remove(self.log_file)
|
||||
if self.log_file in current_analysis:
|
||||
current_analysis.remove(self.log_file)
|
||||
current_analysis_lock.release()
|
||||
|
||||
async def load(
|
||||
@@ -111,6 +121,7 @@ class GuildLogs:
|
||||
await code_message(progress, "Reading saved history (2/4)...")
|
||||
t0 = datetime.now()
|
||||
json_data = gzip.decompress(gziped_data)
|
||||
del gziped_data
|
||||
logging.info(
|
||||
f"log {self.guild.id} > gzip decompress in {delta(t0):,}ms"
|
||||
)
|
||||
@@ -119,6 +130,7 @@ class GuildLogs:
|
||||
await code_message(progress, "Reading saved history (3/4)...")
|
||||
t0 = datetime.now()
|
||||
channels = json.loads(json_data)
|
||||
del json_data
|
||||
logging.info(f"log {self.guild.id} > json parse in {delta(t0):,}ms")
|
||||
if self.check_cancelled():
|
||||
return CANCELLED, 0
|
||||
@@ -267,6 +279,7 @@ class GuildLogs:
|
||||
)
|
||||
t0 = datetime.now()
|
||||
gziped_data = gzip.compress(json_data)
|
||||
del json_data
|
||||
logging.info(
|
||||
f"log {self.guild.id} > gzip in {delta(t0):,}ms -> {real_total_msg / deltas(t0):,.3f} m/s"
|
||||
)
|
||||
@@ -279,6 +292,7 @@ class GuildLogs:
|
||||
t0 = datetime.now()
|
||||
with open(self.log_file, mode="wb") as f:
|
||||
f.write(gziped_data)
|
||||
del gziped_data
|
||||
logging.info(
|
||||
f"log {self.guild.id} > saved in {delta(t0):,}ms -> {real_total_msg / deltas(t0):,.3f} m/s"
|
||||
)
|
||||
|
||||
+115
-114
@@ -36,133 +36,134 @@ class Scanner(ABC):
|
||||
):
|
||||
args = list(args)
|
||||
guild = message.guild
|
||||
logs = GuildLogs(guild)
|
||||
|
||||
# If "%cmd help" redirect to "%help cmd"
|
||||
if "help" in args:
|
||||
await client.bot.help(client, message, "help", args[0])
|
||||
return
|
||||
|
||||
# check args validity
|
||||
str_channel_mentions = [str(channel.id) for channel in message.channel_mentions]
|
||||
str_mentions = [str(member.id) for member in message.mentions]
|
||||
for i, arg in enumerate(args[1:]):
|
||||
if re.match(r"^<@!?\d+>$", arg):
|
||||
arg = arg[3:-1] if "!" in arg else arg[2:-1]
|
||||
elif re.match(r"^<#!?\d+>$", arg):
|
||||
arg = arg[3:-1] if "!" in arg else arg[2:-1]
|
||||
if (
|
||||
arg not in self.valid_args + ["me", "here", "fast", "fresh"]
|
||||
and (not arg.isdigit() or not self.has_digit_args)
|
||||
and arg not in str_channel_mentions
|
||||
and arg not in str_mentions
|
||||
):
|
||||
await message.channel.send(
|
||||
f"Unrecognized argument: `{arg}`", reference=message
|
||||
)
|
||||
with GuildLogs(guild) as logs:
|
||||
# If "%cmd help" redirect to "%help cmd"
|
||||
if "help" in args:
|
||||
await client.bot.help(client, message, "help", args[0])
|
||||
return
|
||||
|
||||
# Get selected channels or all of them if no channel arguments
|
||||
self.channels = no_duplicate(message.channel_mentions)
|
||||
|
||||
# transform the "here" arg
|
||||
if "here" in args:
|
||||
self.channels += [message.channel]
|
||||
|
||||
self.full = len(self.channels) == 0
|
||||
if self.full:
|
||||
self.channels = guild.text_channels
|
||||
|
||||
# Get selected members
|
||||
self.members = no_duplicate(message.mentions)
|
||||
self.raw_members = no_duplicate(message.raw_mentions)
|
||||
|
||||
# transform the "me" arg
|
||||
if "me" in args:
|
||||
self.members += [message.author]
|
||||
self.raw_members += [message.author.id]
|
||||
|
||||
if not await self.init(message, *args):
|
||||
return
|
||||
|
||||
# Start computing data
|
||||
async with message.channel.typing():
|
||||
progress = await message.channel.send(
|
||||
"```Starting analysis...```",
|
||||
reference=message,
|
||||
allowed_mentions=discord.AllowedMentions.none(),
|
||||
)
|
||||
total_msg, total_chan = await logs.load(
|
||||
progress, self.channels, fast="fast" in args, fresh="fresh" in args
|
||||
)
|
||||
if total_msg == CANCELLED:
|
||||
await message.channel.send(
|
||||
"Operation cancelled by user",
|
||||
reference=message,
|
||||
)
|
||||
elif total_msg == ALREADY_RUNNING:
|
||||
await message.channel.send(
|
||||
"An analysis is already running on this server, please be patient.",
|
||||
reference=message,
|
||||
)
|
||||
else:
|
||||
self.msg_count = 0
|
||||
self.total_msg = 0
|
||||
self.chan_count = 0
|
||||
t0 = datetime.now()
|
||||
for channel in self.channels:
|
||||
if channel.id in logs.channels:
|
||||
channel_logs = logs.channels[channel.id]
|
||||
count = sum(
|
||||
[
|
||||
self.compute_message(channel_logs, message_log)
|
||||
for message_log in channel_logs.messages
|
||||
]
|
||||
)
|
||||
self.total_msg += len(channel_logs.messages)
|
||||
self.msg_count += count
|
||||
self.chan_count += 1 if count > 0 else 0
|
||||
logging.info(f"scan {guild.id} > scanned in {delta(t0):,}ms")
|
||||
if self.total_msg == 0:
|
||||
# check args validity
|
||||
str_channel_mentions = [
|
||||
str(channel.id) for channel in message.channel_mentions
|
||||
]
|
||||
str_mentions = [str(member.id) for member in message.mentions]
|
||||
for i, arg in enumerate(args[1:]):
|
||||
if re.match(r"^<@!?\d+>$", arg):
|
||||
arg = arg[3:-1] if "!" in arg else arg[2:-1]
|
||||
elif re.match(r"^<#!?\d+>$", arg):
|
||||
arg = arg[3:-1] if "!" in arg else arg[2:-1]
|
||||
if (
|
||||
arg not in self.valid_args + ["me", "here", "fast", "fresh"]
|
||||
and (not arg.isdigit() or not self.has_digit_args)
|
||||
and arg not in str_channel_mentions
|
||||
and arg not in str_mentions
|
||||
):
|
||||
await message.channel.send(
|
||||
"There are no messages found matching the filters",
|
||||
f"Unrecognized argument: `{arg}`", reference=message
|
||||
)
|
||||
return
|
||||
|
||||
# Get selected channels or all of them if no channel arguments
|
||||
self.channels = no_duplicate(message.channel_mentions)
|
||||
|
||||
# transform the "here" arg
|
||||
if "here" in args:
|
||||
self.channels += [message.channel]
|
||||
|
||||
self.full = len(self.channels) == 0
|
||||
if self.full:
|
||||
self.channels = guild.text_channels
|
||||
|
||||
# Get selected members
|
||||
self.members = no_duplicate(message.mentions)
|
||||
self.raw_members = no_duplicate(message.raw_mentions)
|
||||
|
||||
# transform the "me" arg
|
||||
if "me" in args:
|
||||
self.members += [message.author]
|
||||
self.raw_members += [message.author.id]
|
||||
|
||||
if not await self.init(message, *args):
|
||||
return
|
||||
|
||||
# Start computing data
|
||||
async with message.channel.typing():
|
||||
progress = await message.channel.send(
|
||||
"```Starting analysis...```",
|
||||
reference=message,
|
||||
allowed_mentions=discord.AllowedMentions.none(),
|
||||
)
|
||||
total_msg, total_chan = await logs.load(
|
||||
progress, self.channels, fast="fast" in args, fresh="fresh" in args
|
||||
)
|
||||
if total_msg == CANCELLED:
|
||||
await message.channel.send(
|
||||
"Operation cancelled by user",
|
||||
reference=message,
|
||||
)
|
||||
elif total_msg == ALREADY_RUNNING:
|
||||
await message.channel.send(
|
||||
"An analysis is already running on this server, please be patient.",
|
||||
reference=message,
|
||||
)
|
||||
else:
|
||||
await progress.edit(content="```Computing results...```")
|
||||
# Display results
|
||||
self.msg_count = 0
|
||||
self.total_msg = 0
|
||||
self.chan_count = 0
|
||||
t0 = datetime.now()
|
||||
results = self.get_results(
|
||||
get_intro(
|
||||
self.intro_context,
|
||||
self.full,
|
||||
self.channels,
|
||||
self.members,
|
||||
self.msg_count,
|
||||
self.chan_count,
|
||||
for channel in self.channels:
|
||||
if channel.id in logs.channels:
|
||||
channel_logs = logs.channels[channel.id]
|
||||
count = sum(
|
||||
[
|
||||
self.compute_message(channel_logs, message_log)
|
||||
for message_log in channel_logs.messages
|
||||
]
|
||||
)
|
||||
self.total_msg += len(channel_logs.messages)
|
||||
self.msg_count += count
|
||||
self.chan_count += 1 if count > 0 else 0
|
||||
logging.info(f"scan {guild.id} > scanned in {delta(t0):,}ms")
|
||||
if self.total_msg == 0:
|
||||
await message.channel.send(
|
||||
"There are no messages found matching the filters",
|
||||
reference=message,
|
||||
)
|
||||
)
|
||||
logging.info(f"scan {guild.id} > results in {delta(t0):,}ms")
|
||||
response = ""
|
||||
first = True
|
||||
for r in results:
|
||||
if len(response + "\n" + r) > 2000:
|
||||
else:
|
||||
await progress.edit(content="```Computing results...```")
|
||||
# Display results
|
||||
t0 = datetime.now()
|
||||
results = self.get_results(
|
||||
get_intro(
|
||||
self.intro_context,
|
||||
self.full,
|
||||
self.channels,
|
||||
self.members,
|
||||
self.msg_count,
|
||||
self.chan_count,
|
||||
)
|
||||
)
|
||||
logging.info(f"scan {guild.id} > results in {delta(t0):,}ms")
|
||||
response = ""
|
||||
first = True
|
||||
for r in results:
|
||||
if len(response + "\n" + r) > 2000:
|
||||
await message.channel.send(
|
||||
response,
|
||||
reference=message if first else None,
|
||||
allowed_mentions=discord.AllowedMentions.none(),
|
||||
)
|
||||
first = False
|
||||
response = ""
|
||||
response += "\n" + r
|
||||
if len(response) > 0:
|
||||
await message.channel.send(
|
||||
response,
|
||||
reference=message if first else None,
|
||||
allowed_mentions=discord.AllowedMentions.none(),
|
||||
)
|
||||
first = False
|
||||
response = ""
|
||||
response += "\n" + r
|
||||
if len(response) > 0:
|
||||
await message.channel.send(
|
||||
response,
|
||||
reference=message if first else None,
|
||||
allowed_mentions=discord.AllowedMentions.none(),
|
||||
)
|
||||
# Delete custom progress message
|
||||
await progress.delete()
|
||||
# Delete custom progress message
|
||||
await progress.delete()
|
||||
|
||||
@abstractmethod
|
||||
async def init(self, message: discord.Message, *args: str) -> bool:
|
||||
|
||||
Reference in New Issue
Block a user