trying to optimize things (found it, it's coming)

This commit is contained in:
klemek
2021-01-06 14:36:51 +01:00
parent 2beeb1183f
commit 5b5e3f178b
+20 -15
View File
@@ -8,6 +8,10 @@ import json
# Custom libs # Custom libs
from utils import debug, aggregate, no_duplicate from utils import debug, aggregate, no_duplicate
# CONSTANTS
CHUNK_SIZE = 1000
# preload # preload
EXTRA_EMOJI = { EXTRA_EMOJI = {
@@ -89,11 +93,6 @@ def load_emojis():
load_emojis() load_emojis()
print(f"loaded {len(GLOBAL_EMOJIS)} emojis") print(f"loaded {len(GLOBAL_EMOJIS)} emojis")
# CONSTANTS
CHUNK_SIZE = 10000
# MAIN # MAIN
HELP = ( HELP = (
@@ -232,12 +231,15 @@ async def analyse_channel(
nm = 0 nm = 0
nmm = 0 nmm = 0
try: try:
messages = [None] last_message = None
while len(messages) >= CHUNK_SIZE or messages[-1] is None: done = 0
messages = await channel.history( while done >= CHUNK_SIZE or last_message is None:
limit=CHUNK_SIZE, before=messages[-1] done = 0
).flatten() async for m in channel.history(
for m in messages: limit=CHUNK_SIZE, before=last_message, oldest_first=False
):
done += 1
last_message = m
# If author is not bot or included in the selection (empty list is all) # If author is not bot or included in the selection (empty list is all)
if not m.author.bot and (len(members) == 0 or m.author in members): if not m.author.bot and (len(members) == 0 or m.author in members):
# Find all emotes un the current message in the form "<:emoji:123456789>" # Find all emotes un the current message in the form "<:emoji:123456789>"
@@ -270,10 +272,13 @@ async def analyse_channel(
if member in users: if member in users:
emotes[name].reactions += 1 emotes[name].reactions += 1
emotes[name].update_use(m.created_at) emotes[name].update_use(m.created_at)
nm += len(messages) nm += done
await progress.edit( # await progress.edit(
content=f"```{nm0 + nm:,} messages and {nc} channels analysed```" # content=f"```{nm0 + nm:,} messages and {nc} channels analysed```"
) # )
# await progress.edit(
# content=f"```{nm0 + nm:,} messages and {nc+1} channels analysed```"
# )
return nm, nmm return nm, nmm
except discord.errors.HTTPException: except discord.errors.HTTPException:
# When an exception occurs (like Forbidden) sent -1 # When an exception occurs (like Forbidden) sent -1