klemek/discord-analyst
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

v1.17

Browse Source
This commit is contained in:
Klemek
2022-03-07 13:59:42 +01:00
parent 0ceffca196
commit fb48a256ce
5 changed files with 77 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+4 -1
View File
@@ -125,11 +125,14 @@ python3 src/main.py
## Changelog
* **v1.17**
* compliency with 30 days data keeping policy and data encryption
* improvements and bug fix
* **v1.16**
* `%freq graph` graph hours frequency along the week
* uses discord new time format
* `%freq` now shows quietest day of week and hour of day
* improvments and bug fix
* improvements and bug fix
* **v1.15**
* `nsfw:allow/only` filter nsfw channels
* `%find` can use regexes
requirements.txt
+6 -6
View File
@@ -1,6 +1,6 @@
discord.py==1.7.0
python-dotenv==0.15.0
python-dateutil==2.8.1
git+git://github.com/Klemek/miniscord.git
matplotlib
cryptography
discord.py^=1.7.0
python-dotenv^=0.15.0
python-dateutil^=2.8.1
matplotlib^=3.4.2
cryptography^=2.8
git+git://github.com/Klemek/miniscord.git
src/logs/guild_logs.py
+40 -11
View File
@@ -9,6 +9,7 @@ import logging
import asyncio
import threading
from dotenv import load_dotenv
from cryptography.fernet import Fernet
from . import ChannelLogs
from utils import code_message, delta, deltas
@@ -23,7 +24,7 @@ NO_FILE = -300
load_dotenv()
LOG_DIR = os.getenv("LOG_DIR", "logs")
LOG_EXT = os.getenv("LOG_DIR", ".logz")
LOG_EXT = os.getenv("LOG_EXT", ".logz")
CRYPT_KEY = os.getenv("CRYPT_KEY", "")
# 5 minutes, assume 'fast' arg
@@ -130,29 +131,41 @@ class GuildLogs:
channels = {}
try:
last_time = os.path.getmtime(self.log_file)
gziped_data = None
await code_message(progress, "Reading saved history (1/4)...")
encrypted_data = None
await code_message(progress, "Reading saved history (1/5)...")
t0 = datetime.now()
with open(self.log_file, mode="rb") as f:
gziped_data = f.read()
encrypted_data = f.read()
logging.info(f"log {self.guild.id} > read in {delta(t0):,}ms")
if self.check_cancelled():
return CANCELLED, 0
await code_message(progress, "Reading saved history (2/4)...")
await code_message(progress, "Reading saved history (2/5)...")
if CRYPT_KEY == "" or CRYPT_KEY is None:
gziped_data = encrypted_data
try:
t0 = datetime.now()
fernet = Fernet(CRYPT_KEY)
gziped_data = fernet.decrypt(encrypted_data)
logging.info(f"log {self.guild.id} > decrypted in {delta(t0):,}ms")
except:
gziped_data = encrypted_data
if self.check_cancelled():
return CANCELLED, 0
await code_message(progress, "Reading saved history (3/5)...")
t0 = datetime.now()
json_data = gzip.decompress(gziped_data)
del gziped_data
logging.info(f"log {self.guild.id} > gzip decompress in {delta(t0):,}ms")
if self.check_cancelled():
return CANCELLED, 0
await code_message(progress, "Reading saved history (3/4)...")
await code_message(progress, "Reading saved history (4/5)...")
t0 = datetime.now()
channels = json.loads(json_data)
del json_data
logging.info(f"log {self.guild.id} > json parse in {delta(t0):,}ms")
if self.check_cancelled():
return CANCELLED, 0
await code_message(progress, "Reading saved history (4/4)...")
await code_message(progress, "Reading saved history (5/5)...")
t0 = datetime.now()
self.channels = {
int(id): ChannelLogs(channels[id], self) for id in channels
@@ -289,7 +302,7 @@ class GuildLogs:
return CANCELLED, 0
await code_message(
progress,
f"Saving history (1/3)...\n{real_total_msg:,} messages in {real_total_chan:,} channels",
f"Saving history (1/4)...\n{real_total_msg:,} messages in {real_total_chan:,} channels",
)
t0 = datetime.now()
json_data = bytes(json.dumps(self.dict()), "utf-8")
@@ -300,7 +313,7 @@ class GuildLogs:
return CANCELLED, 0
await code_message(
progress,
f"Saving history (2/3)...\n{real_total_msg:,} messages in {real_total_chan:,} channels",
f"Saving history (2/4)...\n{real_total_msg:,} messages in {real_total_chan:,} channels",
)
t0 = datetime.now()
gziped_data = gzip.compress(json_data)
@@ -312,12 +325,28 @@ class GuildLogs:
return CANCELLED, 0
await code_message(
progress,
f"Saving history (3/3)...\n{real_total_msg:,} messages in {real_total_chan:,} channels",
f"Saving history (3/4)...\n{real_total_msg:,} messages in {real_total_chan:,} channels",
)
if CRYPT_KEY == "" or CRYPT_KEY is None:
encrypted_data = gziped_data
try:
t0 = datetime.now()
fernet = Fernet(CRYPT_KEY)
encrypted_data = fernet.encrypt(gziped_data)
logging.info(f"log {self.guild.id} > encrypted in {delta(t0):,}ms -> {len(gziped_data) / deltas(t0):,.3f} b/s")
except:
encrypted_data = gziped_data
if self.check_cancelled():
return CANCELLED, 0
await code_message(
progress,
f"Saving history (4/4)...\n{real_total_msg:,} messages in {real_total_chan:,} channels",
)
t0 = datetime.now()
with open(self.log_file, mode="wb") as f:
f.write(gziped_data)
f.write(encrypted_data)
del gziped_data
del encrypted_data
logging.info(
f"log {self.guild.id} > saved in {delta(t0):,}ms -> {real_total_msg / deltas(t0):,.3f} m/s"
)
src/main.py
+1 -1
View File
@@ -18,7 +18,7 @@ emojis.load_emojis()
bot = Bot(
"Discord Analyst",
"1.16.2",
"1.17.0",
alias="%",
)
src/tools/convert_logs.py
+26
View File
@@ -0,0 +1,26 @@
import os
import os.path
from dotenv import load_dotenv
from cryptography.fernet import Fernet
load_dotenv()
LOG_DIR = os.getenv("LOG_DIR", "logs")
LOG_EXT = os.getenv("LOG_DIR", ".logz")
CRYPT_KEY = os.getenv("CRYPT_KEY", "")
fernet = Fernet(CRYPT_KEY)
for item in os.listdir(LOG_DIR):
if item.endswith(LOG_EXT):
path = os.path.join(LOG_DIR, item)
data = None
with open(path, mode="rb") as f:
data = f.read()
try:
fernet.decrypt(data)
print(f"{item} already encrypted")
except:
with open(path, mode="wb") as f:
f.write(fernet.encrypt(data))
print(f"{item} was encrypted")