klemek/discord-analyst
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #55 from Klemek/dev

Browse Source 
v1.16
This commit is contained in:
Klemek
2021-07-13 18:14:52 +02:00
committed by GitHub
parent dbd859a828 20e4c05cc5
commit 444c65f343
17 changed files with 313 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+5
View File
@@ -125,6 +125,11 @@ python3 src/main.py
## Changelog
* **v1.16**
* `%freq graph` graph hours frequency along the week
* uses discord new time format
* `%freq` now shows quietest day of week and hour of day
* improvments and bug fix
* **v1.15**
* `nsfw:allow/only` filter nsfw channels
* `%find` can use regexes
requirements.txt
+2
View File
@@ -2,3 +2,5 @@ discord.py==1.7.0
python-dotenv==0.15.0
python-dateutil==2.8.1
git+git://github.com/Klemek/miniscord.git
numpy
matplotlib
src/data_types/frequency.py
+72 -15
View File
@@ -1,10 +1,13 @@
from typing import List
from datetime import timedelta
import calendar
import matplotlib.pyplot as plt
import numpy as np
from io import BytesIO
import discord
import time
from utils import (
str_date,
str_datetime,
from_now,
plural,
percent,
@@ -19,8 +22,7 @@ class Frequency:
self.dates = []
self.longest_break = timedelta(seconds=0)
self.longest_break_start = None
self.week = {i: 0 for i in range(7)}
self.day = {i: 0 for i in range(24)}
self.hours = {i: {j: 0 for j in range(24)} for i in range(7)}
self.busiest_day = None
self.busiest_day_count = 0
self.busiest_hour = None
@@ -33,6 +35,49 @@ class Frequency:
self.longest_streak_start = None
self.longest_streak_author = None
def to_graph(self) -> List[str]:
self.dates.sort()
delta = self.dates[-1] - self.dates[0]
if delta.days == 0:
delta = timedelta(days=1)
day = {j: sum(self.hours[i][j] for i in range(7)) for j in range(24)}
busiest_hour = top_key(day)
n_hours = delta.days
if self.dates[0].hour <= busiest_hour and self.dates[-1].hour >= busiest_hour:
n_hours += 1
plt.style.use("dark_background")
fig, ax = plt.subplots()
fig.patch.set_facecolor("#36393F")
ax.patch.set_alpha(0)
times = range(25)
for i in range(7):
hours = [self.hours[i][hour] * 7 / n_hours for hour in range(24)] + [
self.hours[i][0] * 7 / n_hours
]
ax.plot(
times, hours, label=calendar.day_name[i], linestyle="--", linewidth=0.8
)
hours = [day[hour] / n_hours for hour in range(24)] + [day[0] / n_hours]
ax.plot(times, hours, c="r", label="average", linewidth=1.5)
ax.set_xlabel("hour of day")
ax.set_xlim([0, 24])
ax.set_ylabel("average messages")
ax.legend(framealpha=0)
with BytesIO() as f:
plt.savefig(
f, format="png", facecolor=fig.get_facecolor(), edgecolor="none"
)
f.seek(0)
return [discord.File(f, f"{time.time()}-plot.png")]
def to_string(
self,
*,
@@ -43,8 +88,14 @@ class Frequency:
if delta.days == 0:
delta = timedelta(days=1)
total_msg = len(self.dates)
busiest_weekday = top_key(self.week)
busiest_hour = top_key(self.day)
week = {i: sum(self.hours[i].values()) for i in range(7)}
day = {j: sum(self.hours[i][j] for i in range(7)) for j in range(24)}
busiest_weekday = top_key(week)
busiest_hour = top_key(day)
quietest_weekday = top_key(week, reverse=True)
quietest_hour = top_key(day, reverse=True)
n_weekdays = delta.days // 7
if (
self.dates[0].weekday() <= busiest_weekday
@@ -55,20 +106,26 @@ class Frequency:
if self.dates[0].hour <= busiest_hour and self.dates[-1].hour >= busiest_hour:
n_hours += 1
ret = [
f"- **earliest message**: {str_datetime(self.dates[0])} ({from_now(self.dates[0])})",
f"- **latest message**: {str_datetime(self.dates[-1])} ({from_now(self.dates[-1])})",
f"- **earliest message**: {from_now(self.dates[0])}",
f"- **latest message**: {from_now(self.dates[-1])}",
f"- **messages/day**: {precise(total_msg/delta.days, precision=3)}",
f"- **busiest day of week**: {calendar.day_name[busiest_weekday]} (~{precise(self.week[busiest_weekday]/n_weekdays, precision=3)} msg, {percent(self.week[busiest_weekday]/total_msg)})",
f"- **busiest day ever**: {str_date(self.busiest_day)} ({from_now(self.busiest_day)}, {self.busiest_day_count} msg)"
f"- **busiest day of week**: {calendar.day_name[busiest_weekday]} (~{precise(week[busiest_weekday]/n_weekdays, precision=3)} msg, {percent(week[busiest_weekday]/total_msg)})",
f"- **quietest day of week**: {calendar.day_name[quietest_weekday]} (~{precise(week[quietest_weekday]/n_weekdays, precision=3)} msg, {percent(week[quietest_weekday]/total_msg)})"
if week[quietest_weekday] > 0
else "",
f"- **busiest day ever**: {from_now(self.busiest_day)} ({self.busiest_day_count} msg)"
if self.busiest_day is not None
else "",
f"- **messages/hour**: {precise(total_msg*3600/delta.total_seconds(), precision=3)}",
f"- **busiest hour of day**: {busiest_hour:0>2}:00 (~{precise(self.day[busiest_hour]/n_hours, precision=3)} msg, {percent(self.day[busiest_hour]/total_msg)})",
f"- **busiest hour ever**: {str_datetime(self.busiest_hour)} ({from_now(self.busiest_hour)}, {self.busiest_hour_count} msg)",
f"- **longest break**: {plural(round(self.longest_break.total_seconds()/3600), 'hour')} ({plural(self.longest_break.days,'day')}) from {str_datetime(self.longest_break_start)} ({from_now(self.longest_break_start)})",
f"- **busiest hour of day**: {busiest_hour:0>2}:00 (~{precise(day[busiest_hour]/n_hours, precision=3)} msg, {percent(day[busiest_hour]/total_msg)})",
f"- **quietest hour of day**: {quietest_hour:0>2}:00 (~{precise(day[quietest_hour]/n_hours, precision=3)} msg, {percent(day[quietest_hour]/total_msg)})"
if day[quietest_hour] > 0
else "",
f"- **busiest hour ever**: {from_now(self.busiest_hour)} ({self.busiest_hour_count} msg)",
f"- **longest break**: {plural(round(self.longest_break.total_seconds()/3600), 'hour')} ({plural(self.longest_break.days,'day')}), started {from_now(self.longest_break_start)}",
f"- **avg. streak**: {precise(sum(self.streaks)/len(self.streaks), precision=3)} msg",
f"- **longest streak**: {self.longest_streak:,} msg from {str_datetime(self.longest_streak_start)} ({from_now(self.longest_streak_start)})"
f"- **longest streak**: {self.longest_streak:,} msg, started {from_now(self.longest_streak_start)}"
if member_specific
else f"- **longest streak**: {mention(self.longest_streak_author)} ({self.longest_streak:,} msg from {str_datetime(self.longest_streak_start)}, {from_now(self.longest_streak_start)})",
else f"- **longest streak**: {mention(self.longest_streak_author)} ({self.longest_streak:,} msg, started {from_now(self.longest_streak_start)})",
]
return ret
src/data_types/history.py
+2 -3
View File
@@ -6,7 +6,6 @@ import random
from utils import (
mention,
from_now,
str_datetime,
message_link,
SPLIT_TOKEN,
FilterLevel,
@@ -78,7 +77,7 @@ class History:
return [
intro,
f"{str_datetime(message.created_at)} ({from_now(message.created_at)}) {mention(message.author)} sent:",
f"{from_now(message.created_at)}, {mention(message.author)} sent:",
f"<{message_link(message)}>",
SPLIT_TOKEN,
image,
@@ -107,7 +106,7 @@ class History:
return [
intro,
f"{str_datetime(message.created_at)} ({from_now(message.created_at)}) {mention(message.author)} said:",
f"{from_now(message.created_at)}, {mention(message.author)} said:",
*text,
f"<{message_link(message)}>",
]
src/logs/channel_logs.py
+1 -1
View File
@@ -51,7 +51,7 @@ class ChannelLogs:
def preload(self, channel: discord.TextChannel):
self.name = channel.name
self.channel = channel
@property
def sorted_messages(self):
return sorted(self.messages)
src/logs/message_log.py
+1 -1
View File
@@ -66,7 +66,7 @@ class MessageLog:
def __eq__(self, other: object) -> bool:
return isinstance(other, self.__class__) and other.id == self.id
def __gt__(self, other: 'MessageLog') -> bool:
def __gt__(self, other: "MessageLog") -> bool:
return self.created_at > other.created_at
def __hash__(self) -> int:
src/main.py
+1 -1
View File
@@ -18,7 +18,7 @@ emojis.load_emojis()
bot = Bot(
"Discord Analyst",
"1.15.3",
"1.16",
alias="%",
)
src/scanners/frequency_scanner.py
+18 -8
View File
@@ -14,11 +14,17 @@ from utils import generate_help
class FrequencyScanner(Scanner):
@staticmethod
def help() -> str:
return generate_help("freq", "Show frequency-related statistics")
return generate_help(
"freq",
"(BETA) Show frequency-related statistics",
args=[
"graph - plot hours of week",
],
)
def __init__(self):
super().__init__(
valid_args=["all", "everyone"],
valid_args=["all", "everyone", "graph"],
help=FrequencyScanner.help(),
intro_context="Frequency",
)
@@ -26,6 +32,8 @@ class FrequencyScanner(Scanner):
async def init(self, message: discord.Message, *args: str) -> bool:
self.freq = Frequency()
self.all_messages = "all" in args or "everyone" in args
self.member_specific = len(self.members) > 0
self.to_graph = "graph" in args
return True
def compute_message(self, channel: ChannelLogs, message: MessageLog):
@@ -35,10 +43,13 @@ class FrequencyScanner(Scanner):
def get_results(self, intro: str) -> List[str]:
FrequencyScanner.compute_results(self.freq)
res = [intro]
res += self.freq.to_string(
member_specific=self.member_specific,
)
if self.to_graph:
res = self.freq.to_graph()
else:
res = [intro]
res += self.freq.to_string(
member_specific=self.member_specific,
)
return res
@staticmethod
@@ -90,8 +101,7 @@ class FrequencyScanner(Scanner):
freq.longest_break_start = latest
latest = date
# calculate busiest weekday / hours
freq.week[date.weekday()] += 1
freq.day[date.hour] += 1
freq.hours[date.weekday()][date.hour] += 1
# calculate busiest day ever
start_delta = date - freq.dates[0]
if start_delta.days > current_day:
src/scanners/scanner.py
+13 -3
View File
@@ -92,7 +92,9 @@ class Scanner(ABC):
dates = []
for i, arg in enumerate(args[1:]):
skip_check = False
if self.all_args and f"'{arg}'" in message.content or f"\"{arg}\"" in message.content:
if self.all_args and (
f"'{arg}'" in message.content or f'"{arg}"' in message.content
):
self.other_args += [arg]
elif re.match(r"^<@!?\d+>$", arg):
arg = arg[3:-1] if "!" in arg else arg[2:-1]
@@ -286,15 +288,20 @@ class Scanner(ABC):
if self.mention_users
else discord.AllowedMentions.none()
)
file = None
for r in results:
if r:
if isinstance(r, int) and r == SPLIT_TOKEN:
if isinstance(r, discord.File):
file = r
elif isinstance(r, int) and r == SPLIT_TOKEN:
await message.channel.send(
response,
reference=message if first else None,
allowed_mentions=allowed_mentions,
file=file,
)
first = False
file = None
response = ""
elif isinstance(r, str):
if len(response + "\n" + r) > 2000:
@@ -302,15 +309,18 @@ class Scanner(ABC):
response,
reference=message if first else None,
allowed_mentions=allowed_mentions,
file=file,
)
first = False
file = None
response = ""
response += "\n" + r
if len(response) > 0:
if len(response) > 0 or file is not None:
await message.channel.send(
response,
reference=message if first else None,
allowed_mentions=allowed_mentions,
file=file,
)
command_cache.cache(self, message, args)
# Delete custom progress message
src/utils/utils.py
+6 -10
View File
@@ -6,6 +6,7 @@ import discord
import math
from datetime import datetime, timedelta
import re
import time
import dateutil.parser
from dateutil.relativedelta import relativedelta
@@ -177,13 +178,13 @@ def no_duplicate(seq: list) -> list:
def top_key(
d: Dict[Union[str, int], int], key: Optional[Callable] = None
d: Dict[Union[str, int], int], key: Optional[Callable] = None, reverse: bool = False
) -> Union[str, int]:
if len(d) == 0:
return None
if key is None:
key = lambda k: d[k]
return sorted(d, key=key)[-1]
return sorted(d, key=key, reverse=reverse)[-1]
def val_sum(d: Dict[Any, int]) -> int:
@@ -290,11 +291,11 @@ def parse_time(src: str) -> datetime:
def str_date(date: datetime) -> str:
return date.strftime("%d %b. %Y") # 12 Jun. 2018
return f"<t:{int(time.mktime(date.timetuple()))}:D>"
def str_datetime(date: datetime) -> str:
return date.strftime("%H:%M, %d %b. %Y") # 12:05, 12 Jun. 2018
return f"<t:{int(time.mktime(date.timetuple()))}:f>"
def str_delta(delay: timedelta) -> str:
@@ -322,12 +323,7 @@ def str_delta(delay: timedelta) -> str:
def from_now(src: Optional[datetime]) -> str:
if src is None:
return "never"
output = str_delta(datetime.utcnow() - src)
if output == "no time":
return "now"
elif output == "one day":
return "yesterday"
return output + " ago"
return f"<t:{int(time.mktime(src.timetuple()))}:R>"
# APP SPECIFIC
test-requirements.txt
+3
View File
@@ -0,0 +1,3 @@
pytest~=6.2.3
pytest-cov
coveralls
tests/__init__.py
View File
tests/integration/__init__.py
View File
tests/integration/scanners/__init__.py
View File
tests/integration/scanners/test_first_scanner.py
+90
View File
@@ -0,0 +1,90 @@
from unittest import TestCase
from unittest.mock import MagicMock
from src.scanners import FirstScanner
from datetime import datetime, timedelta
from tests.utils import AsyncTestCase, fake_message
class TestFirstScanner(AsyncTestCase):
def test_help(self):
self.assertGreater(len(FirstScanner.help()), 0)
self.assertIn("%first", FirstScanner.help())
def test_empty_no_messages(self):
scanner = FirstScanner()
command_msg = MagicMock()
self._await(scanner.init(command_msg, []))
results = self._await(scanner.get_results(""))
self.assertListEqual(["There was no messages matching your filters"], results)
def test_empty_filtered(self):
scanner = FirstScanner()
scanner.raw_members = [1]
self._await(scanner.init(fake_message(), []))
messages = [fake_message(author=2), fake_message(author=3)]
for msg in messages:
scanner.compute_message(msg.channel, msg)
results = self._await(scanner.get_results(""))
self.assertListEqual(["There was no messages matching your filters"], results)
def test_normal(self):
scanner = FirstScanner()
self._await(scanner.init(fake_message(), []))
messages = [
fake_message(id=1, created_at=timedelta(days=-2)),
fake_message(id=2, created_at=timedelta(days=-3)),
fake_message(id=3, created_at=timedelta(days=-1)),
]
for msg in messages:
scanner.compute_message(msg.channel, msg)
results = self._await(scanner.get_results(""))
expected = messages[1]
self.assertListEqual(
[
"First message out of 3",
f"{expected.created_at.strftime('%H:%M, %d %b. %Y')} (2 days ago) <@1> said:",
f"> {expected.content}",
"<https://discord.com/channels/1/1/2>",
],
results,
)
def test_filtered(self):
scanner = FirstScanner()
scanner.raw_members = [1]
self._await(scanner.init(fake_message(), []))
messages = [
fake_message(id=1, author=1, created_at=timedelta(days=-2)),
fake_message(id=2, author=2, created_at=timedelta(days=-3)),
fake_message(id=3, author=1, created_at=timedelta(days=-1)),
]
for msg in messages:
scanner.compute_message(msg.channel, msg)
results = self._await(scanner.get_results(""))
expected = messages[0]
self.assertListEqual(
[
"First message out of 2",
f"{expected.created_at.strftime('%H:%M, %d %b. %Y')} (yesterday) <@1> said:",
f"> {expected.content}",
"<https://discord.com/channels/1/1/1>",
],
results,
)
tests/unit/__init__.py
View File
tests/utils.py
+99
View File
@@ -0,0 +1,99 @@
from typing import List, Optional, Dict, Union
from unittest import TestCase
import asyncio
from datetime import datetime, timedelta
from unittest.mock import MagicMock
import random
import string
class AsyncTestCase(TestCase):
def setUp(self):
self.loop = asyncio.new_event_loop()
asyncio.set_event_loop(None)
def tearDown(self):
self.loop.close()
def _await(self, fn):
return self.loop.run_until_complete(fn)
RANDOM_TEXT_CHARS = string.ascii_letters + string.digits + string.punctuation
def random_text(min_len: int = 3, max_len: int = 45):
return "".join(
random.choice(RANDOM_TEXT_CHARS)
for _ in range(random.randrange(min_len, max_len))
)
def fake_guild(id: int = 1):
return MagicMock(id=id)
def fake_channel(id: int = 1, name: str = "fake-channel"):
return MagicMock(id=id, name=name, guild=fake_guild())
def fake_message(
id: int = 1,
channel_id: int = 1,
channel_name: str = "fake-channel",
created_at: Optional[Union[datetime, timedelta]] = None,
edited_at: Optional[datetime] = None,
author: int = 1,
pinned: bool = False,
mention_everyone: bool = False,
tts: bool = False,
bot: bool = False,
content: Optional[str] = None,
mentions: Optional[List[int]] = None,
reference: Optional[int] = None,
role_mentions: Optional[List[int]] = None,
channel_mentions: Optional[List[int]] = None,
image: bool = False,
attachment: bool = False,
embed: bool = False,
reactions: Optional[Dict[str, List[int]]] = None,
):
if created_at is None:
created_at = datetime.now() + timedelta(hours=random.randrange(-30 * 24, 0))
elif isinstance(created_at, timedelta):
created_at = datetime.now() + created_at
if isinstance(edited_at, timedelta):
edited_at = datetime.now() + edited_at
if content is None:
content = random_text()
if mentions is None:
mentions = []
if role_mentions is None:
role_mentions = []
if channel_mentions is None:
channel_mentions = []
if reactions is None:
reactions = {}
return MagicMock(
id=id,
channel=fake_channel(channel_id, channel_name),
created_at=created_at,
edited_at=edited_at,
author=author,
pinned=pinned,
mention_everyone=mention_everyone,
tts=tts,
bot=bot,
content=content,
mentions=mentions,
raw_mentions=mentions,
reference=reference,
role_mentions=role_mentions,
raw_role_mentions=role_mentions,
channel_mentions=channel_mentions,
raw_channel_mentions=channel_mentions,
image=image,
attachment=attachment,
embed=embed,
reactions=reactions,
)